2023-10-09T05:34:33.571 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T05:34:33.571 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/orig.config.yaml 2023-10-09T05:34:33.632 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T05:34:33.707 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7417513' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi107.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T05:34:33.708 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T05:34:33.708 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T05:34:33.709 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T05:34:33.710 INFO:teuthology.task.internal:Checking packages... 2023-10-09T05:34:33.733 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T05:34:33.733 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:34:33.733 INFO:teuthology.packaging:ref: None 2023-10-09T05:34:33.733 INFO:teuthology.packaging:tag: None 2023-10-09T05:34:33.733 INFO:teuthology.packaging:branch: reef 2023-10-09T05:34:33.733 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:34:33.733 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T05:34:33.932 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T05:34:33.932 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T05:34:33.939 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T05:34:33.939 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T05:34:33.963 INFO:teuthology.task.internal:Saving configuration 2023-10-09T05:34:33.989 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T05:34:33.995 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:34:34.017 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi107.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:29:18.766103', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:34:34.039 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:29:18.764250', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:34:34.039 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T05:34:34.046 INFO:teuthology.task.internal:roles: ubuntu@smithi107.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T05:34:34.046 INFO:teuthology.task.internal:roles: ubuntu@smithi150.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T05:34:34.046 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T05:34:34.102 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7fc1d83af0>, signals=[15]) 2023-10-09T05:34:34.103 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T05:34:34.109 INFO:teuthology.task.internal:Opening connections... 2023-10-09T05:34:34.110 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi107.front.sepia.ceph.com 2023-10-09T05:34:34.112 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi107.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:34:34.179 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi150.front.sepia.ceph.com 2023-10-09T05:34:34.180 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:34:34.250 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T05:34:34.257 DEBUG:teuthology.orchestra.run.smithi107:> uname -m 2023-10-09T05:34:34.277 INFO:teuthology.orchestra.run.smithi107.stdout:x86_64 2023-10-09T05:34:34.278 DEBUG:teuthology.orchestra.run.smithi107:> cat /etc/os-release 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:ID="rhel" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:ID_LIKE="fedora" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:VERSION_ID="8.6" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:PLATFORM_ID="platform:el8" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:ANSI_COLOR="0;31" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T05:34:34.336 INFO:teuthology.orchestra.run.smithi107.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T05:34:34.337 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T05:34:34.338 INFO:teuthology.lock.ops:Updating smithi107.front.sepia.ceph.com on lock server 2023-10-09T05:34:34.359 DEBUG:teuthology.orchestra.run.smithi150:> uname -m 2023-10-09T05:34:34.379 INFO:teuthology.orchestra.run.smithi150.stdout:x86_64 2023-10-09T05:34:34.379 DEBUG:teuthology.orchestra.run.smithi150:> cat /etc/os-release 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:ID="rhel" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:ID_LIKE="fedora" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION_ID="8.6" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:PLATFORM_ID="platform:el8" 2023-10-09T05:34:34.441 INFO:teuthology.orchestra.run.smithi150.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:ANSI_COLOR="0;31" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T05:34:34.442 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T05:34:34.443 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T05:34:34.443 INFO:teuthology.lock.ops:Updating smithi150.front.sepia.ceph.com on lock server 2023-10-09T05:34:34.467 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T05:34:34.477 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T05:34:34.481 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T05:34:34.481 DEBUG:teuthology.orchestra.run.smithi107:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:34:34.484 DEBUG:teuthology.orchestra.run.smithi150:> test '!' -e /home/ubuntu/cephtest 2023-10-09T05:34:34.502 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T05:34:34.509 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T05:34:34.509 DEBUG:teuthology.orchestra.run.smithi107:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:34:34.543 DEBUG:teuthology.orchestra.run.smithi150:> test -z $(ls -A /var/lib/ceph) 2023-10-09T05:34:34.572 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T05:34:34.631 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T05:34:34.646 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T05:34:34.646 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T05:34:34.647 DEBUG:teuthology.orchestra.run.smithi107:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:34:34.647 DEBUG:teuthology.orchestra.run.smithi150:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T05:34:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.665 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2023-10-09T05:34:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:34.666 DEBUG:teuthology.orchestra.run.smithi107:> uname -r 2023-10-09T05:34:34.724 INFO:teuthology.orchestra.run.smithi107.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:34.724 INFO:teuthology.task.kernel:Running kernel on smithi107: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:34.725 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum install -y kernel 2023-10-09T05:34:34.726 INFO:teuthology.orchestra.run.smithi150.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:34.726 INFO:teuthology.task.kernel:Running kernel on smithi150: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:34.726 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum install -y kernel 2023-10-09T05:34:35.911 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:35.911 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2023-10-09T05:34:35.911 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:35.911 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2023-10-09T05:34:37.347 INFO:teuthology.orchestra.run.smithi150.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-10-09T05:34:38.064 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 23 MB/s | 16 MB 00:00 2023-10-09T05:34:38.613 INFO:teuthology.orchestra.run.smithi107.stdout:Copr repo for python3-asyncssh owned by ceph 2.6 kB/s | 3.5 kB 00:01 2023-10-09T05:34:39.922 INFO:teuthology.orchestra.run.smithi107.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2023-10-09T05:34:41.531 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 507 kB/s | 24 kB 00:00 2023-10-09T05:34:42.175 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T05:34:42.226 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:34:42.227 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:34:42.227 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:34:42.358 DEBUG:teuthology.orchestra.run.smithi150:> echo no | sudo yum reinstall kernel || true 2023-10-09T05:34:42.692 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:42.692 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2023-10-09T05:34:42.952 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:01 ago on Mon 09 Oct 2023 05:34:41 AM UTC. 2023-10-09T05:34:42.993 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:34:43.035 INFO:teuthology.orchestra.run.smithi150.stderr:Error: No packages marked for reinstall. 2023-10-09T05:34:43.081 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum reinstall -y kernel || true 2023-10-09T05:34:43.413 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:43.414 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2023-10-09T05:34:43.422 INFO:teuthology.orchestra.run.smithi107.stdout:lab-extras 489 kB/s | 24 kB 00:00 2023-10-09T05:34:43.667 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 05:34:41 AM UTC. 2023-10-09T05:34:43.707 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:34:43.740 INFO:teuthology.orchestra.run.smithi150.stderr:Error: No packages marked for reinstall. 2023-10-09T05:34:43.786 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T05:34:43.866 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:43.866 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:43.866 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T05:34:43.866 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T05:34:43.867 DEBUG:teuthology.orchestra.run.smithi150:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:34:43.903 INFO:teuthology.orchestra.run.smithi150.stdout:ttyS1 2023-10-09T05:34:43.915 DEBUG:teuthology.parallel:result is None 2023-10-09T05:34:44.166 INFO:teuthology.orchestra.run.smithi107.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T05:34:44.226 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:34:44.227 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:34:44.227 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:34:44.355 DEBUG:teuthology.orchestra.run.smithi107:> echo no | sudo yum reinstall kernel || true 2023-10-09T05:34:44.688 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:44.688 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2023-10-09T05:34:44.947 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:00:01 ago on Mon 09 Oct 2023 05:34:43 AM UTC. 2023-10-09T05:34:44.987 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:34:45.017 INFO:teuthology.orchestra.run.smithi107.stderr:Error: No packages marked for reinstall. 2023-10-09T05:34:45.063 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum reinstall -y kernel || true 2023-10-09T05:34:45.392 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:34:45.392 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2023-10-09T05:34:45.646 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 05:34:43 AM UTC. 2023-10-09T05:34:45.688 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T05:34:45.709 INFO:teuthology.orchestra.run.smithi107.stderr:Error: No packages marked for reinstall. 2023-10-09T05:34:45.755 DEBUG:teuthology.orchestra.run.smithi107:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T05:34:45.834 INFO:teuthology.orchestra.run.smithi107.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:45.835 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T05:34:45.835 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T05:34:45.835 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T05:34:45.835 DEBUG:teuthology.orchestra.run.smithi107:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T05:34:45.867 INFO:teuthology.orchestra.run.smithi107.stdout:ttyS1 2023-10-09T05:34:45.881 DEBUG:teuthology.parallel:result is None 2023-10-09T05:34:45.881 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T05:34:45.888 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T05:34:45.888 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:34:45.922 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T05:34:45.943 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T05:34:45.950 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T05:34:45.958 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T05:34:45.959 DEBUG:teuthology.orchestra.run.smithi107:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:34:45.980 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T05:34:46.031 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T05:34:46.037 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T05:34:46.038 DEBUG:teuthology.orchestra.run.smithi107:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:34:46.040 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T05:34:46.064 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:34:46.075 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:34:46.091 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:34:46.103 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T05:34:46.105 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T05:34:46.112 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T05:34:46.112 DEBUG:teuthology.orchestra.run.smithi107:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:34:46.119 DEBUG:teuthology.orchestra.run.smithi150:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T05:34:46.149 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T05:34:46.158 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T05:34:46.159 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:34:46.186 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T05:34:46.207 DEBUG:teuthology.orchestra.run.smithi107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:34:46.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:34:46.336 DEBUG:teuthology.orchestra.run.smithi107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:34:46.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:34:46.459 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:34:46.459 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:34:46.526 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:34:46.571 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T05:34:46.628 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:34:46.715 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T05:34:46.762 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:34:46.762 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T05:34:46.832 DEBUG:teuthology.orchestra.run.smithi107:> sudo service rsyslog restart 2023-10-09T05:34:46.835 DEBUG:teuthology.orchestra.run.smithi150:> sudo service rsyslog restart 2023-10-09T05:34:46.892 INFO:teuthology.orchestra.run.smithi107.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:34:46.969 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:34:47.165 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T05:34:47.173 INFO:teuthology.task.internal:Starting timer... 2023-10-09T05:34:47.173 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T05:34:47.183 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T05:34:47.190 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T05:34:47.191 DEBUG:teuthology.orchestra.run.smithi107:> sudo service auditd rotate 2023-10-09T05:34:47.267 INFO:teuthology.orchestra.run.smithi107.stdout:Rotating logs: 2023-10-09T05:34:47.269 DEBUG:teuthology.orchestra.run.smithi150:> sudo service auditd rotate 2023-10-09T05:34:47.328 INFO:teuthology.orchestra.run.smithi150.stdout:Rotating logs: 2023-10-09T05:34:47.330 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T05:34:47.330 DEBUG:teuthology.orchestra.run.smithi107:> /usr/sbin/getenforce 2023-10-09T05:34:47.400 INFO:teuthology.orchestra.run.smithi107.stdout:Permissive 2023-10-09T05:34:47.400 DEBUG:teuthology.orchestra.run.smithi150:> /usr/sbin/getenforce 2023-10-09T05:34:47.419 INFO:teuthology.orchestra.run.smithi150.stdout:Permissive 2023-10-09T05:34:47.419 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi107.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi150.front.sepia.ceph.com': 'permissive'} 2023-10-09T05:34:47.419 DEBUG:teuthology.orchestra.run.smithi107:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:34:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.445 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:34:47.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:34:47.487 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T05:34:47.488 DEBUG:teuthology.orchestra.run.smithi107:> sudo /usr/sbin/setenforce permissive 2023-10-09T05:34:47.515 DEBUG:teuthology.orchestra.run.smithi150:> sudo /usr/sbin/setenforce permissive 2023-10-09T05:34:47.562 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T05:34:47.573 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-10-09T05:34:47.573 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T05:34:47.588 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T05:34:47.589 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi107.front.sepia.ceph.com,smithi150.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T05:43:45.792 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi107.front.sepia.ceph.com'), Remote(name='ubuntu@smithi150.front.sepia.ceph.com')] 2023-10-09T05:43:45.807 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi107.front.sepia.ceph.com' 2023-10-09T05:43:45.808 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi107.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:43:45.876 DEBUG:teuthology.orchestra.run.smithi107:> true 2023-10-09T05:43:45.947 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi107.front.sepia.ceph.com' 2023-10-09T05:43:45.948 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2023-10-09T05:43:45.948 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T05:43:46.016 DEBUG:teuthology.orchestra.run.smithi150:> true 2023-10-09T05:43:46.083 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2023-10-09T05:43:46.084 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T05:43:46.094 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T05:43:46.095 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:43:46.095 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:43:46.097 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T05:43:46.098 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:43:46.126 INFO:teuthology.orchestra.run.smithi107.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:43:46.140 INFO:teuthology.orchestra.run.smithi107.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:43:46.156 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T05:43:46.170 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T05:43:46.209 INFO:teuthology.orchestra.run.smithi107.stderr:sudo: ntpd: command not found 2023-10-09T05:43:46.220 INFO:teuthology.orchestra.run.smithi107.stdout:506 Cannot talk to daemon 2023-10-09T05:43:46.235 INFO:teuthology.orchestra.run.smithi107.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:43:46.245 INFO:teuthology.orchestra.run.smithi150.stderr:sudo: ntpd: command not found 2023-10-09T05:43:46.248 INFO:teuthology.orchestra.run.smithi107.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:43:46.257 INFO:teuthology.orchestra.run.smithi150.stdout:506 Cannot talk to daemon 2023-10-09T05:43:46.271 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T05:43:46.285 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T05:43:46.323 INFO:teuthology.orchestra.run.smithi107.stderr:bash: ntpq: command not found 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:=============================================================================== 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.326 INFO:teuthology.orchestra.run.smithi107.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.368 INFO:teuthology.orchestra.run.smithi150.stderr:bash: ntpq: command not found 2023-10-09T05:43:46.371 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:43:46.371 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2023-10-09T05:43:46.371 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.372 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.372 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.372 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:43:46.373 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T05:43:46.400 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T05:43:46.401 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi107.front.sepia.ceph.com 2023-10-09T05:43:46.401 DEBUG:teuthology.orchestra.run.smithi107:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:43:46.401 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi150.front.sepia.ceph.com 2023-10-09T05:43:46.402 DEBUG:teuthology.orchestra.run.smithi150:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T05:43:46.776 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:43:46.796 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:43:47.596 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:29 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:47.647 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:29 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:49.314 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:43:49.315 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:43:49.315 INFO:teuthology.orchestra.run.smithi107.stdout: Package Architecture Version Repository Size 2023-10-09T05:43:49.315 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout:Resetting modules: 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout: container-tools 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:43:49.316 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:43:49.524 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:43:49.525 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:43:49.525 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2023-10-09T05:43:49.525 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:43:49.525 INFO:teuthology.orchestra.run.smithi150.stdout:Resetting modules: 2023-10-09T05:43:49.526 INFO:teuthology.orchestra.run.smithi150.stdout: container-tools 2023-10-09T05:43:49.526 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:43:49.526 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:43:49.526 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:43:49.526 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:43:50.265 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:50.490 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:32 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:53.023 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:43:53.239 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:43:53.480 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:43:53.697 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:43:54.345 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:35 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:54.623 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:36 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:43:56.047 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout:========================================================================================================================== 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout:========================================================================================================================== 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout:Installing group/module packages: 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:43:56.139 INFO:teuthology.orchestra.run.smithi107.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout:Downgrading: 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:43:56.140 INFO:teuthology.orchestra.run.smithi107.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:Installing module profiles: 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: container-tools/common 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:Enabling module streams: 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: container-tools 3.0 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:========================================================================================================================== 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:Install 7 Packages 2023-10-09T05:43:56.141 INFO:teuthology.orchestra.run.smithi107.stdout:Downgrade 12 Packages 2023-10-09T05:43:56.142 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:43:56.142 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 51 M 2023-10-09T05:43:56.142 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:43:56.231 INFO:teuthology.orchestra.run.smithi107.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 599 kB/s | 53 kB 00:00 2023-10-09T05:43:56.248 INFO:teuthology.orchestra.run.smithi107.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 488 kB/s | 51 kB 00:00 2023-10-09T05:43:56.273 INFO:teuthology.orchestra.run.smithi107.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 697 kB/s | 91 kB 00:00 2023-10-09T05:43:56.300 INFO:teuthology.orchestra.run.smithi107.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-10-09T05:43:56.328 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:43:56.348 INFO:teuthology.orchestra.run.smithi107.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.3 MB/s | 511 kB 00:00 2023-10-09T05:43:56.374 INFO:teuthology.orchestra.run.smithi107.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 956 kB/s | 72 kB 00:00 2023-10-09T05:43:56.407 INFO:teuthology.orchestra.run.smithi107.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.7 MB/s | 56 kB 00:00 2023-10-09T05:43:56.420 INFO:teuthology.orchestra.run.smithi150.stdout:========================================================================================================================== 2023-10-09T05:43:56.420 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:========================================================================================================================== 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:Installing group/module packages: 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:Downgrading: 2023-10-09T05:43:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout:Installing module profiles: 2023-10-09T05:43:56.422 INFO:teuthology.orchestra.run.smithi150.stdout: container-tools/common 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:Enabling module streams: 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout: container-tools 3.0 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:========================================================================================================================== 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:Install 7 Packages 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:Downgrade 12 Packages 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:43:56.423 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 51 M 2023-10-09T05:43:56.424 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:43:56.458 INFO:teuthology.orchestra.run.smithi107.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-10-09T05:43:56.483 INFO:teuthology.orchestra.run.smithi107.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 677 kB/s | 51 kB 00:00 2023-10-09T05:43:56.504 INFO:teuthology.orchestra.run.smithi150.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.1 MB/s | 91 kB 00:00 2023-10-09T05:43:56.529 INFO:teuthology.orchestra.run.smithi150.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 507 kB/s | 53 kB 00:00 2023-10-09T05:43:56.554 INFO:teuthology.orchestra.run.smithi150.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 396 kB/s | 51 kB 00:00 2023-10-09T05:43:56.590 INFO:teuthology.orchestra.run.smithi107.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 3.0 MB/s | 322 kB 00:00 2023-10-09T05:43:56.596 INFO:teuthology.orchestra.run.smithi150.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 5.4 MB/s | 511 kB 00:00 2023-10-09T05:43:56.621 INFO:teuthology.orchestra.run.smithi150.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.0 MB/s | 69 kB 00:00 2023-10-09T05:43:56.642 INFO:teuthology.orchestra.run.smithi107.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 786 kB/s | 41 kB 00:00 2023-10-09T05:43:56.647 INFO:teuthology.orchestra.run.smithi150.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-10-09T05:43:56.706 INFO:teuthology.orchestra.run.smithi150.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 964 kB/s | 56 kB 00:00 2023-10-09T05:43:56.764 INFO:teuthology.orchestra.run.smithi150.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 14 MB/s | 2.9 MB 00:00 2023-10-09T05:43:56.767 INFO:teuthology.orchestra.run.smithi107.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 26 MB/s | 11 MB 00:00 2023-10-09T05:43:56.789 INFO:teuthology.orchestra.run.smithi150.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 610 kB/s | 51 kB 00:00 2023-10-09T05:43:56.840 INFO:teuthology.orchestra.run.smithi150.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-10-09T05:43:56.868 INFO:teuthology.orchestra.run.smithi107.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-10-09T05:43:56.881 INFO:teuthology.orchestra.run.smithi150.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 982 kB/s | 41 kB 00:00 2023-10-09T05:43:57.035 INFO:teuthology.orchestra.run.smithi107.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 24 MB/s | 6.4 MB 00:00 2023-10-09T05:43:57.040 INFO:teuthology.orchestra.run.smithi150.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 26 MB/s | 11 MB 00:00 2023-10-09T05:43:57.107 INFO:teuthology.orchestra.run.smithi150.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-10-09T05:43:57.143 INFO:teuthology.orchestra.run.smithi107.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-10-09T05:43:57.194 INFO:teuthology.orchestra.run.smithi107.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 6.7 MB/s | 1.1 MB 00:00 2023-10-09T05:43:57.218 INFO:teuthology.orchestra.run.smithi107.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.4 MB/s | 185 kB 00:00 2023-10-09T05:43:57.244 INFO:teuthology.orchestra.run.smithi107.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-10-09T05:43:57.274 INFO:teuthology.orchestra.run.smithi150.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 27 MB/s | 6.4 MB 00:00 2023-10-09T05:43:57.383 INFO:teuthology.orchestra.run.smithi150.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-10-09T05:43:57.441 INFO:teuthology.orchestra.run.smithi150.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 6.4 MB/s | 1.1 MB 00:00 2023-10-09T05:43:57.467 INFO:teuthology.orchestra.run.smithi150.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.2 MB/s | 185 kB 00:00 2023-10-09T05:43:57.477 INFO:teuthology.orchestra.run.smithi107.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 19 MB/s | 19 MB 00:01 2023-10-09T05:43:57.478 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:43:57.478 INFO:teuthology.orchestra.run.smithi107.stdout:Total 38 MB/s | 51 MB 00:01 2023-10-09T05:43:57.492 INFO:teuthology.orchestra.run.smithi150.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-10-09T05:43:57.684 INFO:teuthology.orchestra.run.smithi150.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 21 MB/s | 19 MB 00:00 2023-10-09T05:43:57.684 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:43:57.684 INFO:teuthology.orchestra.run.smithi150.stdout:Total 40 MB/s | 51 MB 00:01 2023-10-09T05:43:57.954 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:43:57.996 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:43:57.996 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:43:58.163 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:43:58.206 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:43:58.206 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:43:58.436 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:43:58.440 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:43:58.642 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:43:58.645 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:43:59.670 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:43:59.723 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:43:59.848 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:43:59.886 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T05:43:59.921 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:43:59.930 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:43:59.950 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:00.081 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:44:00.091 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T05:44:00.109 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:00.154 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:00.281 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:17.050 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:17.178 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T05:44:18.843 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:44:18.954 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T05:44:19.132 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:44:19.139 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T05:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:44:19.381 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T05:44:19.519 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:44:19.598 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T05:44:19.721 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:44:19.721 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:44:19.721 INFO:teuthology.orchestra.run.smithi150.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:44:19.721 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:44:19.826 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T05:44:19.827 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T05:44:19.827 INFO:teuthology.orchestra.run.smithi107.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T05:44:19.827 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:44:20.506 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:44:20.682 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T05:44:20.720 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:44:20.924 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T05:44:22.253 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:44:22.404 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T05:44:22.556 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:44:22.754 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T05:44:22.920 INFO:teuthology.orchestra.run.smithi150.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:44:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T05:44:23.219 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:44:23.482 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T05:44:23.938 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:44:24.259 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T05:44:24.695 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:44:24.902 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:44:25.017 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T05:44:25.115 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:44:25.290 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T05:44:25.301 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:44:25.409 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:44:25.410 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:44:25.523 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T05:44:25.706 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:44:25.740 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T05:44:25.872 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:44:25.872 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:44:25.897 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T05:44:25.898 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:44:26.062 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:44:26.195 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:44:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T05:44:26.354 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:44:26.445 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T05:44:26.445 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:44:26.562 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:44:26.658 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T05:44:26.747 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:44:26.774 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:44:26.895 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:44:26.966 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T05:44:27.045 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:44:27.149 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T05:44:27.192 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:44:27.192 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:27.319 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T05:44:27.361 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:27.491 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T05:44:27.657 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T05:44:27.811 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T05:44:27.811 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:27.908 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:44:28.006 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:44:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:44:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:44:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:44:28.528 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T05:44:28.957 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T05:44:28.958 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T05:44:28.959 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T05:44:28.960 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T05:44:28.961 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T05:44:28.961 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T05:44:28.961 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T05:44:28.961 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T05:44:29.780 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:44:29.780 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:11 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:44:30.378 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T05:44:30.379 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:11 ago on Mon 09 Oct 2023 05:42:18 AM UTC. 2023-10-09T05:44:32.137 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:44:32.335 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:44:32.335 INFO:teuthology.orchestra.run.smithi150.stdout:Downgraded: 2023-10-09T05:44:32.335 INFO:teuthology.orchestra.run.smithi150.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.336 INFO:teuthology.orchestra.run.smithi150.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:44:32.337 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:44:32.338 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:44:32.516 DEBUG:teuthology.parallel:result is None 2023-10-09T05:44:32.706 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout:Downgraded: 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.901 INFO:teuthology.orchestra.run.smithi107.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.902 INFO:teuthology.orchestra.run.smithi107.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T05:44:32.903 INFO:teuthology.orchestra.run.smithi107.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T05:44:32.903 INFO:teuthology.orchestra.run.smithi107.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T05:44:32.903 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:44:32.903 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:44:33.105 DEBUG:teuthology.parallel:result is None 2023-10-09T05:44:33.105 INFO:teuthology.run_tasks:Running task install... 2023-10-09T05:44:33.115 DEBUG:teuthology.task.install:project ceph 2023-10-09T05:44:33.116 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T05:44:33.116 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:44:33.116 INFO:teuthology.task.install:Using flavor: default 2023-10-09T05:44:33.122 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T05:44:33.122 INFO:teuthology.task.install:extra packages: [] 2023-10-09T05:44:33.122 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:44:33.122 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:44:33.123 INFO:teuthology.packaging:ref: None 2023-10-09T05:44:33.123 INFO:teuthology.packaging:tag: None 2023-10-09T05:44:33.123 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:44:33.123 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:44:33.123 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:44:33.125 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T05:44:33.125 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:44:33.125 INFO:teuthology.packaging:ref: None 2023-10-09T05:44:33.125 INFO:teuthology.packaging:tag: None 2023-10-09T05:44:33.125 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:44:33.125 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:44:33.126 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:44:33.382 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T05:44:33.383 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T05:44:33.521 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T05:44:33.522 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T05:44:33.557 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:44:33.557 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:44:33.557 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:44:33.586 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:44:33.587 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:44:33.587 INFO:teuthology.packaging:ref: None 2023-10-09T05:44:33.587 INFO:teuthology.packaging:tag: None 2023-10-09T05:44:33.587 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:44:33.587 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:44:33.587 DEBUG:teuthology.orchestra.run.smithi107:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:44:33.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:44:33.704 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T05:44:33.704 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:44:33.704 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T05:44:33.733 DEBUG:teuthology.orchestra.run.smithi107:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:44:33.737 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T05:44:33.737 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:44:33.737 INFO:teuthology.packaging:ref: None 2023-10-09T05:44:33.737 INFO:teuthology.packaging:tag: None 2023-10-09T05:44:33.737 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:44:33.737 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:44:33.737 DEBUG:teuthology.orchestra.run.smithi150:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T05:44:33.799 INFO:teuthology.orchestra.run.smithi107.stdout:check_obsoletes = 1 2023-10-09T05:44:33.801 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum clean all 2023-10-09T05:44:33.808 DEBUG:teuthology.orchestra.run.smithi150:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T05:44:33.890 DEBUG:teuthology.orchestra.run.smithi150:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T05:44:33.958 INFO:teuthology.orchestra.run.smithi150.stdout:check_obsoletes = 1 2023-10-09T05:44:33.959 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum clean all 2023-10-09T05:44:34.169 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:44:34.332 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:44:34.431 INFO:teuthology.orchestra.run.smithi107.stdout:44 files removed 2023-10-09T05:44:34.476 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-radosgw 2023-10-09T05:44:34.623 INFO:teuthology.orchestra.run.smithi150.stdout:44 files removed 2023-10-09T05:44:34.670 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-radosgw 2023-10-09T05:44:34.805 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:44:35.003 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:44:35.487 INFO:teuthology.orchestra.run.smithi107.stdout:ceph packages for x86_64 256 kB/s | 78 kB 00:00 2023-10-09T05:44:35.702 INFO:teuthology.orchestra.run.smithi150.stdout:ceph packages for x86_64 233 kB/s | 78 kB 00:00 2023-10-09T05:44:35.787 INFO:teuthology.orchestra.run.smithi107.stdout:ceph noarch packages 34 kB/s | 9.1 kB 00:00 2023-10-09T05:44:36.021 INFO:teuthology.orchestra.run.smithi150.stdout:ceph noarch packages 32 kB/s | 9.1 kB 00:00 2023-10-09T05:44:36.188 INFO:teuthology.orchestra.run.smithi107.stdout:ceph source packages 3.8 kB/s | 1.4 kB 00:00 2023-10-09T05:44:36.330 INFO:teuthology.orchestra.run.smithi107.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2023-10-09T05:44:36.370 INFO:teuthology.orchestra.run.smithi150.stdout:ceph source packages 4.4 kB/s | 1.4 kB 00:00 2023-10-09T05:44:36.496 INFO:teuthology.orchestra.run.smithi150.stdout:Copr repo for python3-asyncssh owned by ceph 33 kB/s | 3.5 kB 00:00 2023-10-09T05:44:37.509 INFO:teuthology.orchestra.run.smithi107.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-10-09T05:44:38.047 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2023-10-09T05:44:41.056 INFO:teuthology.orchestra.run.smithi107.stdout:lab-extras 621 kB/s | 24 kB 00:00 2023-10-09T05:44:41.555 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 422 kB/s | 24 kB 00:00 2023-10-09T05:44:42.008 INFO:teuthology.orchestra.run.smithi107.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 60 MB/s | 53 MB 00:00 2023-10-09T05:44:42.440 INFO:teuthology.orchestra.run.smithi150.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 65 MB/s | 53 MB 00:00 2023-10-09T05:44:50.217 INFO:teuthology.orchestra.run.smithi107.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 55 MB/s | 47 MB 00:00 2023-10-09T05:44:50.558 INFO:teuthology.orchestra.run.smithi150.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-10-09T05:45:01.208 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:45:01.210 INFO:teuthology.orchestra.run.smithi107.stdout:==================================================================================================== 2023-10-09T05:45:01.210 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout:==================================================================================================== 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout:Upgrading: 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T05:45:01.211 INFO:teuthology.orchestra.run.smithi107.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T05:45:01.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout:==================================================================================================== 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout:Install 21 Packages 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout:Upgrade 2 Packages 2023-10-09T05:45:01.213 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:01.214 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 63 M 2023-10-09T05:45:01.214 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:45:01.598 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:45:01.601 INFO:teuthology.orchestra.run.smithi150.stdout:==================================================================================================== 2023-10-09T05:45:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:45:01.601 INFO:teuthology.orchestra.run.smithi150.stdout:==================================================================================================== 2023-10-09T05:45:01.601 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:45:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:45:01.602 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrading: 2023-10-09T05:45:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:45:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T05:45:01.602 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T05:45:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T05:45:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout:==================================================================================================== 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout:Install 21 Packages 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrade 2 Packages 2023-10-09T05:45:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:01.606 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 63 M 2023-10-09T05:45:01.606 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:45:01.825 INFO:teuthology.orchestra.run.smithi107.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 12 MB/s | 5.2 MB 00:00 2023-10-09T05:45:01.867 INFO:teuthology.orchestra.run.smithi107.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 586 kB/s | 24 kB 00:00 2023-10-09T05:45:01.942 INFO:teuthology.orchestra.run.smithi107.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 10 MB/s | 797 kB 00:00 2023-10-09T05:45:02.001 INFO:teuthology.orchestra.run.smithi107.stdout:(4/23): libradosstriper1-17.2.6-1303.gb2903cb0. 8.5 MB/s | 509 kB 00:00 2023-10-09T05:45:02.393 INFO:teuthology.orchestra.run.smithi107.stdout:(5/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 14 MB/s | 5.6 MB 00:00 2023-10-09T05:45:02.435 INFO:teuthology.orchestra.run.smithi107.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 1.1 MB/s | 45 kB 00:00 2023-10-09T05:45:02.494 INFO:teuthology.orchestra.run.smithi107.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.9 MB/s | 115 kB 00:00 2023-10-09T05:45:02.544 INFO:teuthology.orchestra.run.smithi107.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 3.7 MB/s | 190 kB 00:00 2023-10-09T05:45:02.551 INFO:teuthology.orchestra.run.smithi150.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 7.2 MB/s | 5.2 MB 00:00 2023-10-09T05:45:02.594 INFO:teuthology.orchestra.run.smithi107.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 7.2 MB/s | 370 kB 00:00 2023-10-09T05:45:02.601 INFO:teuthology.orchestra.run.smithi150.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 488 kB/s | 24 kB 00:00 2023-10-09T05:45:02.645 INFO:teuthology.orchestra.run.smithi107.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 7.0 MB/s | 358 kB 00:00 2023-10-09T05:45:02.701 INFO:teuthology.orchestra.run.smithi150.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 7.8 MB/s | 797 kB 00:00 2023-10-09T05:45:02.719 INFO:teuthology.orchestra.run.smithi107.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.4 MB/s | 106 kB 00:00 2023-10-09T05:45:02.785 INFO:teuthology.orchestra.run.smithi150.stdout:(4/23): libradosstriper1-17.2.6-1303.gb2903cb0. 6.0 MB/s | 509 kB 00:00 2023-10-09T05:45:03.112 INFO:teuthology.orchestra.run.smithi107.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 780 kB/s | 306 kB 00:00 2023-10-09T05:45:03.226 INFO:teuthology.orchestra.run.smithi107.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 513 kB/s | 59 kB 00:00 2023-10-09T05:45:03.318 INFO:teuthology.orchestra.run.smithi150.stdout:(5/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 10 MB/s | 5.6 MB 00:00 2023-10-09T05:45:03.337 INFO:teuthology.orchestra.run.smithi107.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 680 kB/s | 75 kB 00:00 2023-10-09T05:45:03.418 INFO:teuthology.orchestra.run.smithi150.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 448 kB/s | 45 kB 00:00 2023-10-09T05:45:03.486 INFO:teuthology.orchestra.run.smithi150.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 1.7 MB/s | 115 kB 00:00 2023-10-09T05:45:03.561 INFO:teuthology.orchestra.run.smithi150.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 2.5 MB/s | 190 kB 00:00 2023-10-09T05:45:03.587 INFO:teuthology.orchestra.run.smithi107.stdout:(15/23): thrift-0.13.0-2.el8.x86_64.rpm 6.9 MB/s | 1.7 MB 00:00 2023-10-09T05:45:03.646 INFO:teuthology.orchestra.run.smithi107.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 805 kB/s | 47 kB 00:00 2023-10-09T05:45:03.653 INFO:teuthology.orchestra.run.smithi150.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 3.9 MB/s | 370 kB 00:00 2023-10-09T05:45:03.688 INFO:teuthology.orchestra.run.smithi107.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-10-09T05:45:03.728 INFO:teuthology.orchestra.run.smithi150.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.7 MB/s | 358 kB 00:00 2023-10-09T05:45:03.755 INFO:teuthology.orchestra.run.smithi107.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 659 kB/s | 44 kB 00:00 2023-10-09T05:45:03.795 INFO:teuthology.orchestra.run.smithi150.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.6 MB/s | 106 kB 00:00 2023-10-09T05:45:03.797 INFO:teuthology.orchestra.run.smithi107.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 8.3 MB/s | 354 kB 00:00 2023-10-09T05:45:04.112 INFO:teuthology.orchestra.run.smithi150.stdout:(12/23): ceph-common-17.2.6-1303.gb2903cb0.el8. 10 MB/s | 23 MB 00:02 2023-10-09T05:45:04.131 INFO:teuthology.orchestra.run.smithi107.stdout:(20/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 10 MB/s | 3.5 MB 00:00 2023-10-09T05:45:04.320 INFO:teuthology.orchestra.run.smithi150.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 583 kB/s | 306 kB 00:00 2023-10-09T05:45:04.364 INFO:teuthology.orchestra.run.smithi107.stdout:(21/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 15 MB/s | 3.6 MB 00:00 2023-10-09T05:45:04.429 INFO:teuthology.orchestra.run.smithi150.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 693 kB/s | 75 kB 00:00 2023-10-09T05:45:04.454 INFO:teuthology.orchestra.run.smithi150.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 172 kB/s | 59 kB 00:00 2023-10-09T05:45:04.513 INFO:teuthology.orchestra.run.smithi150.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 808 kB/s | 47 kB 00:00 2023-10-09T05:45:04.555 INFO:teuthology.orchestra.run.smithi150.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-10-09T05:45:04.597 INFO:teuthology.orchestra.run.smithi150.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.0 MB/s | 44 kB 00:00 2023-10-09T05:45:04.639 INFO:teuthology.orchestra.run.smithi150.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 8.3 MB/s | 354 kB 00:00 2023-10-09T05:45:04.823 INFO:teuthology.orchestra.run.smithi150.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 4.4 MB/s | 1.7 MB 00:00 2023-10-09T05:45:04.948 INFO:teuthology.orchestra.run.smithi150.stdout:(21/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 11 MB/s | 3.5 MB 00:00 2023-10-09T05:45:05.123 INFO:teuthology.orchestra.run.smithi150.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 12 MB/s | 3.6 MB 00:00 2023-10-09T05:45:07.089 INFO:teuthology.orchestra.run.smithi107.stdout:(22/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 2.9 MB/s | 17 MB 00:05 2023-10-09T05:45:12.347 INFO:teuthology.orchestra.run.smithi107.stdout:(23/23): ceph-common-17.2.6-1303.gb2903cb0.el8. 2.1 MB/s | 23 MB 00:10 2023-10-09T05:45:12.350 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:45:12.350 INFO:teuthology.orchestra.run.smithi107.stdout:Total 5.7 MB/s | 63 MB 00:11 2023-10-09T05:45:12.395 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:45:12.440 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:45:12.440 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:45:12.457 INFO:teuthology.orchestra.run.smithi150.stdout:(23/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 1.6 MB/s | 17 MB 00:10 2023-10-09T05:45:12.459 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:45:12.459 INFO:teuthology.orchestra.run.smithi150.stdout:Total 5.8 MB/s | 63 MB 00:10 2023-10-09T05:45:12.505 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:45:12.550 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:45:12.550 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:45:12.794 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:45:12.798 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:45:12.904 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:45:12.907 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:45:14.040 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:45:14.132 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:45:14.354 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:45:14.413 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:45:14.729 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:45:14.804 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T05:45:14.929 INFO:teuthology.orchestra.run.smithi107.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:14.985 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:15.012 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:15.080 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:15.226 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:45:15.251 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:45:15.487 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:45:15.501 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T05:45:15.654 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:15.670 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:16.058 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:16.066 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:16.254 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:45:16.303 INFO:teuthology.orchestra.run.smithi107.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:45:16.334 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:45:16.424 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T05:45:16.504 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:45:16.571 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:45:16.587 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:45:16.687 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-09T05:45:16.893 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T05:45:16.999 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-09T05:45:17.570 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T05:45:17.721 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:17.731 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T05:45:17.792 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:17.912 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:17.947 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T05:45:17.965 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:18.088 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T05:45:18.113 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:45:18.272 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:45:18.331 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:45:18.496 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:45:18.558 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T05:45:18.721 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:45:18.796 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:45:18.819 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T05:45:18.955 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:45:19.147 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:45:19.545 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:45:19.629 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T05:45:19.650 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:45:19.758 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T05:45:19.813 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T05:45:20.014 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T05:45:20.282 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T05:45:20.470 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T05:45:23.605 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:23.837 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:24.236 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:24.415 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:24.503 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:24.688 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:45:24.752 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:45:24.977 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:25.107 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T05:45:25.398 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:45:25.468 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T05:45:25.643 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /sys 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /proc 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /dev 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /run 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /mnt 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /var/tmp 2023-10-09T05:45:43.698 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /home 2023-10-09T05:45:43.699 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /tmp 2023-10-09T05:45:43.699 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /dev 2023-10-09T05:45:43.699 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:44.334 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /sys 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /proc 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /dev 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /run 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /mnt 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /var/tmp 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /home 2023-10-09T05:45:44.391 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /tmp 2023-10-09T05:45:44.392 INFO:teuthology.orchestra.run.smithi107.stdout:skipping the directory /dev 2023-10-09T05:45:44.392 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:45.377 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:45:45.438 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:45:45.439 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:45:45.669 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:45:45.669 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:46.043 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:45:46.108 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T05:45:46.109 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:46.297 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T05:45:46.298 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:45:46.299 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T05:45:46.349 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T05:45:46.349 INFO:teuthology.orchestra.run.smithi107.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T05:45:46.966 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-09T05:45:46.967 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-09T05:45:46.968 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T05:45:46.968 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T05:45:46.968 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T05:45:47.669 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:47.670 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:57 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:45:48.411 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T05:45:48.411 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:00:59 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:45:50.083 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:45:50.268 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:50.268 INFO:teuthology.orchestra.run.smithi150.stdout:Upgraded: 2023-10-09T05:45:50.268 INFO:teuthology.orchestra.run.smithi150.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.268 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.269 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:50.270 INFO:teuthology.orchestra.run.smithi150.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T05:45:50.271 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:50.271 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:45:50.416 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-test 2023-10-09T05:45:50.765 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:45:51.215 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:45:51.396 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout:Upgraded: 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.397 INFO:teuthology.orchestra.run.smithi107.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T05:45:51.398 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.399 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.399 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:45:51.399 INFO:teuthology.orchestra.run.smithi107.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T05:45:51.399 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:51.399 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:45:51.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-test 2023-10-09T05:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:01 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:45:51.918 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:45:53.026 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout:============================================================================================ 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout:============================================================================================ 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:45:53.027 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:============================================================================================ 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:Install 6 Packages 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 53 M 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 233 M 2023-10-09T05:45:53.028 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:45:53.072 INFO:teuthology.orchestra.run.smithi150.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.5 MB/s | 70 kB 00:00 2023-10-09T05:45:53.230 INFO:teuthology.orchestra.run.smithi150.stdout:(2/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 859 kB/s | 175 kB 00:00 2023-10-09T05:45:53.272 INFO:teuthology.orchestra.run.smithi150.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.6 MB/s | 323 kB 00:00 2023-10-09T05:45:53.381 INFO:teuthology.orchestra.run.smithi150.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 1.9 MB/s | 202 kB 00:00 2023-10-09T05:45:53.414 INFO:teuthology.orchestra.run.smithi150.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.0 MB/s | 187 kB 00:00 2023-10-09T05:45:53.458 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:04 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:45:54.894 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:45:54.895 INFO:teuthology.orchestra.run.smithi107.stdout:============================================================================================ 2023-10-09T05:45:54.895 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:45:54.895 INFO:teuthology.orchestra.run.smithi107.stdout:============================================================================================ 2023-10-09T05:45:54.896 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:45:54.896 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T05:45:54.896 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:45:54.896 INFO:teuthology.orchestra.run.smithi107.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T05:45:54.896 INFO:teuthology.orchestra.run.smithi107.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:45:54.897 INFO:teuthology.orchestra.run.smithi107.stdout:============================================================================================ 2023-10-09T05:45:54.898 INFO:teuthology.orchestra.run.smithi107.stdout:Install 6 Packages 2023-10-09T05:45:54.898 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:45:54.899 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 53 M 2023-10-09T05:45:54.899 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 233 M 2023-10-09T05:45:54.899 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:45:54.942 INFO:teuthology.orchestra.run.smithi107.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.5 MB/s | 70 kB 00:00 2023-10-09T05:45:55.159 INFO:teuthology.orchestra.run.smithi107.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.5 MB/s | 323 kB 00:00 2023-10-09T05:45:55.285 INFO:teuthology.orchestra.run.smithi107.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.5 MB/s | 187 kB 00:00 2023-10-09T05:45:55.371 INFO:teuthology.orchestra.run.smithi107.stdout:(4/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 369 kB/s | 175 kB 00:00 2023-10-09T05:45:55.451 INFO:teuthology.orchestra.run.smithi107.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 1.2 MB/s | 202 kB 00:00 2023-10-09T05:45:56.381 INFO:teuthology.orchestra.run.smithi150.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 15 MB/s | 52 MB 00:03 2023-10-09T05:45:56.383 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:45:56.383 INFO:teuthology.orchestra.run.smithi150.stdout:Total 16 MB/s | 53 MB 00:03 2023-10-09T05:45:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:45:56.442 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:45:56.443 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:45:56.718 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:45:56.721 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:45:57.163 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:45:57.291 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:45:57.385 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:45:57.605 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:45:57.747 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:45:57.922 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:45:58.099 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:45:59.342 INFO:teuthology.orchestra.run.smithi107.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 12 MB/s | 52 MB 00:04 2023-10-09T05:45:59.344 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:45:59.344 INFO:teuthology.orchestra.run.smithi107.stdout:Total 12 MB/s | 53 MB 00:04 2023-10-09T05:45:59.381 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:45:59.401 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:45:59.401 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:45:59.674 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:45:59.677 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:46:00.164 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:46:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:46:00.406 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T05:46:00.651 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T05:46:00.836 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T05:46:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T05:46:01.194 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:46:03.030 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:46:03.287 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:46:03.515 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:46:04.638 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:46:04.638 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:14 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:06.128 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T05:46:06.432 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:46:06.662 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T05:46:06.663 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T05:46:06.663 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T05:46:06.663 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T05:46:06.663 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T05:46:06.663 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T05:46:07.857 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T05:46:07.858 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:18 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:10.424 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:46:10.602 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:10.603 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:46:10.740 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph 2023-10-09T05:46:11.075 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:11.994 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:46:12.172 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: jq-1.6-3.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:12.173 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:46:12.310 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph 2023-10-09T05:46:12.336 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:22 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:12.642 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:13.615 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:24 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:13.735 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:46:13.738 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================= 2023-10-09T05:46:13.738 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:46:13.738 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================= 2023-10-09T05:46:13.738 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:46:13.739 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:46:13.740 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:46:13.741 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================= 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout:Install 32 Packages 2023-10-09T05:46:13.742 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:13.743 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 28 M 2023-10-09T05:46:13.743 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 110 M 2023-10-09T05:46:13.743 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:46:14.265 INFO:teuthology.orchestra.run.smithi150.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 35 kB/s | 6.4 kB 00:00 2023-10-09T05:46:14.382 INFO:teuthology.orchestra.run.smithi150.stdout:(2/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 7.7 MB/s | 2.3 MB 00:00 2023-10-09T05:46:14.977 INFO:teuthology.orchestra.run.smithi150.stdout:(3/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 1.8 MB/s | 1.6 MB 00:00 2023-10-09T05:46:15.055 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================= 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================= 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T05:46:15.083 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T05:46:15.084 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T05:46:15.085 INFO:teuthology.orchestra.run.smithi107.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================= 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout:Install 32 Packages 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:15.086 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 28 M 2023-10-09T05:46:15.087 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 110 M 2023-10-09T05:46:15.087 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:46:15.107 INFO:teuthology.orchestra.run.smithi150.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 1.8 MB/s | 238 kB 00:00 2023-10-09T05:46:15.284 INFO:teuthology.orchestra.run.smithi107.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 34 kB/s | 6.4 kB 00:00 2023-10-09T05:46:15.324 INFO:teuthology.orchestra.run.smithi150.stdout:(5/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 18 MB/s | 17 MB 00:00 2023-10-09T05:46:15.400 INFO:teuthology.orchestra.run.smithi150.stdout:(6/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 3.5 MB/s | 3.9 MB 00:01 2023-10-09T05:46:15.627 INFO:teuthology.orchestra.run.smithi107.stdout:(2/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 4.3 MB/s | 2.3 MB 00:00 2023-10-09T05:46:15.650 INFO:teuthology.orchestra.run.smithi150.stdout:(7/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 80 kB/s | 44 kB 00:00 2023-10-09T05:46:15.668 INFO:teuthology.orchestra.run.smithi107.stdout:(3/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 2.7 MB/s | 1.6 MB 00:00 2023-10-09T05:46:15.793 INFO:teuthology.orchestra.run.smithi107.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 1.9 MB/s | 238 kB 00:00 2023-10-09T05:46:15.925 INFO:teuthology.orchestra.run.smithi150.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 311 kB/s | 185 kB 00:00 2023-10-09T05:46:15.927 INFO:teuthology.orchestra.run.smithi107.stdout:(5/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 6.1 MB/s | 3.9 MB 00:00 2023-10-09T05:46:16.001 INFO:teuthology.orchestra.run.smithi150.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 287 kB/s | 173 kB 00:00 2023-10-09T05:46:16.051 INFO:teuthology.orchestra.run.smithi150.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 89 kB/s | 11 kB 00:00 2023-10-09T05:46:16.117 INFO:teuthology.orchestra.run.smithi150.stdout:(11/32): python3-cherrypy-18.4.0-1.el8.noarch.r 823 kB/s | 384 kB 00:00 2023-10-09T05:46:16.143 INFO:teuthology.orchestra.run.smithi150.stdout:(12/32): python3-jaraco-functools-2.0-4.el8.noa 123 kB/s | 18 kB 00:00 2023-10-09T05:46:16.202 INFO:teuthology.orchestra.run.smithi150.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 323 kB/s | 49 kB 00:00 2023-10-09T05:46:16.235 INFO:teuthology.orchestra.run.smithi107.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 99 kB/s | 44 kB 00:00 2023-10-09T05:46:16.268 INFO:teuthology.orchestra.run.smithi150.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 390 kB/s | 59 kB 00:00 2023-10-09T05:46:16.343 INFO:teuthology.orchestra.run.smithi150.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 110 kB/s | 16 kB 00:00 2023-10-09T05:46:16.411 INFO:teuthology.orchestra.run.smithi150.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 132 kB/s | 19 kB 00:00 2023-10-09T05:46:16.444 INFO:teuthology.orchestra.run.smithi150.stdout:(17/32): python3-pecan-1.3.2-9.el8.noarch.rpm 939 kB/s | 283 kB 00:00 2023-10-09T05:46:16.495 INFO:teuthology.orchestra.run.smithi150.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 159 kB/s | 24 kB 00:00 2023-10-09T05:46:16.537 INFO:teuthology.orchestra.run.smithi150.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 232 kB/s | 29 kB 00:00 2023-10-09T05:46:16.544 INFO:teuthology.orchestra.run.smithi107.stdout:(7/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 18 MB/s | 17 MB 00:00 2023-10-09T05:46:16.562 INFO:teuthology.orchestra.run.smithi150.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 235 kB/s | 27 kB 00:00 2023-10-09T05:46:16.569 INFO:teuthology.orchestra.run.smithi107.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 290 kB/s | 185 kB 00:00 2023-10-09T05:46:16.628 INFO:teuthology.orchestra.run.smithi107.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 440 kB/s | 173 kB 00:00 2023-10-09T05:46:16.653 INFO:teuthology.orchestra.run.smithi107.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 133 kB/s | 11 kB 00:00 2023-10-09T05:46:16.687 INFO:teuthology.orchestra.run.smithi150.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 683 kB/s | 85 kB 00:00 2023-10-09T05:46:16.703 INFO:teuthology.orchestra.run.smithi107.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 247 kB/s | 18 kB 00:00 2023-10-09T05:46:16.712 INFO:teuthology.orchestra.run.smithi150.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.1 MB/s | 255 kB 00:00 2023-10-09T05:46:16.737 INFO:teuthology.orchestra.run.smithi107.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 582 kB/s | 49 kB 00:00 2023-10-09T05:46:16.746 INFO:teuthology.orchestra.run.smithi150.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.2 MB/s | 251 kB 00:00 2023-10-09T05:46:16.796 INFO:teuthology.orchestra.run.smithi107.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 637 kB/s | 59 kB 00:00 2023-10-09T05:46:16.796 INFO:teuthology.orchestra.run.smithi150.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 211 kB/s | 23 kB 00:00 2023-10-09T05:46:16.880 INFO:teuthology.orchestra.run.smithi150.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 652 kB/s | 109 kB 00:00 2023-10-09T05:46:16.896 INFO:teuthology.orchestra.run.smithi107.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 156 kB/s | 16 kB 00:00 2023-10-09T05:46:16.905 INFO:teuthology.orchestra.run.smithi150.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 3.4 MB/s | 559 kB 00:00 2023-10-09T05:46:16.921 INFO:teuthology.orchestra.run.smithi107.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.5 MB/s | 283 kB 00:00 2023-10-09T05:46:16.939 INFO:teuthology.orchestra.run.smithi150.stdout:(27/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.6 MB/s | 238 kB 00:00 2023-10-09T05:46:16.963 INFO:teuthology.orchestra.run.smithi107.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 279 kB/s | 19 kB 00:00 2023-10-09T05:46:16.988 INFO:teuthology.orchestra.run.smithi107.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 358 kB/s | 24 kB 00:00 2023-10-09T05:46:17.006 INFO:teuthology.orchestra.run.smithi150.stdout:(28/32): python3-pytz-2017.2-9.el8.noarch.rpm 432 kB/s | 54 kB 00:00 2023-10-09T05:46:17.022 INFO:teuthology.orchestra.run.smithi107.stdout:(18/32): python3-cherrypy-18.4.0-1.el8.noarch.r 805 kB/s | 384 kB 00:00 2023-10-09T05:46:17.031 INFO:teuthology.orchestra.run.smithi150.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 309 kB/s | 39 kB 00:00 2023-10-09T05:46:17.047 INFO:teuthology.orchestra.run.smithi107.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 349 kB/s | 29 kB 00:00 2023-10-09T05:46:17.056 INFO:teuthology.orchestra.run.smithi150.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.3 MB/s | 157 kB 00:00 2023-10-09T05:46:17.073 INFO:teuthology.orchestra.run.smithi107.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 328 kB/s | 27 kB 00:00 2023-10-09T05:46:17.098 INFO:teuthology.orchestra.run.smithi150.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.9 MB/s | 457 kB 00:00 2023-10-09T05:46:17.148 INFO:teuthology.orchestra.run.smithi107.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.0 MB/s | 255 kB 00:00 2023-10-09T05:46:17.156 INFO:teuthology.orchestra.run.smithi150.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 826 kB/s | 103 kB 00:00 2023-10-09T05:46:17.161 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:17.161 INFO:teuthology.orchestra.run.smithi150.stdout:Total 8.3 MB/s | 28 MB 00:03 2023-10-09T05:46:17.173 INFO:teuthology.orchestra.run.smithi107.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 852 kB/s | 85 kB 00:00 2023-10-09T05:46:17.224 INFO:teuthology.orchestra.run.smithi107.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.4 MB/s | 251 kB 00:00 2023-10-09T05:46:17.249 INFO:teuthology.orchestra.run.smithi107.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 229 kB/s | 23 kB 00:00 2023-10-09T05:46:17.253 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:46:17.290 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:46:17.290 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:46:17.316 INFO:teuthology.orchestra.run.smithi107.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 768 kB/s | 109 kB 00:00 2023-10-09T05:46:17.391 INFO:teuthology.orchestra.run.smithi107.stdout:(26/32): python3-pytz-2017.2-9.el8.noarch.rpm 723 kB/s | 54 kB 00:00 2023-10-09T05:46:17.425 INFO:teuthology.orchestra.run.smithi107.stdout:(27/32): python3-cryptography-3.2.1-5.el8.x86_6 3.1 MB/s | 559 kB 00:00 2023-10-09T05:46:17.450 INFO:teuthology.orchestra.run.smithi107.stdout:(28/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.2 MB/s | 238 kB 00:00 2023-10-09T05:46:17.493 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:46:17.497 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:46:17.534 INFO:teuthology.orchestra.run.smithi107.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 273 kB/s | 39 kB 00:00 2023-10-09T05:46:17.559 INFO:teuthology.orchestra.run.smithi107.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.1 MB/s | 157 kB 00:00 2023-10-09T05:46:17.643 INFO:teuthology.orchestra.run.smithi107.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.3 MB/s | 457 kB 00:00 2023-10-09T05:46:17.667 INFO:teuthology.orchestra.run.smithi107.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 775 kB/s | 103 kB 00:00 2023-10-09T05:46:17.672 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:17.672 INFO:teuthology.orchestra.run.smithi107.stdout:Total 11 MB/s | 28 MB 00:02 2023-10-09T05:46:17.760 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:46:17.795 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:46:17.795 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:46:17.998 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:46:18.002 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:46:18.596 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:46:19.190 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:46:19.372 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:46:19.925 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T05:46:20.028 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:46:20.254 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T05:46:20.259 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T05:46:20.486 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T05:46:20.511 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-09T05:46:20.729 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T05:46:20.763 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-09T05:46:20.925 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T05:46:21.009 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-09T05:46:21.176 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T05:46:21.221 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-09T05:46:21.478 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-09T05:46:21.531 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T05:46:21.798 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T05:46:21.849 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-09T05:46:21.979 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T05:46:22.159 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-09T05:46:22.196 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T05:46:22.338 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-09T05:46:22.396 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T05:46:22.531 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-09T05:46:22.631 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:46:22.773 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-09T05:46:22.811 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:46:22.984 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:46:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T05:46:23.165 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:46:23.230 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T05:46:23.340 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:46:23.452 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T05:46:23.485 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:46:23.600 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T05:46:23.725 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:46:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T05:46:23.953 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T05:46:23.957 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:46:24.191 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T05:46:24.218 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:46:24.459 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T05:46:24.482 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:46:24.645 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T05:46:24.726 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:46:24.955 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:46:25.250 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T05:46:25.299 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:46:25.544 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T05:46:25.614 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:46:25.790 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T05:46:25.978 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T05:46:26.103 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T05:46:26.197 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:46:26.453 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T05:46:26.823 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T05:46:27.026 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:46:27.674 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:46:27.872 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:46:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:46:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:46:28.503 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T05:46:28.638 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:46:28.742 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:46:28.759 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:46:28.778 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:46:28.910 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:46:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:46:29.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:46:29.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T05:46:29.258 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T05:46:29.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T05:46:29.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T05:46:29.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T05:46:29.288 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:46:29.504 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T05:46:29.696 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:46:29.713 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T05:46:29.903 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T05:46:30.255 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T05:46:30.256 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-09T05:46:30.257 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-09T05:46:30.258 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-09T05:46:30.258 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-09T05:46:30.880 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T05:46:30.881 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:40 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:31.835 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-09T05:46:31.836 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:42 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:37.428 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:37.616 INFO:teuthology.orchestra.run.smithi107.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:46:37.617 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:46:37.618 INFO:teuthology.orchestra.run.smithi107.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:46:37.619 INFO:teuthology.orchestra.run.smithi107.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:46:37.619 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:37.619 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:46:37.815 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-base 2023-10-09T05:46:37.930 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:46:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:38.119 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:46:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T05:46:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T05:46:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T05:46:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:38.123 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:46:38.164 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:38.287 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-base 2023-10-09T05:46:38.634 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:39.836 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:40.022 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:50 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:41.100 INFO:teuthology.orchestra.run.smithi107.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:46:41.210 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:46:41.211 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:46:41.211 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:46:41.334 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install cephadm 2023-10-09T05:46:41.353 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:46:41.461 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:46:41.462 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:46:41.462 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:46:41.577 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install cephadm 2023-10-09T05:46:41.677 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:41.920 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:43.203 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:54 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:43.441 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:53 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:44.565 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:46:44.565 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:44.565 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 80 k 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 364 k 2023-10-09T05:46:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:46:44.757 INFO:teuthology.orchestra.run.smithi107.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 422 kB/s | 80 kB 00:00 2023-10-09T05:46:44.757 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:44.758 INFO:teuthology.orchestra.run.smithi107.stdout:Total 419 kB/s | 80 kB 00:00 2023-10-09T05:46:44.758 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:46:44.766 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:46:44.766 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:46:44.768 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:46:44.771 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:46:44.806 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:46:44.807 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:44.807 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:46:44.807 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:44.807 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:46:44.807 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 80 k 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 364 k 2023-10-09T05:46:44.808 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:46:44.814 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:46:44.972 INFO:teuthology.orchestra.run.smithi150.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 491 kB/s | 80 kB 00:00 2023-10-09T05:46:44.973 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:44.973 INFO:teuthology.orchestra.run.smithi150.stdout:Total 487 kB/s | 80 kB 00:00 2023-10-09T05:46:44.973 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:46:44.981 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:46:44.981 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:46:44.983 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:46:44.986 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:46:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:46:46.562 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:46.748 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:46.752 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:46.912 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:46.930 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:47.095 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:47.980 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:47.980 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:48.122 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T05:46:48.122 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:58 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:53.175 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:46:53.352 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:53.352 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:46:53.352 INFO:teuthology.orchestra.run.smithi107.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:46:53.352 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:53.352 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:46:53.487 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:46:53.842 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:54.311 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:46:54.493 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:54.493 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:46:54.493 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:46:54.494 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:54.494 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:46:54.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T05:46:54.990 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:46:55.197 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:06 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:56.391 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:06 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:46:56.555 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:46:56.555 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:56.555 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repo Size 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:46:56.556 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:46:56.557 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 161 k 2023-10-09T05:46:56.557 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 490 k 2023-10-09T05:46:56.557 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:46:56.855 INFO:teuthology.orchestra.run.smithi107.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 542 kB/s | 161 kB 00:00 2023-10-09T05:46:56.855 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:56.855 INFO:teuthology.orchestra.run.smithi107.stdout:Total 539 kB/s | 161 kB 00:00 2023-10-09T05:46:56.856 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:46:56.866 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:46:56.866 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:46:56.917 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:46:56.921 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:46:57.029 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:46:57.207 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:46:57.561 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:46:57.752 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:46:57.752 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:57.752 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:46:57.753 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:46:57.754 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 161 k 2023-10-09T05:46:57.754 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 490 k 2023-10-09T05:46:57.754 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:46:58.154 INFO:teuthology.orchestra.run.smithi150.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 404 kB/s | 161 kB 00:00 2023-10-09T05:46:58.154 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:46:58.154 INFO:teuthology.orchestra.run.smithi150.stdout:Total 403 kB/s | 161 kB 00:00 2023-10-09T05:46:58.155 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:46:58.166 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:46:58.166 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:46:58.220 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:46:58.224 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:46:58.625 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:46:58.625 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:46:58.994 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:46:59.189 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:46:59.568 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:47:00.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T05:47:00.608 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:10 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:06.870 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:47:07.051 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:07.051 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:47:07.051 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:47:07.051 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:07.051 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:47:07.215 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-mgr 2023-10-09T05:47:07.561 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:08.634 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:47:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:08.817 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:47:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:47:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:08.818 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:47:08.950 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr 2023-10-09T05:47:08.973 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:19 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:09.300 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:10.228 INFO:teuthology.orchestra.run.smithi107.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:47:10.339 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:47:10.340 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:47:10.340 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:47:10.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:47:10.793 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:11.007 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:21 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:12.267 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:47:12.309 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:23 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:12.377 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:47:12.378 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:47:12.378 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:47:12.496 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T05:47:12.838 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:13.676 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:47:13.677 INFO:teuthology.orchestra.run.smithi107.stdout:=================================================================================================== 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout:=================================================================================================== 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:47:13.678 INFO:teuthology.orchestra.run.smithi107.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout:=================================================================================================== 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout:Install 6 Packages 2023-10-09T05:47:13.679 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:13.680 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 2.0 M 2023-10-09T05:47:13.680 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 8.0 M 2023-10-09T05:47:13.680 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:47:13.967 INFO:teuthology.orchestra.run.smithi107.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1303.gb290 101 kB/s | 14 kB 00:00 2023-10-09T05:47:14.090 INFO:teuthology.orchestra.run.smithi107.stdout:(2/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 272 kB/s | 34 kB 00:00 2023-10-09T05:47:14.118 INFO:teuthology.orchestra.run.smithi107.stdout:(3/6): ceph-grafana-dashboards-17.2.6-1303.gb29 78 kB/s | 23 kB 00:00 2023-10-09T05:47:14.176 INFO:teuthology.orchestra.run.smithi107.stdout:(4/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 4.8 MB/s | 1.7 MB 00:00 2023-10-09T05:47:14.226 INFO:teuthology.orchestra.run.smithi107.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.4 MB/s | 196 kB 00:00 2023-10-09T05:47:14.243 INFO:teuthology.orchestra.run.smithi107.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 340 kB/s | 43 kB 00:00 2023-10-09T05:47:14.245 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:14.245 INFO:teuthology.orchestra.run.smithi107.stdout:Total 3.5 MB/s | 2.0 MB 00:00 2023-10-09T05:47:14.256 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:47:14.265 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:47:14.265 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:47:14.321 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:47:14.325 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:47:14.387 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:24 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:14.619 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:47:14.778 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:47:14.984 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:47:15.163 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:47:15.276 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T05:47:15.638 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T05:47:15.770 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout:=================================================================================================== 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout:=================================================================================================== 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T05:47:15.771 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout:=================================================================================================== 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout:Install 6 Packages 2023-10-09T05:47:15.772 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:15.773 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 2.0 M 2023-10-09T05:47:15.773 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 8.0 M 2023-10-09T05:47:15.773 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:47:15.855 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:47:15.958 INFO:teuthology.orchestra.run.smithi150.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1303.gb290 95 kB/s | 14 kB 00:00 2023-10-09T05:47:16.067 INFO:teuthology.orchestra.run.smithi150.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1303.gb29 88 kB/s | 23 kB 00:00 2023-10-09T05:47:16.089 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:47:16.090 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T05:47:16.090 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T05:47:16.090 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T05:47:16.090 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:47:16.090 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:47:16.150 INFO:teuthology.orchestra.run.smithi150.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 4.9 MB/s | 1.7 MB 00:00 2023-10-09T05:47:16.342 INFO:teuthology.orchestra.run.smithi150.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 222 kB/s | 43 kB 00:00 2023-10-09T05:47:16.425 INFO:teuthology.orchestra.run.smithi150.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 72 kB/s | 34 kB 00:00 2023-10-09T05:47:16.767 INFO:teuthology.orchestra.run.smithi150.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 282 kB/s | 196 kB 00:00 2023-10-09T05:47:16.770 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:16.770 INFO:teuthology.orchestra.run.smithi150.stdout:Total 2.0 MB/s | 2.0 MB 00:00 2023-10-09T05:47:16.782 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:47:16.792 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:47:16.792 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:47:16.850 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:47:16.854 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:47:17.147 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:47:17.160 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:47:17.160 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:28 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:17.346 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T05:47:17.545 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T05:47:17.733 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T05:47:17.853 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T05:47:18.170 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T05:47:18.390 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T05:47:18.607 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T05:47:19.678 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T05:47:19.678 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:29 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:25.246 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:47:25.427 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:25.427 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:47:25.427 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:25.427 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:25.427 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:25.428 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:47:25.428 INFO:teuthology.orchestra.run.smithi107.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:47:25.428 INFO:teuthology.orchestra.run.smithi107.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:47:25.428 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:25.428 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:47:25.562 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:47:25.908 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:27.548 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:38 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:28.561 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:28.742 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T05:47:28.743 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T05:47:28.743 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T05:47:28.743 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:28.743 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:47:28.869 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T05:47:28.914 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================================= 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================================= 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:47:28.918 INFO:teuthology.orchestra.run.smithi107.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:47:28.919 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================================= 2023-10-09T05:47:28.920 INFO:teuthology.orchestra.run.smithi107.stdout:Install 8 Packages 2023-10-09T05:47:28.920 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:28.920 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 35 M 2023-10-09T05:47:28.920 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 210 M 2023-10-09T05:47:28.920 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:47:29.184 INFO:teuthology.orchestra.run.smithi107.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 643 kB/s | 171 kB 00:00 2023-10-09T05:47:29.220 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:29.224 INFO:teuthology.orchestra.run.smithi107.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.1 MB/s | 644 kB 00:00 2023-10-09T05:47:29.558 INFO:teuthology.orchestra.run.smithi107.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 37 MB/s | 14 MB 00:00 2023-10-09T05:47:29.633 INFO:teuthology.orchestra.run.smithi107.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 9.1 MB/s | 3.7 MB 00:00 2023-10-09T05:47:29.767 INFO:teuthology.orchestra.run.smithi107.stdout:(5/8): ceph-mgr-diskprediction-local-17.2.6-130 8.7 MB/s | 7.4 MB 00:00 2023-10-09T05:47:29.792 INFO:teuthology.orchestra.run.smithi107.stdout:(6/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 963 kB/s | 225 kB 00:00 2023-10-09T05:47:29.893 INFO:teuthology.orchestra.run.smithi107.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 19 MB/s | 4.9 MB 00:00 2023-10-09T05:47:30.026 INFO:teuthology.orchestra.run.smithi107.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-10-09T05:47:30.027 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:30.027 INFO:teuthology.orchestra.run.smithi107.stdout:Total 32 MB/s | 35 MB 00:01 2023-10-09T05:47:30.264 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:47:30.283 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:47:30.283 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:47:30.574 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:47:30.578 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:47:30.776 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:40 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:31.357 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:47:31.532 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:47:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:47:31.849 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:32.127 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================================= 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================================= 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:47:32.129 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:47:32.130 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================================= 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi150.stdout:Install 8 Packages 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 35 M 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 210 M 2023-10-09T05:47:32.131 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:47:32.289 INFO:teuthology.orchestra.run.smithi150.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.0 MB/s | 644 kB 00:00 2023-10-09T05:47:32.331 INFO:teuthology.orchestra.run.smithi150.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 859 kB/s | 171 kB 00:00 2023-10-09T05:47:32.468 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:32.698 INFO:teuthology.orchestra.run.smithi150.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 34 MB/s | 14 MB 00:00 2023-10-09T05:47:32.723 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:47:32.756 INFO:teuthology.orchestra.run.smithi150.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 8.8 MB/s | 3.7 MB 00:00 2023-10-09T05:47:32.824 INFO:teuthology.orchestra.run.smithi150.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.8 MB/s | 225 kB 00:00 2023-10-09T05:47:32.974 INFO:teuthology.orchestra.run.smithi150.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 22 MB/s | 4.9 MB 00:00 2023-10-09T05:47:33.074 INFO:teuthology.orchestra.run.smithi150.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 19 MB/s | 4.7 MB 00:00 2023-10-09T05:47:33.328 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:47:33.540 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:47:34.232 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:47:34.576 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:47:34.806 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:35.074 INFO:teuthology.orchestra.run.smithi150.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-130 2.5 MB/s | 7.4 MB 00:02 2023-10-09T05:47:35.075 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:35.076 INFO:teuthology.orchestra.run.smithi150.stdout:Total 12 MB/s | 35 MB 00:02 2023-10-09T05:47:35.320 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:47:35.339 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:47:35.339 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:47:35.633 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:47:35.637 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:47:36.222 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:47:36.350 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:47:36.485 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T05:47:36.557 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:36.619 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:37.258 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:47:37.819 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:47:38.007 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T05:47:38.022 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:47:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:47:38.273 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:38.274 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T05:47:38.888 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T05:47:39.172 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:47:39.309 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T05:47:39.309 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:02:50 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:39.377 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:41.139 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:42.399 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T05:47:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:47:42.860 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T05:47:42.860 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T05:47:42.860 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T05:47:42.860 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-09T05:47:42.860 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-09T05:47:42.861 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T05:47:42.861 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T05:47:42.861 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-09T05:47:43.905 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-09T05:47:43.905 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:53 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:48.080 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:47:48.262 INFO:teuthology.orchestra.run.smithi107.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:47:48.263 INFO:teuthology.orchestra.run.smithi107.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:47:48.263 INFO:teuthology.orchestra.run.smithi107.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:47:48.263 INFO:teuthology.orchestra.run.smithi107.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:47:48.263 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:48.263 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:47:48.409 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:47:48.757 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:50.155 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:01 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:51.596 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================== 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================== 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:47:51.598 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:47:51.599 INFO:teuthology.orchestra.run.smithi107.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout:======================================================================================================== 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout:Install 13 Packages 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:47:51.600 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 1.9 M 2023-10-09T05:47:51.601 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 25 M 2023-10-09T05:47:51.601 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:47:52.047 INFO:teuthology.orchestra.run.smithi107.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 170 kB/s | 16 kB 00:00 2023-10-09T05:47:52.073 INFO:teuthology.orchestra.run.smithi107.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 272 kB/s | 33 kB 00:00 2023-10-09T05:47:52.123 INFO:teuthology.orchestra.run.smithi107.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.1 MB/s | 81 kB 00:00 2023-10-09T05:47:52.148 INFO:teuthology.orchestra.run.smithi107.stdout:(4/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 240 kB/s | 47 kB 00:00 2023-10-09T05:47:52.166 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:47:52.173 INFO:teuthology.orchestra.run.smithi107.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 1.2 MB/s | 64 kB 00:00 2023-10-09T05:47:52.273 INFO:teuthology.orchestra.run.smithi107.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 5.8 MB/s | 1.2 MB 00:00 2023-10-09T05:47:52.299 INFO:teuthology.orchestra.run.smithi107.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 407 kB/s | 61 kB 00:00 2023-10-09T05:47:52.324 INFO:teuthology.orchestra.run.smithi107.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 283 kB/s | 43 kB 00:00 2023-10-09T05:47:52.349 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:52.349 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:47:52.349 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:52.350 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:47:52.498 INFO:teuthology.orchestra.run.smithi107.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 692 kB/s | 155 kB 00:00 2023-10-09T05:47:52.504 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-rook 2023-10-09T05:47:52.533 INFO:teuthology.orchestra.run.smithi107.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 134 kB/s | 27 kB 00:00 2023-10-09T05:47:52.558 INFO:teuthology.orchestra.run.smithi107.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 425 kB/s | 110 kB 00:00 2023-10-09T05:47:52.634 INFO:teuthology.orchestra.run.smithi107.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 934 kB/s | 126 kB 00:00 2023-10-09T05:47:52.650 INFO:teuthology.orchestra.run.smithi107.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 171 kB/s | 20 kB 00:00 2023-10-09T05:47:52.651 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:52.651 INFO:teuthology.orchestra.run.smithi107.stdout:Total 1.8 MB/s | 1.9 MB 00:01 2023-10-09T05:47:52.718 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:47:52.734 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:47:52.734 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:47:52.806 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:47:52.809 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:47:52.853 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:47:53.314 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:47:53.485 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:47:53.696 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:47:53.900 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:47:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T05:47:54.313 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T05:47:54.598 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T05:47:54.645 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:04 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:47:54.815 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T05:47:55.009 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:47:55.207 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:47:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:47:56.044 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================== 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================== 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T05:47:56.046 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:56.047 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================================== 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout:Install 13 Packages 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 1.9 M 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 25 M 2023-10-09T05:47:56.048 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:47:56.254 INFO:teuthology.orchestra.run.smithi150.stdout:(1/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 268 kB/s | 47 kB 00:00 2023-10-09T05:47:56.380 INFO:teuthology.orchestra.run.smithi150.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 54 kB/s | 16 kB 00:00 2023-10-09T05:47:56.405 INFO:teuthology.orchestra.run.smithi150.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 101 kB/s | 33 kB 00:00 2023-10-09T05:47:56.535 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:47:56.539 INFO:teuthology.orchestra.run.smithi150.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 481 kB/s | 64 kB 00:00 2023-10-09T05:47:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 264 kB/s | 81 kB 00:00 2023-10-09T05:47:56.631 INFO:teuthology.orchestra.run.smithi150.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 667 kB/s | 61 kB 00:00 2023-10-09T05:47:56.664 INFO:teuthology.orchestra.run.smithi150.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 426 kB/s | 43 kB 00:00 2023-10-09T05:47:56.731 INFO:teuthology.orchestra.run.smithi150.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.3 MB/s | 1.2 MB 00:00 2023-10-09T05:47:56.756 INFO:teuthology.orchestra.run.smithi150.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.2 MB/s | 155 kB 00:00 2023-10-09T05:47:56.782 INFO:teuthology.orchestra.run.smithi150.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 940 kB/s | 110 kB 00:00 2023-10-09T05:47:56.810 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:47:56.849 INFO:teuthology.orchestra.run.smithi150.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.4 MB/s | 126 kB 00:00 2023-10-09T05:47:56.874 INFO:teuthology.orchestra.run.smithi150.stdout:(12/13): python3-jsonpatch-1.21-2.el8.noarch.rp 186 kB/s | 27 kB 00:00 2023-10-09T05:47:56.899 INFO:teuthology.orchestra.run.smithi150.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 171 kB/s | 20 kB 00:00 2023-10-09T05:47:56.901 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:47:56.902 INFO:teuthology.orchestra.run.smithi150.stdout:Total 2.3 MB/s | 1.9 MB 00:00 2023-10-09T05:47:56.961 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:47:56.976 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:47:56.976 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:47:56.993 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:47:57.046 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:47:57.050 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:47:57.264 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:47:57.265 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T05:47:57.265 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T05:47:57.265 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T05:47:57.265 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T05:47:57.265 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T05:47:57.545 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:47:57.758 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T05:47:57.986 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T05:47:58.181 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T05:47:58.348 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-09T05:47:58.397 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T05:47:58.397 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:09 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:47:58.561 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-09T05:47:58.829 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-09T05:47:59.018 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-09T05:47:59.265 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T05:47:59.447 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T05:47:59.687 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T05:48:00.760 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T05:48:00.991 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T05:48:01.449 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:48:01.697 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-09T05:48:01.698 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-09T05:48:01.699 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-09T05:48:02.819 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-09T05:48:02.819 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:12 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:06.028 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:48:06.211 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:48:06.212 INFO:teuthology.orchestra.run.smithi107.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:48:06.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:48:06.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:48:06.213 INFO:teuthology.orchestra.run.smithi107.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:48:06.213 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:06.213 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:48:06.346 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:48:06.697 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:08.208 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:19 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:09.629 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout:=============================================================================================== 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout:=============================================================================================== 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:48:09.630 INFO:teuthology.orchestra.run.smithi107.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout:=============================================================================================== 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout:Install 5 Packages 2023-10-09T05:48:09.631 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:09.632 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 6.0 M 2023-10-09T05:48:09.632 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 26 M 2023-10-09T05:48:09.632 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:48:09.876 INFO:teuthology.orchestra.run.smithi107.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 560 kB/s | 121 kB 00:00 2023-10-09T05:48:10.010 INFO:teuthology.orchestra.run.smithi107.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.9 MB/s | 538 kB 00:00 2023-10-09T05:48:10.027 INFO:teuthology.orchestra.run.smithi107.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 164 kB/s | 60 kB 00:00 2023-10-09T05:48:10.260 INFO:teuthology.orchestra.run.smithi107.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 19 MB/s | 4.8 MB 00:00 2023-10-09T05:48:10.286 INFO:teuthology.orchestra.run.smithi107.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 760 kB/s | 476 kB 00:00 2023-10-09T05:48:10.287 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:10.287 INFO:teuthology.orchestra.run.smithi107.stdout:Total 9.1 MB/s | 6.0 MB 00:00 2023-10-09T05:48:10.310 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:48:10.340 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:48:10.350 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:48:10.351 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:48:10.429 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:48:10.433 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:48:10.492 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:10.492 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T05:48:10.493 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T05:48:10.494 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T05:48:10.494 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T05:48:10.494 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:10.494 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:48:10.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T05:48:10.981 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:11.374 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:48:11.760 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T05:48:12.024 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T05:48:12.368 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:48:12.451 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:22 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:12.666 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T05:48:12.889 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:48:13.127 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:48:13.127 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T05:48:13.127 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T05:48:13.127 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:48:13.128 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T05:48:13.821 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:48:13.822 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================================== 2023-10-09T05:48:13.822 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:48:13.822 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================================== 2023-10-09T05:48:13.822 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:48:13.822 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================================== 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout:Install 5 Packages 2023-10-09T05:48:13.823 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:13.824 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 6.0 M 2023-10-09T05:48:13.824 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 26 M 2023-10-09T05:48:13.824 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:48:14.127 INFO:teuthology.orchestra.run.smithi150.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 388 kB/s | 60 kB 00:00 2023-10-09T05:48:14.152 INFO:teuthology.orchestra.run.smithi150.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 670 kB/s | 121 kB 00:00 2023-10-09T05:48:14.188 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T05:48:14.188 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:25 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:14.227 INFO:teuthology.orchestra.run.smithi150.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.8 MB/s | 476 kB 00:00 2023-10-09T05:48:14.294 INFO:teuthology.orchestra.run.smithi150.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.1 MB/s | 538 kB 00:00 2023-10-09T05:48:14.419 INFO:teuthology.orchestra.run.smithi150.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 18 MB/s | 4.8 MB 00:00 2023-10-09T05:48:14.420 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:14.421 INFO:teuthology.orchestra.run.smithi150.stdout:Total 10 MB/s | 6.0 MB 00:00 2023-10-09T05:48:14.474 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:48:14.485 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:48:14.486 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:48:14.565 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:48:14.569 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:48:15.514 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:48:15.868 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T05:48:16.107 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T05:48:16.470 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:48:16.732 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T05:48:16.914 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:48:17.149 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T05:48:17.149 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T05:48:17.149 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T05:48:17.149 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T05:48:17.149 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T05:48:18.185 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T05:48:18.185 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:23.445 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T05:48:23.628 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:23.629 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:48:23.817 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install ceph-fuse 2023-10-09T05:48:24.168 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:25.157 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:48:25.343 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:25.343 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:48:25.343 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T05:48:25.343 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T05:48:25.343 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T05:48:25.344 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T05:48:25.344 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T05:48:25.344 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:25.344 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:48:25.496 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-fuse 2023-10-09T05:48:25.832 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:36 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:25.849 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:27.255 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repository Size 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-fuse 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T05:48:27.256 INFO:teuthology.orchestra.run.smithi107.stdout:Installing dependencies: 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout:Install 2 Packages 2023-10-09T05:48:27.257 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:27.258 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 959 k 2023-10-09T05:48:27.258 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 3.0 M 2023-10-09T05:48:27.258 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:48:27.476 INFO:teuthology.orchestra.run.smithi107.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 378 kB/s | 83 kB 00:00 2023-10-09T05:48:27.566 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:37 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:27.659 INFO:teuthology.orchestra.run.smithi107.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 2.1 MB/s | 876 kB 00:00 2023-10-09T05:48:27.660 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:27.660 INFO:teuthology.orchestra.run.smithi107.stdout:Total 2.3 MB/s | 959 kB 00:00 2023-10-09T05:48:27.671 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:48:27.692 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:48:27.692 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:48:27.765 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:48:27.769 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:48:27.964 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:48:28.239 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:48:28.394 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:48:28.754 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:48:28.754 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T05:48:28.943 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:48:28.943 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:28.943 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:28.944 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:48:28.945 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:28.945 INFO:teuthology.orchestra.run.smithi150.stdout:Install 2 Packages 2023-10-09T05:48:28.945 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:28.945 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 959 k 2023-10-09T05:48:28.945 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 3.0 M 2023-10-09T05:48:28.946 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:48:29.060 INFO:teuthology.orchestra.run.smithi150.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 719 kB/s | 83 kB 00:00 2023-10-09T05:48:29.194 INFO:teuthology.orchestra.run.smithi150.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 3.4 MB/s | 876 kB 00:00 2023-10-09T05:48:29.195 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:29.195 INFO:teuthology.orchestra.run.smithi150.stdout:Total 3.7 MB/s | 959 kB 00:00 2023-10-09T05:48:29.206 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:48:29.227 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:48:29.227 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:48:29.302 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:48:29.305 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:48:29.472 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:48:29.739 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T05:48:29.831 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:48:29.831 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:40 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:29.919 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:48:30.273 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T05:48:30.274 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T05:48:31.344 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T05:48:31.344 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:38.262 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:48:38.441 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:38.441 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:48:38.441 INFO:teuthology.orchestra.run.smithi107.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:48:38.442 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:38.442 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:48:38.611 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install librados-devel 2023-10-09T05:48:38.825 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:48:38.970 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:39.006 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:39.006 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:48:39.006 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T05:48:39.007 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:39.007 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:48:39.140 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados-devel 2023-10-09T05:48:39.497 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:40.583 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:51 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:41.174 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:51 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repo Size 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:48:41.942 INFO:teuthology.orchestra.run.smithi107.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T05:48:41.943 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:41.943 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:48:41.943 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:41.943 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:48:41.943 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:41.944 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 130 k 2023-10-09T05:48:41.944 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 464 k 2023-10-09T05:48:41.944 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:48:42.274 INFO:teuthology.orchestra.run.smithi107.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 392 kB/s | 130 kB 00:00 2023-10-09T05:48:42.275 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:42.275 INFO:teuthology.orchestra.run.smithi107.stdout:Total 391 kB/s | 130 kB 00:00 2023-10-09T05:48:42.275 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:48:42.281 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:48:42.281 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:48:42.336 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:48:42.340 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:48:42.454 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:48:42.552 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T05:48:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 130 k 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 464 k 2023-10-09T05:48:42.554 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:48:42.647 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:42.750 INFO:teuthology.orchestra.run.smithi150.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 661 kB/s | 130 kB 00:00 2023-10-09T05:48:42.751 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:42.751 INFO:teuthology.orchestra.run.smithi150.stdout:Total 657 kB/s | 130 kB 00:00 2023-10-09T05:48:42.751 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:48:42.757 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:48:42.757 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:48:42.812 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:48:42.815 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:48:42.877 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:42.931 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:48:43.083 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:43.329 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:43.935 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:43.935 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:03:54 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:44.374 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:44.375 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:03:54 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:49.201 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:48:49.382 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:49.383 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:48:49.383 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:48:49.383 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:49.383 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:48:49.514 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs2 2023-10-09T05:48:49.810 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:48:49.871 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:49.992 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:49.993 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:48:49.993 INFO:teuthology.orchestra.run.smithi107.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:48:49.993 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:49.993 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:48:50.135 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install libcephfs2 2023-10-09T05:48:50.491 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:51.583 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:01 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:52.189 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:03 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:52.834 INFO:teuthology.orchestra.run.smithi150.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:48:52.941 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:48:52.942 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:48:52.942 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:48:53.058 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs-devel 2023-10-09T05:48:53.415 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:53.437 INFO:teuthology.orchestra.run.smithi107.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:48:53.548 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:48:53.550 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:48:53.550 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:48:53.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install libcephfs-devel 2023-10-09T05:48:54.016 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:48:55.163 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:05 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:55.594 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:06 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:48:56.601 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:48:56.601 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:56.601 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2023-10-09T05:48:56.601 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:48:56.602 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:48:56.603 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 25 k 2023-10-09T05:48:56.603 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 109 k 2023-10-09T05:48:56.603 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:48:56.815 INFO:teuthology.orchestra.run.smithi150.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 117 kB/s | 25 kB 00:00 2023-10-09T05:48:56.816 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:56.816 INFO:teuthology.orchestra.run.smithi150.stdout:Total 116 kB/s | 25 kB 00:00 2023-10-09T05:48:56.816 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:48:56.820 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:48:56.820 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:48:56.822 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:48:56.826 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:48:56.875 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:48:57.031 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repo Size 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:48:57.032 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:48:57.033 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:48:57.033 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 25 k 2023-10-09T05:48:57.033 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 109 k 2023-10-09T05:48:57.033 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:48:57.062 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:57.271 INFO:teuthology.orchestra.run.smithi107.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 105 kB/s | 25 kB 00:00 2023-10-09T05:48:57.271 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:48:57.272 INFO:teuthology.orchestra.run.smithi107.stdout:Total 104 kB/s | 25 kB 00:00 2023-10-09T05:48:57.272 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:48:57.276 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:48:57.276 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:48:57.278 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:48:57.282 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:48:57.282 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:57.331 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:48:57.544 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:57.757 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:58.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:58.257 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:08 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:48:58.765 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:48:58.765 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:09 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:08.595 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:49:08.728 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:49:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:08.777 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:49:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:08.777 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:08.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados2 2023-10-09T05:49:08.908 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:08.908 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:49:08.908 INFO:teuthology.orchestra.run.smithi107.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:08.908 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:08.908 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:09.037 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install librados2 2023-10-09T05:49:09.259 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:09.389 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:10.968 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:20 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:11.010 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:22 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:12.217 INFO:teuthology.orchestra.run.smithi150.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:12.292 INFO:teuthology.orchestra.run.smithi107.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:12.326 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:12.328 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:12.328 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:12.402 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:12.403 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:12.403 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:12.448 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librbd1 2023-10-09T05:49:12.526 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install librbd1 2023-10-09T05:49:12.801 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:12.878 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:14.485 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:24 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:14.551 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:25 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:15.751 INFO:teuthology.orchestra.run.smithi150.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:15.801 INFO:teuthology.orchestra.run.smithi107.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:15.859 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:15.861 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:15.861 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:15.910 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:15.912 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:15.912 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:15.970 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rados 2023-10-09T05:49:16.016 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install python3-rados 2023-10-09T05:49:16.324 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:16.368 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:17.972 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:28 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:18.010 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:28 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:19.245 INFO:teuthology.orchestra.run.smithi107.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:19.331 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:19.352 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:19.354 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:19.354 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:19.438 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:19.439 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:19.439 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:19.465 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install python3-rgw 2023-10-09T05:49:19.553 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rgw 2023-10-09T05:49:19.817 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:19.906 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:21.288 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:32 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:21.442 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:31 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:22.598 INFO:teuthology.orchestra.run.smithi107.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:22.710 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:22.711 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:22.711 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:22.767 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:22.841 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install python3-cephfs 2023-10-09T05:49:22.876 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:22.877 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:22.877 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:22.996 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-cephfs 2023-10-09T05:49:23.193 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:23.349 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:24.707 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:35 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:24.835 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:34 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:25.961 INFO:teuthology.orchestra.run.smithi107.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:26.068 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:26.069 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:26.069 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:26.092 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:26.182 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install python3-rbd 2023-10-09T05:49:26.202 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:26.203 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:26.203 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:26.310 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rbd 2023-10-09T05:49:26.533 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:26.662 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:28.132 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:39 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:28.178 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:38 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:29.391 INFO:teuthology.orchestra.run.smithi107.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:29.430 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T05:49:29.499 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:29.500 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2023-10-09T05:49:29.501 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:29.536 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:29.538 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2023-10-09T05:49:29.538 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:29.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install rbd-fuse 2023-10-09T05:49:29.662 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-fuse 2023-10-09T05:49:29.993 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:30.015 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:31.694 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:41 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:31.890 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:42 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:33.114 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:33.114 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2023-10-09T05:49:33.114 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:49:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:33.116 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 90 k 2023-10-09T05:49:33.116 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 264 k 2023-10-09T05:49:33.116 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:49:33.247 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout: Package Architecture Version Repository Size 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:33.248 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 90 k 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 264 k 2023-10-09T05:49:33.249 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:49:33.275 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 566 kB/s | 90 kB 00:00 2023-10-09T05:49:33.275 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:49:33.276 INFO:teuthology.orchestra.run.smithi150.stdout:Total 562 kB/s | 90 kB 00:00 2023-10-09T05:49:33.276 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:49:33.286 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:49:33.286 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:49:33.340 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:49:33.343 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:49:33.411 INFO:teuthology.orchestra.run.smithi107.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 556 kB/s | 90 kB 00:00 2023-10-09T05:49:33.411 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:49:33.412 INFO:teuthology.orchestra.run.smithi107.stdout:Total 552 kB/s | 90 kB 00:00 2023-10-09T05:49:33.412 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:49:33.422 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:49:33.422 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:49:33.452 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:49:33.476 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:49:33.480 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:49:33.593 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:33.622 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:49:33.811 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:33.813 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:34.039 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:34.885 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:34.885 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:44 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:35.147 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:35.147 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:46 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:49:43.415 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:43.415 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:49:43.415 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:43.416 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:43.416 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:43.547 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install rbd-mirror 2023-10-09T05:49:43.902 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:44.302 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:49:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:44.483 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:49:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:44.483 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:44.608 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-mirror 2023-10-09T05:49:44.962 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:45.326 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:04:56 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:46.431 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:04:56 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:46.723 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:46.723 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:46.723 INFO:teuthology.orchestra.run.smithi107.stdout: Package Arch Version Repo Size 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:49:46.724 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:46.725 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 3.5 M 2023-10-09T05:49:46.725 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 16 M 2023-10-09T05:49:46.725 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:49:47.104 INFO:teuthology.orchestra.run.smithi107.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 9.1 MB/s | 3.5 MB 00:00 2023-10-09T05:49:47.105 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:49:47.105 INFO:teuthology.orchestra.run.smithi107.stdout:Total 9.1 MB/s | 3.5 MB 00:00 2023-10-09T05:49:47.105 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:49:47.117 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:49:47.117 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:49:47.184 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:49:47.188 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:49:47.634 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:49:47.798 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:49:47.798 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:47.798 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:49:47.799 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:47.800 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 3.5 M 2023-10-09T05:49:47.800 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 16 M 2023-10-09T05:49:47.800 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:49:47.844 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:48.200 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:48.444 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 5.4 MB/s | 3.5 MB 00:00 2023-10-09T05:49:48.445 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:49:48.445 INFO:teuthology.orchestra.run.smithi150.stdout:Total 5.3 MB/s | 3.5 MB 00:00 2023-10-09T05:49:48.446 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:49:48.457 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:49:48.457 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:49:48.525 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:49:48.528 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:49:48.975 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:49:49.152 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:49.244 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:49.244 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:05:00 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:49.510 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:50.543 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:49:50.544 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:05:00 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:49:55.888 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:49:56.070 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:56.070 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:49:56.070 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:56.070 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:56.070 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:49:56.213 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum -y install rbd-nbd 2023-10-09T05:49:56.568 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:58.047 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:49:58.197 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:05:09 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:49:58.231 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:58.231 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:49:58.231 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:49:58.231 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:49:58.231 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:49:58.364 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-nbd 2023-10-09T05:49:58.725 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2023-10-09T05:49:59.571 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout: Package Architecture Version Repository Size 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:Installing: 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction Summary 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:================================================================================ 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout:Install 1 Package 2023-10-09T05:49:59.572 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:49:59.573 INFO:teuthology.orchestra.run.smithi107.stdout:Total download size: 173 k 2023-10-09T05:49:59.573 INFO:teuthology.orchestra.run.smithi107.stdout:Installed size: 552 k 2023-10-09T05:49:59.573 INFO:teuthology.orchestra.run.smithi107.stdout:Downloading Packages: 2023-10-09T05:49:59.874 INFO:teuthology.orchestra.run.smithi107.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 575 kB/s | 173 kB 00:00 2023-10-09T05:49:59.874 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:49:59.875 INFO:teuthology.orchestra.run.smithi107.stdout:Total 573 kB/s | 173 kB 00:00 2023-10-09T05:49:59.875 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction check 2023-10-09T05:49:59.885 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction check succeeded. 2023-10-09T05:49:59.885 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction test 2023-10-09T05:49:59.939 INFO:teuthology.orchestra.run.smithi107.stdout:Transaction test succeeded. 2023-10-09T05:49:59.943 INFO:teuthology.orchestra.run.smithi107.stdout:Running transaction 2023-10-09T05:50:00.058 INFO:teuthology.orchestra.run.smithi107.stdout: Preparing : 1/1 2023-10-09T05:50:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:00.261 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:05:10 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:50:00.463 INFO:teuthology.orchestra.run.smithi107.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:01.504 INFO:teuthology.orchestra.run.smithi107.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:01.504 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:05:12 ago on Mon 09 Oct 2023 05:44:49 AM UTC. 2023-10-09T05:50:01.680 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2023-10-09T05:50:01.681 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2023-10-09T05:50:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:50:01.682 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 173 k 2023-10-09T05:50:01.682 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 552 k 2023-10-09T05:50:01.682 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2023-10-09T05:50:02.039 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 484 kB/s | 173 kB 00:00 2023-10-09T05:50:02.040 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:50:02.040 INFO:teuthology.orchestra.run.smithi150.stdout:Total 483 kB/s | 173 kB 00:00 2023-10-09T05:50:02.040 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2023-10-09T05:50:02.051 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2023-10-09T05:50:02.051 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2023-10-09T05:50:02.106 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2023-10-09T05:50:02.110 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2023-10-09T05:50:02.234 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2023-10-09T05:50:02.392 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:02.630 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:03.650 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T05:50:03.651 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:05:13 ago on Mon 09 Oct 2023 05:44:50 AM UTC. 2023-10-09T05:50:10.173 INFO:teuthology.orchestra.run.smithi107.stdout:Installed products updated. 2023-10-09T05:50:10.351 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:50:10.351 INFO:teuthology.orchestra.run.smithi107.stdout:Installed: 2023-10-09T05:50:10.352 INFO:teuthology.orchestra.run.smithi107.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:50:10.352 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:50:10.352 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2023-10-09T05:50:10.501 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:12.568 INFO:teuthology.orchestra.run.smithi150.stdout:Installed products updated. 2023-10-09T05:50:12.749 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:50:12.749 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2023-10-09T05:50:12.749 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T05:50:12.749 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:50:12.749 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2023-10-09T05:50:12.880 DEBUG:teuthology.parallel:result is None 2023-10-09T05:50:12.881 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:50:12.881 INFO:teuthology.packaging:ref: None 2023-10-09T05:50:12.881 INFO:teuthology.packaging:tag: None 2023-10-09T05:50:12.881 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:50:12.881 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:50:12.882 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:50:13.359 DEBUG:teuthology.orchestra.run.smithi107:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:50:13.385 INFO:teuthology.orchestra.run.smithi107.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T05:50:13.386 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T05:50:13.386 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T05:50:13.386 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T05:50:13.386 INFO:teuthology.packaging:ref: None 2023-10-09T05:50:13.386 INFO:teuthology.packaging:tag: None 2023-10-09T05:50:13.387 INFO:teuthology.packaging:branch: quincy 2023-10-09T05:50:13.387 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:50:13.387 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T05:50:13.659 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T05:50:13.684 INFO:teuthology.orchestra.run.smithi150.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T05:50:13.684 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T05:50:13.684 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T05:50:13.685 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T05:50:13.685 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:50:13.685 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:50:13.718 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:50:13.718 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T05:50:13.754 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T05:50:13.754 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:50:13.754 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:50:13.788 DEBUG:teuthology.orchestra.run.smithi107:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:50:13.857 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:50:13.857 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T05:50:13.886 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T05:50:13.955 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T05:50:13.956 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:50:13.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:50:13.984 DEBUG:teuthology.orchestra.run.smithi107:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:50:14.050 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:50:14.050 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T05:50:14.078 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T05:50:14.147 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T05:50:14.148 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:50:14.148 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:50:14.174 DEBUG:teuthology.orchestra.run.smithi107:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:50:14.241 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:50:14.241 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T05:50:14.268 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T05:50:14.337 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T05:50:14.406 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T05:50:14.406 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T05:50:14.406 INFO:tasks.cephadm:Cluster fsid is b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:50:14.407 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T05:50:14.407 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.107', 'mon.c': '[v2:172.21.15.107:3301,v1:172.21.15.107:6790]', 'mon.b': '172.21.15.150'} 2023-10-09T05:50:14.407 INFO:tasks.cephadm:First mon is mon.a on smithi107 2023-10-09T05:50:14.407 INFO:tasks.cephadm:First mgr is y 2023-10-09T05:50:14.407 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T05:50:14.407 DEBUG:teuthology.orchestra.run.smithi107:> sudo hostname $(hostname -s) 2023-10-09T05:50:14.435 DEBUG:teuthology.orchestra.run.smithi150:> sudo hostname $(hostname -s) 2023-10-09T05:50:14.465 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T05:50:14.465 DEBUG:teuthology.orchestra.run.smithi107:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.624 INFO:teuthology.orchestra.run.smithi107.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 05:50 /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.625 DEBUG:teuthology.orchestra.run.smithi150:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.761 INFO:teuthology.orchestra.run.smithi150.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 05:50 /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.761 DEBUG:teuthology.orchestra.run.smithi107:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.779 DEBUG:teuthology.orchestra.run.smithi150:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T05:50:14.808 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T05:50:14.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T05:50:14.822 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T05:50:15.982 INFO:teuthology.orchestra.run.smithi150.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:50:16.006 INFO:teuthology.orchestra.run.smithi107.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:50:57.942 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout: "repo_digests": [ 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T05:50:57.943 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T05:51:07.921 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T05:51:07.921 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T05:51:07.921 INFO:teuthology.orchestra.run.smithi150.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T05:51:07.921 INFO:teuthology.orchestra.run.smithi150.stdout: "repo_digests": [ 2023-10-09T05:51:07.921 INFO:teuthology.orchestra.run.smithi150.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T05:51:07.922 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T05:51:07.922 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T05:51:07.945 DEBUG:teuthology.orchestra.run.smithi107:> sudo mkdir -p /etc/ceph 2023-10-09T05:51:07.983 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /etc/ceph 2023-10-09T05:51:08.016 DEBUG:teuthology.orchestra.run.smithi107:> sudo chmod 777 /etc/ceph 2023-10-09T05:51:08.053 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 777 /etc/ceph 2023-10-09T05:51:08.084 INFO:tasks.cephadm:Writing seed config... 2023-10-09T05:51:08.085 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T05:51:08.085 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T05:51:08.085 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T05:51:08.085 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T05:51:08.085 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T05:51:08.086 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T05:51:08.087 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:51:08.087 DEBUG:teuthology.orchestra.run.smithi107:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T05:51:08.111 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b7a98308-6667-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T05:51:08.112 DEBUG:teuthology.orchestra.run.smithi107:mon.a> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T05:51:08.154 DEBUG:teuthology.orchestra.run.smithi107:mgr.y> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T05:51:08.182 INFO:journalctl@ceph.mon.a.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:51:08.183 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T05:51:08.183 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.107 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:51:08.209 INFO:journalctl@ceph.mgr.y.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:51:08.383 INFO:teuthology.orchestra.run.smithi107.stdout:-------------------------------------------------------------------------------- 2023-10-09T05:51:08.384 INFO:teuthology.orchestra.run.smithi107.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'b7a98308-6667-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.107', '--skip-admin-label'] 2023-10-09T05:51:08.448 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:51:08.448 INFO:teuthology.orchestra.run.smithi107.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T05:51:08.449 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying podman|docker is present... 2023-10-09T05:51:08.506 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:51:08.506 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying lvm2 is present... 2023-10-09T05:51:08.507 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying time synchronization is in place... 2023-10-09T05:51:08.513 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T05:51:08.513 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:51:08.519 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T05:51:08.519 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:51:08.525 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout enabled 2023-10-09T05:51:08.531 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout active 2023-10-09T05:51:08.531 INFO:teuthology.orchestra.run.smithi107.stdout:Unit chronyd.service is enabled and running 2023-10-09T05:51:08.531 INFO:teuthology.orchestra.run.smithi107.stdout:Repeating the final host check... 2023-10-09T05:51:08.586 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stdout 3.0.1 2023-10-09T05:51:08.586 INFO:teuthology.orchestra.run.smithi107.stdout:podman (/bin/podman) version 3.0.1 is present 2023-10-09T05:51:08.586 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl is present 2023-10-09T05:51:08.587 INFO:teuthology.orchestra.run.smithi107.stdout:lvcreate is present 2023-10-09T05:51:08.592 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T05:51:08.592 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T05:51:08.598 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T05:51:08.598 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:51:08.603 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout enabled 2023-10-09T05:51:08.609 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout active 2023-10-09T05:51:08.609 INFO:teuthology.orchestra.run.smithi107.stdout:Unit chronyd.service is enabled and running 2023-10-09T05:51:08.609 INFO:teuthology.orchestra.run.smithi107.stdout:Host looks OK 2023-10-09T05:51:08.609 INFO:teuthology.orchestra.run.smithi107.stdout:Cluster fsid: b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:08.610 INFO:teuthology.orchestra.run.smithi107.stdout:Acquiring lock 140694255420584 on /run/cephadm/b7a98308-6667-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:51:08.610 INFO:teuthology.orchestra.run.smithi107.stdout:Lock 140694255420584 acquired on /run/cephadm/b7a98308-6667-11ee-8db5-212e2dc638e7.lock 2023-10-09T05:51:08.610 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying IP 172.21.15.107 port 3300 ... 2023-10-09T05:51:08.610 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying IP 172.21.15.107 port 6789 ... 2023-10-09T05:51:08.610 INFO:teuthology.orchestra.run.smithi107.stdout:Base mon IP(s) is [172.21.15.107:3300, 172.21.15.107:6789], mon addrv is [v2:172.21.15.107:3300,v1:172.21.15.107:6789] 2023-10-09T05:51:08.614 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-10-09T05:51:08.614 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.107 metric 100 2023-10-09T05:51:08.616 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T05:51:08.616 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-10-09T05:51:08.616 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout inet6 fe80::ef4:540c:b3ee:d7d8/64 scope link noprefixroute 2023-10-09T05:51:08.619 INFO:teuthology.orchestra.run.smithi107.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T05:51:08.620 INFO:teuthology.orchestra.run.smithi107.stdout:Mon IP `172.21.15.107` is in CIDR network `172.21.0.0/20` 2023-10-09T05:51:08.620 INFO:teuthology.orchestra.run.smithi107.stdout:Mon IP `172.21.15.107` is in CIDR network `172.21.0.0/20` 2023-10-09T05:51:08.620 INFO:teuthology.orchestra.run.smithi107.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T05:51:08.620 INFO:teuthology.orchestra.run.smithi107.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T05:51:08.621 INFO:teuthology.orchestra.run.smithi107.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:51:09.063 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stdout 5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Copying blob sha256:1b55aded3a89dccf5f05a82d583393d22b3476851fe870b413c77ca909f5f760 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Copying config sha256:5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T05:51:09.064 INFO:teuthology.orchestra.run.smithi107.stdout:/bin/podman: stderr Storing signatures 2023-10-09T05:51:10.204 INFO:teuthology.orchestra.run.smithi107.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T05:51:10.205 INFO:teuthology.orchestra.run.smithi107.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T05:51:10.205 INFO:teuthology.orchestra.run.smithi107.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T05:51:11.222 INFO:teuthology.orchestra.run.smithi107.stdout:stat: stdout 167 167 2023-10-09T05:51:11.223 INFO:teuthology.orchestra.run.smithi107.stdout:Creating initial keys... 2023-10-09T05:51:12.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph-authtool: stdout AQDPlCNlimckIxAASOIst/qs30j74NukO1D1uw== 2023-10-09T05:51:13.249 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph-authtool: stdout AQDQlCNlkwfkIhAA0zSQZCzOpuWL0mIHoylF3Q== 2023-10-09T05:51:14.182 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph-authtool: stdout AQDRlCNlVfbOIxAAuy+cwzYqhMXQccxsc/WboA== 2023-10-09T05:51:14.183 INFO:teuthology.orchestra.run.smithi107.stdout:Creating initial monmap... 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T05:51:15.242 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:monmaptool for a [v2:172.21.15.107:3300,v1:172.21.15.107:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:setting min_mon_release = octopus 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: set fsid to b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:51:15.245 INFO:teuthology.orchestra.run.smithi107.stdout:Creating mon... 2023-10-09T05:51:16.345 INFO:teuthology.orchestra.run.smithi107.stdout:create mon.a on 2023-10-09T05:51:16.447 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T05:51:16.535 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T05:51:16.629 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:51:16.629 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target. 2023-10-09T05:51:16.745 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T05:51:16.745 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T05:51:16.829 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target.wants/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:51:16.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:16 smithi107 systemd[1]: Starting Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:17.403 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:51:17.403 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout disabled 2023-10-09T05:51:17.410 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:51:17.410 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:51:17.410 INFO:teuthology.orchestra.run.smithi107.stdout:firewalld.service is not enabled 2023-10-09T05:51:17.410 INFO:teuthology.orchestra.run.smithi107.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T05:51:17.410 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mon to start... 2023-10-09T05:51:17.411 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mon... 2023-10-09T05:51:17.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:17 smithi107 bash[100081]: 6acfd33672afc3efd37258057b6852478f90f708de72842d03e399f810845b6a 2023-10-09T05:51:17.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:17 smithi107 systemd[1]: Started Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:17.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:17 smithi107 ceph-mon[100146]: mkfs b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:17.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:17 smithi107 ceph-mon[100146]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:51:18.674 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T05:51:18.674 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout id: b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout services: 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.519421s) 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout data: 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T05:51:18.675 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T05:51:18.676 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:18.676 INFO:teuthology.orchestra.run.smithi107.stdout:mon is available 2023-10-09T05:51:18.676 INFO:teuthology.orchestra.run.smithi107.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: monmap e1: 1 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0]} removed_ranks: {} 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: fsmap 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: mgrmap e1: no daemons active 2023-10-09T05:51:18.681 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:18 smithi107 ceph-mon[100146]: from='client.? 172.21.15.107:0/610955646' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T05:51:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:19 smithi107 ceph-mon[100146]: from='client.? 172.21.15.107:0/1386770854' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:51:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:19 smithi107 ceph-mon[100146]: from='client.? 172.21.15.107:0/1386770854' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T05:51:19.856 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:19.856 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout fsid = b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.107:3300,v1:172.21.15.107:6789] 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T05:51:19.857 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T05:51:19.858 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T05:51:19.858 INFO:teuthology.orchestra.run.smithi107.stdout:Generating new minimal ceph.conf... 2023-10-09T05:51:20.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:20 smithi107 ceph-mon[100146]: from='client.? 172.21.15.107:0/1301736375' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:51:21.047 INFO:teuthology.orchestra.run.smithi107.stdout:Restarting the monitor... 2023-10-09T05:51:21.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:21 smithi107 systemd[1]: Stopping Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:21.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:21 smithi107 conmon[100122]: 2023-10-09T05:51:21.311+0000 7fdc2c6f1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:51:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:21 smithi107 conmon[100122]: 2023-10-09T05:51:21.311+0000 7fdc2c6f1700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T05:51:22.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:21 smithi107 bash[100709]: 6acfd33672afc3efd37258057b6852478f90f708de72842d03e399f810845b6a 2023-10-09T05:51:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T05:51:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 systemd[1]: Stopped Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 systemd[1]: Starting Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:22.784 INFO:teuthology.orchestra.run.smithi107.stdout:Setting mon public_network to 172.21.0.0/20 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 8 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: pidfile_write: ignore empty --pid-file 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: load: jerasure load: lrc 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:51:23.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: DB SUMMARY 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: DB Session ID: 51GLA70NPGUCB9YGSYX0 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83613 ; 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:51:23.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.env: 0x5598cde7ca80 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.fs: Posix File System 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.info_log: 0x5598ce8085a0 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.statistics: (nil) 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.use_fsync: 0 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:51:23.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.db_log_dir: 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:51:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.write_buffer_manager: 0x5598cf57a900 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:51:23.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.unordered_write: 0 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.row_cache: None 2023-10-09T05:51:23.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.wal_filter: None 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:51:23.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:51:23.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_open_files: -1 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:51:23.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Compression algorithms supported: 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kZSTD supported: 0 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T05:51:23.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.merge_operator: 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_filter: None 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5598ce85afb8) 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:51:23.036 INFO:journalctl@ceph.mon.a.smithi107.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_type: 0 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: data_block_index_type: 0 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_shortening: 1 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: hash_index_allow_collision: 1 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: checksum: 1 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: no_block_cache: 0 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache: 0x5598ce88d090 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_options: 2023-10-09T05:51:23.037 INFO:journalctl@ceph.mon.a.smithi107.stdout: capacity : 536870912 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: num_shard_bits : 4 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: strict_capacity_limit : 0 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_compressed: (nil) 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: persistent_cache: (nil) 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_size: 4096 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_size_deviation: 10 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_restart_interval: 16 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_block_restart_interval: 1 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: metadata_block_size: 4096 2023-10-09T05:51:23.038 INFO:journalctl@ceph.mon.a.smithi107.stdout: partition_filters: 0 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: use_delta_encoding: 1 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: whole_key_filtering: 1 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: verify_compression: 0 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: format_version: 4 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: enable_index_compression: 1 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_align: 0 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression: NoCompression 2023-10-09T05:51:23.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 bash[101036]: b3f3a3055d815d719b789734cd608f9bc277544a5c87d93a8707e62f41111da5 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.num_levels: 7 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:51:23.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:51:23.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:51:23.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:51:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:51:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.ttl: 2592000 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:51:23.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830682776091, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T05:51:23.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830682776783, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80160, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78428, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72636, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696830682, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9da4782a-2b14-4df3-9d90-f353f10b30c2", "db_session_id": "51GLA70NPGUCB9YGSYX0"}} 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830682779701, "job": 1, "event": "recovery_finished"} 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5598ce821180 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: DB pointer 0x5598ce8f6000 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** DB Stats ** 2023-10-09T05:51:23.047 INFO:journalctl@ceph.mon.a.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.048 INFO:journalctl@ceph.mon.a.smithi107.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:51:23.049 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative compaction: 0.00 GB write, 13.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval compaction: 0.00 GB write, 13.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:23.050 INFO:journalctl@ceph.mon.a.smithi107.stdout: L0 2/0 80.04 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: Sum 2/0 80.04 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.2 0.00 0.00 1 0.000 0 0 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:51:23.051 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative compaction: 0.00 GB write, 13.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: starting mon.a rank 0 at public addrs [v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] at bind addrs [v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???) e1 preinit fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).mds e1 new map 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).mds e1 print_map 2023-10-09T05:51:23.052 INFO:journalctl@ceph.mon.a.smithi107.stdout: e1 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout: legacy client fscid: -1 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout: No filesystems configured 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 systemd[1]: Started Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T05:51:23.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: monmap e1: 1 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0]} removed_ranks: {} 2023-10-09T05:51:23.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: fsmap 2023-10-09T05:51:23.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T05:51:23.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:22 smithi107 ceph-mon[101102]: mgrmap e1: no daemons active 2023-10-09T05:51:24.079 INFO:teuthology.orchestra.run.smithi107.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T05:51:24.080 INFO:teuthology.orchestra.run.smithi107.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:51:24.080 INFO:teuthology.orchestra.run.smithi107.stdout:Creating mgr... 2023-10-09T05:51:24.081 INFO:teuthology.orchestra.run.smithi107.stdout:Verifying port 9283 ... 2023-10-09T05:51:24.209 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T05:51:24.209 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T05:51:24.298 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7.target.wants/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@.service. 2023-10-09T05:51:24.358 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:24 smithi107 systemd[1]: Starting Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:51:24.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:24 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/562951300' entity='client.admin' 2023-10-09T05:51:24.994 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:51:24.994 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout disabled 2023-10-09T05:51:25.001 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:51:25.001 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:51:25.001 INFO:teuthology.orchestra.run.smithi107.stdout:firewalld.service is not enabled 2023-10-09T05:51:25.001 INFO:teuthology.orchestra.run.smithi107.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T05:51:25.009 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:51:25.009 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout disabled 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:firewalld.service is not enabled 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mgr to start... 2023-10-09T05:51:25.017 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mgr... 2023-10-09T05:51:25.064 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:24 smithi107 bash[101483]: 64caccec47d25147b3079387044924a81e2b54f14ecf9b68f04d7bae65fa598b 2023-10-09T05:51:25.064 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:24 smithi107 systemd[1]: Started Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:51:25.337 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.063+0000 7f1c2dc9a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:51:25.337 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.143+0000 7f1c2dc9a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:25.337 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.237+0000 7f1c2dc9a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:51:25.337 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.336+0000 7f1c2dc9a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:51:25.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.503+0000 7f1c2dc9a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:51:26.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:25 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1793457991' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:26.046 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.826+0000 7f1c2dc9a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:51:26.046 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:25 smithi107 conmon[101524]: 2023-10-09T05:51:25.915+0000 7f1c2dc9a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:51:26.331 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:26.331 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:26.331 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsid": "b7a98308-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:26.331 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:51:26.332 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:51:26.333 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:51:26.334 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:51:26.336 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.337 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:26.338 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:51:17.417494+0000", 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:26.339 INFO:teuthology.orchestra.run.smithi107.stdout:mgr not available, waiting (1/15)... 2023-10-09T05:51:26.339 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:26 smithi107 conmon[101524]: 2023-10-09T05:51:26.044+0000 7f1c2dc9a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:51:26.340 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:26 smithi107 conmon[101524]: 2023-10-09T05:51:26.136+0000 7f1c2dc9a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:51:26.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:26 smithi107 conmon[101524]: 2023-10-09T05:51:26.551+0000 7f1c2dc9a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:51:27.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:27 smithi107 conmon[101524]: 2023-10-09T05:51:27.109+0000 7f1c2dc9a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:51:27.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:27 smithi107 conmon[101524]: 2023-10-09T05:51:27.274+0000 7f1c2dc9a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:51:28.288 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.030+0000 7f1c2dc9a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:28.288 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.120+0000 7f1c2dc9a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:51:28.553 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.287+0000 7f1c2dc9a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:28.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.581+0000 7f1c2dc9a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:51:28.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.668+0000 7f1c2dc9a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:51:29.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:28 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2540175923' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:29.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:28 smithi107 conmon[101524]: 2023-10-09T05:51:28.969+0000 7f1c2dc9a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsid": "b7a98308-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:51:29.591 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:51:29.592 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:51:29.593 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:51:29.597 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:51:29.598 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:51:17.417494+0000", 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:29.600 INFO:teuthology.orchestra.run.smithi107.stdout:mgr not available, waiting (2/15)... 2023-10-09T05:51:29.600 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:29 smithi107 conmon[101524]: 2023-10-09T05:51:29.228+0000 7f1c2dc9a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:51:30.228 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:29 smithi107 conmon[101524]: 2023-10-09T05:51:29.918+0000 7f1c2dc9a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:51:30.228 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:30 smithi107 conmon[101524]: 2023-10-09T05:51:30.045+0000 7f1c2dc9a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:51:30.546 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:30 smithi107 conmon[101524]: 2023-10-09T05:51:30.227+0000 7f1c2dc9a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:51:30.546 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:30 smithi107 conmon[101524]: 2023-10-09T05:51:30.337+0000 7f1c2dc9a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:30.900 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:30 smithi107 conmon[101524]: 2023-10-09T05:51:30.545+0000 7f1c2dc9a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:51:31.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:31 smithi107 conmon[101524]: 2023-10-09T05:51:31.372+0000 7f1c2dc9a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: Activating manager daemon y 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: mgrmap e2: y(active, starting, since 0.00270685s) 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:51:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: Manager daemon y is now available 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' 2023-10-09T05:51:31.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:31 smithi107 ceph-mon[101102]: from='mgr.14100 172.21.15.107:0/3135182670' entity='mgr.y' 2023-10-09T05:51:32.769 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:32.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:32.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsid": "b7a98308-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:32.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:51:32.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:51:32.778 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:51:32.779 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:51:32.780 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:51:17.417494+0000", 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:32.781 INFO:teuthology.orchestra.run.smithi107.stdout:mgr not available, waiting (3/15)... 2023-10-09T05:51:32.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1927433664' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:32.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:32 smithi107 ceph-mon[101102]: mgrmap e3: y(active, since 1.00581s) 2023-10-09T05:51:35.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:34 smithi107 ceph-mon[101102]: mgrmap e4: y(active, since 2s) 2023-10-09T05:51:35.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3704693945' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T05:51:36.220 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:36.220 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:36.220 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsid": "b7a98308-6667-11ee-8db5-212e2dc638e7", 2023-10-09T05:51:36.220 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 0 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T05:51:36.221 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T05:51:36.222 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T05:51:36.226 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:51:36.227 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ], 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T05:51:17.417494+0000", 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout }, 2023-10-09T05:51:36.228 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T05:51:36.229 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:36.229 INFO:teuthology.orchestra.run.smithi107.stdout:mgr is available 2023-10-09T05:51:37.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:36 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/413385445' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T05:51:37.527 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout fsid = b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T05:51:37.528 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T05:51:37.529 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T05:51:37.529 INFO:teuthology.orchestra.run.smithi107.stdout:Enabling cephadm module... 2023-10-09T05:51:39.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:38 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/856648775' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T05:51:39.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:38 smithi107 conmon[101524]: ignoring --setuser ceph since I am not root 2023-10-09T05:51:39.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:38 smithi107 conmon[101524]: ignoring --setgroup ceph since I am not root 2023-10-09T05:51:39.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:38 smithi107 conmon[101524]: 2023-10-09T05:51:38.945+0000 7fad9571e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:51:39.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.024+0000 7fad9571e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:39.474 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.113+0000 7fad9571e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.204+0000 7fad9571e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:51:39.475 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.367+0000 7fad9571e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:51:39.787 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.698+0000 7fad9571e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:51:40.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:39 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/856648775' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T05:51:40.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:39 smithi107 ceph-mon[101102]: mgrmap e5: y(active, since 7s) 2023-10-09T05:51:40.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.786+0000 7fad9571e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:51:40.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:39 smithi107 conmon[101524]: 2023-10-09T05:51:39.918+0000 7fad9571e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:51:40.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:40 smithi107 conmon[101524]: 2023-10-09T05:51:40.017+0000 7fad9571e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:51:40.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:40 smithi107 conmon[101524]: 2023-10-09T05:51:40.445+0000 7fad9571e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:51:40.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:40.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T05:51:40.776 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:51:40.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T05:51:40.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T05:51:40.777 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:40.777 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for the mgr to restart... 2023-10-09T05:51:40.777 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mgr epoch 5... 2023-10-09T05:51:41.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:40 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2231877318' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:51:41.044 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:41 smithi107 conmon[101524]: 2023-10-09T05:51:41.028+0000 7fad9571e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:51:41.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:41 smithi107 conmon[101524]: 2023-10-09T05:51:41.202+0000 7fad9571e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:51:42.212 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:41 smithi107 conmon[101524]: 2023-10-09T05:51:41.956+0000 7fad9571e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:42.212 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:42 smithi107 conmon[101524]: 2023-10-09T05:51:42.045+0000 7fad9571e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:51:42.479 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:42 smithi107 conmon[101524]: 2023-10-09T05:51:42.211+0000 7fad9571e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:51:42.846 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:42 smithi107 conmon[101524]: 2023-10-09T05:51:42.478+0000 7fad9571e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:51:42.846 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:42 smithi107 conmon[101524]: 2023-10-09T05:51:42.562+0000 7fad9571e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:51:43.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:42 smithi107 conmon[101524]: 2023-10-09T05:51:42.845+0000 7fad9571e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:51:43.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:43 smithi107 conmon[101524]: 2023-10-09T05:51:43.096+0000 7fad9571e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:51:44.101 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:43 smithi107 conmon[101524]: 2023-10-09T05:51:43.788+0000 7fad9571e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:51:44.101 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:43 smithi107 conmon[101524]: 2023-10-09T05:51:43.916+0000 7fad9571e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:51:44.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:44 smithi107 conmon[101524]: 2023-10-09T05:51:44.100+0000 7fad9571e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:51:44.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:44 smithi107 conmon[101524]: 2023-10-09T05:51:44.211+0000 7fad9571e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:51:44.900 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:44 smithi107 conmon[101524]: 2023-10-09T05:51:44.422+0000 7fad9571e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:51:45.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:45 smithi107 conmon[101524]: 2023-10-09T05:51:45.258+0000 7fad9571e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: Active manager daemon y restarted 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: Activating manager daemon y 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: mgrmap e6: y(active, starting, since 0.00545384s) 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:51:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: Manager daemon y is now available 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:45 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:46 smithi107 ceph-mon[101102]: Found migration_current of "None". Setting to last migration. 2023-10-09T05:51:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:46 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:46 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:51:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:46 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:51:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:46 smithi107 ceph-mon[101102]: mgrmap e7: y(active, since 1.00503s) 2023-10-09T05:51:47.016 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:51:47.017 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T05:51:47.017 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T05:51:47.017 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:51:47.017 INFO:teuthology.orchestra.run.smithi107.stdout:mgr epoch 5 is available 2023-10-09T05:51:47.017 INFO:teuthology.orchestra.run.smithi107.stdout:Setting orchestrator backend to cephadm... 2023-10-09T05:51:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:51:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:51:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: [09/Oct/2023:05:51:46] ENGINE Bus STARTING 2023-10-09T05:51:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: [09/Oct/2023:05:51:46] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T05:51:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: [09/Oct/2023:05:51:46] ENGINE Bus STARTED 2023-10-09T05:51:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:47 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:48.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:48 smithi107 ceph-mon[101102]: mgrmap e8: y(active, since 2s) 2023-10-09T05:51:48.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:48 smithi107 ceph-mon[101102]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:48.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:48 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:48.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:48 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:49.723 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T05:51:49.724 INFO:teuthology.orchestra.run.smithi107.stdout:Generating ssh key... 2023-10-09T05:51:49.986 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:49 smithi107 ceph-mon[101102]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:50 smithi107 ceph-mon[101102]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:50 smithi107 ceph-mon[101102]: Generating ssh key... 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:50 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:50 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: Generating public/private rsa key pair. 2023-10-09T05:51:51.106 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: Your identification has been saved in /tmp/tmpf3gz0kat/key. 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: Your public key has been saved in /tmp/tmpf3gz0kat/key.pub. 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: The key fingerprint is: 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: SHA256:yMxPJMhlIEMWFI4Tp2PqBzwLI9e3/SUV6EQXv131HsE ceph-b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: The key's randomart image is: 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: +---[RSA 3072]----+ 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: |.oXo..o . o....| 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: | B + + . o . E+| 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: |=.. o . . o . ..o| 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: |+o . + + o . +o| 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: |=+. . * S . . . o| 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: |+o+ . = . | 2023-10-09T05:51:51.107 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: | o . . o . . | 2023-10-09T05:51:51.108 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: | . . o | 2023-10-09T05:51:51.108 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: | . | 2023-10-09T05:51:51.108 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:51:50 smithi107 conmon[101524]: +----[SHA256]-----+ 2023-10-09T05:51:52.765 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD40TnPsEeaPB9l8EeYjLx2RgeyYp3c4Q6++gh1nDIazMo1yiu4GLjST6SXraSmO/uv4xyQyCzdEkJ2w2BhAbGzc7/8/Ypin2dRBIK8/o36qMGaYlv42fwmdhnl5L+JxnG9NIF9RjbHHKAPA/4S/CWiGajFNyX9HLCvAUDOahG54Dp5LlaZxKRz8cJFITlB6wLPKkDyRNCkZy6bWuI9nqu2E5Uf0Tq8FN2mumRll0TldbvJH+4AMWOyqQSuO5pJ9nigho6sgpAiTNT9G7wHtdl4fu1x+KgzRLdO62xMrLJpNZZd5L/oJP5Zrq+pD4fSK4v83w6Ln7oe6Io68osm20pVOzvk2tw0TS25L1rGh9mK12kgFuI9cE6Qf0OgQLMiXJW/Uj2yQQjzH8lIO72EVymcwvcyIWj5MFmZ1Xv34uh66o9JlW7tdoWDthCXUDWO89RLg9KEBcRWbspiCWf4U9oPsvnqbExtwUYVNnihOfYTRDjCc3+tak5nR4mWDsdIfyc= ceph-b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:51:52.765 INFO:teuthology.orchestra.run.smithi107.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:51:52.765 INFO:teuthology.orchestra.run.smithi107.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T05:51:52.766 INFO:teuthology.orchestra.run.smithi107.stdout:Adding host smithi107... 2023-10-09T05:51:53.994 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:53 smithi107 ceph-mon[101102]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:55.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:54 smithi107 ceph-mon[101102]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi107", "addr": "172.21.15.107", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:55.959 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout Added host 'smithi107' with addr '172.21.15.107' 2023-10-09T05:51:55.960 INFO:teuthology.orchestra.run.smithi107.stdout:Deploying unmanaged mon service... 2023-10-09T05:51:56.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:55 smithi107 ceph-mon[101102]: Deploying cephadm binary to smithi107 2023-10-09T05:51:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:55 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:55 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:51:57.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:56 smithi107 ceph-mon[101102]: Added host smithi107 2023-10-09T05:51:57.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:56 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:57.430 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T05:51:57.430 INFO:teuthology.orchestra.run.smithi107.stdout:Deploying unmanaged mgr service... 2023-10-09T05:51:57.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:57 smithi107 ceph-mon[101102]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:57 smithi107 ceph-mon[101102]: Saving service mon spec with placement count:5 2023-10-09T05:51:59.247 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T05:51:59.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:59 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:59 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:51:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:59 smithi107 ceph-mon[101102]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:51:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:59 smithi107 ceph-mon[101102]: Saving service mgr spec with placement count:2 2023-10-09T05:51:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:51:59 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:52:01.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:00 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/254495679' entity='client.admin' 2023-10-09T05:52:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:00 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:52:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:00 smithi107 ceph-mon[101102]: from='mgr.14120 172.21.15.107:0/3895500760' entity='mgr.y' 2023-10-09T05:52:01.938 INFO:teuthology.orchestra.run.smithi107.stdout:Enabling the dashboard module... 2023-10-09T05:52:02.478 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:02 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3259868807' entity='client.admin' 2023-10-09T05:52:04.050 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:03 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3570317032' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T05:52:04.050 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:03 smithi107 conmon[101524]: ignoring --setuser ceph since I am not root 2023-10-09T05:52:04.050 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:03 smithi107 conmon[101524]: ignoring --setgroup ceph since I am not root 2023-10-09T05:52:04.050 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:03 smithi107 conmon[101524]: 2023-10-09T05:52:03.969+0000 7f74dd8791c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:52:04.358 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.049+0000 7f74dd8791c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:52:04.358 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.136+0000 7f74dd8791c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:52:04.358 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.226+0000 7f74dd8791c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:52:04.611 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.399+0000 7f74dd8791c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:52:04.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:04 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3570317032' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T05:52:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:04 smithi107 ceph-mon[101102]: mgrmap e9: y(active, since 18s) 2023-10-09T05:52:04.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.721+0000 7f74dd8791c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:52:04.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.821+0000 7f74dd8791c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:52:05.352 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:04 smithi107 conmon[101524]: 2023-10-09T05:52:04.953+0000 7f74dd8791c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:52:05.353 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:05 smithi107 conmon[101524]: 2023-10-09T05:52:05.047+0000 7f74dd8791c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:52:05.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:05 smithi107 conmon[101524]: 2023-10-09T05:52:05.481+0000 7f74dd8791c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for the mgr to restart... 2023-10-09T05:52:05.883 INFO:teuthology.orchestra.run.smithi107.stdout:Waiting for mgr epoch 9... 2023-10-09T05:52:06.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:05 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1777002029' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T05:52:06.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:06 smithi107 conmon[101524]: 2023-10-09T05:52:06.048+0000 7f74dd8791c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:52:06.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:06 smithi107 conmon[101524]: 2023-10-09T05:52:06.226+0000 7f74dd8791c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:52:07.246 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:06 smithi107 conmon[101524]: 2023-10-09T05:52:06.990+0000 7f74dd8791c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:52:07.247 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:07 smithi107 conmon[101524]: 2023-10-09T05:52:07.079+0000 7f74dd8791c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:52:07.887 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:07 smithi107 conmon[101524]: 2023-10-09T05:52:07.246+0000 7f74dd8791c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:52:07.887 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:07 smithi107 conmon[101524]: 2023-10-09T05:52:07.517+0000 7f74dd8791c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:52:07.887 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:07 smithi107 conmon[101524]: 2023-10-09T05:52:07.602+0000 7f74dd8791c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:52:07.887 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:07 smithi107 conmon[101524]: 2023-10-09T05:52:07.887+0000 7f74dd8791c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:52:08.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:08 smithi107 conmon[101524]: 2023-10-09T05:52:08.142+0000 7f74dd8791c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:52:09.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:08 smithi107 conmon[101524]: 2023-10-09T05:52:08.837+0000 7f74dd8791c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:52:09.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:08 smithi107 conmon[101524]: 2023-10-09T05:52:08.966+0000 7f74dd8791c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:52:09.478 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:09 smithi107 conmon[101524]: 2023-10-09T05:52:09.153+0000 7f74dd8791c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:52:09.478 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:09 smithi107 conmon[101524]: 2023-10-09T05:52:09.266+0000 7f74dd8791c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:52:10.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:09 smithi107 conmon[101524]: 2023-10-09T05:52:09.478+0000 7f74dd8791c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:52:10 smithi107 conmon[101524]: 2023-10-09T05:52:10.314+0000 7f74dd8791c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: Active manager daemon y restarted 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: Activating manager daemon y 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: mgrmap e10: y(active, starting, since 0.00302257s) 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:52:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:52:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:52:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: Manager daemon y is now available 2023-10-09T05:52:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:11.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: [09/Oct/2023:05:52:10] ENGINE Bus STARTING 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: [09/Oct/2023:05:52:10] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: [09/Oct/2023:05:52:10] ENGINE Bus STARTED 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:11 smithi107 ceph-mon[101102]: mgrmap e11: y(active, since 1.00528s) 2023-10-09T05:52:11.927 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout { 2023-10-09T05:52:11.927 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T05:52:11.927 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T05:52:11.928 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout } 2023-10-09T05:52:11.928 INFO:teuthology.orchestra.run.smithi107.stdout:mgr epoch 9 is available 2023-10-09T05:52:11.928 INFO:teuthology.orchestra.run.smithi107.stdout:Generating a dashboard self-signed certificate... 2023-10-09T05:52:12.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:12 smithi107 ceph-mon[101102]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T05:52:12.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:12 smithi107 ceph-mon[101102]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T05:52:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:12 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:13.344 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T05:52:13.344 INFO:teuthology.orchestra.run.smithi107.stdout:Creating initial admin user... 2023-10-09T05:52:13.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:13 smithi107 ceph-mon[101102]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:13 smithi107 ceph-mon[101102]: mgrmap e12: y(active, since 2s) 2023-10-09T05:52:15.101 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$71djyb/w3Z/5ijjexVT1M.5ziwhc4VHgFh6uhJd/xtxGsGuEvYQ9O", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696830734, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T05:52:15.102 INFO:teuthology.orchestra.run.smithi107.stdout:Fetching dashboard port number... 2023-10-09T05:52:15.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:15 smithi107 ceph-mon[101102]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:16.376 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T05:52:16.388 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T05:52:16.388 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout disabled 2023-10-09T05:52:16.395 INFO:teuthology.orchestra.run.smithi107.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T05:52:16.395 INFO:teuthology.orchestra.run.smithi107.stdout:systemctl: stdout inactive 2023-10-09T05:52:16.395 INFO:teuthology.orchestra.run.smithi107.stdout:firewalld.service is not enabled 2023-10-09T05:52:16.395 INFO:teuthology.orchestra.run.smithi107.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout:Ceph Dashboard is now available at: 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout: URL: https://smithi107.front.sepia.ceph.com:8443/ 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout: User: admin 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout: Password: k4ts7rbja6 2023-10-09T05:52:16.397 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:16.398 INFO:teuthology.orchestra.run.smithi107.stdout:Saving cluster configuration to /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config directory 2023-10-09T05:52:16.398 INFO:teuthology.orchestra.run.smithi107.stdout:Enabling autotune for osd_memory_target 2023-10-09T05:52:16.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:16 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3192815391' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T05:52:17.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:17 smithi107 ceph-mon[101102]: mgrmap e13: y(active, since 6s) 2023-10-09T05:52:19.017 INFO:teuthology.orchestra.run.smithi107.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.018 INFO:teuthology.orchestra.run.smithi107.stdout: ceph telemetry on 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout:For more information see: 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:19.019 INFO:teuthology.orchestra.run.smithi107.stdout:Bootstrap complete. 2023-10-09T05:52:19.070 INFO:tasks.cephadm:Fetching config... 2023-10-09T05:52:19.070 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:52:19.071 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T05:52:19.091 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T05:52:19.091 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:52:19.091 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T05:52:19.148 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T05:52:19.148 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:52:19.148 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd if=/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T05:52:19.217 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T05:52:19.217 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:52:19.217 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T05:52:19.274 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T05:52:19.274 DEBUG:teuthology.orchestra.run.smithi107:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD40TnPsEeaPB9l8EeYjLx2RgeyYp3c4Q6++gh1nDIazMo1yiu4GLjST6SXraSmO/uv4xyQyCzdEkJ2w2BhAbGzc7/8/Ypin2dRBIK8/o36qMGaYlv42fwmdhnl5L+JxnG9NIF9RjbHHKAPA/4S/CWiGajFNyX9HLCvAUDOahG54Dp5LlaZxKRz8cJFITlB6wLPKkDyRNCkZy6bWuI9nqu2E5Uf0Tq8FN2mumRll0TldbvJH+4AMWOyqQSuO5pJ9nigho6sgpAiTNT9G7wHtdl4fu1x+KgzRLdO62xMrLJpNZZd5L/oJP5Zrq+pD4fSK4v83w6Ln7oe6Io68osm20pVOzvk2tw0TS25L1rGh9mK12kgFuI9cE6Qf0OgQLMiXJW/Uj2yQQjzH8lIO72EVymcwvcyIWj5MFmZ1Xv34uh66o9JlW7tdoWDthCXUDWO89RLg9KEBcRWbspiCWf4U9oPsvnqbExtwUYVNnihOfYTRDjCc3+tak5nR4mWDsdIfyc= ceph-b7a98308-6667-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:52:19.352 INFO:teuthology.orchestra.run.smithi107.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD40TnPsEeaPB9l8EeYjLx2RgeyYp3c4Q6++gh1nDIazMo1yiu4GLjST6SXraSmO/uv4xyQyCzdEkJ2w2BhAbGzc7/8/Ypin2dRBIK8/o36qMGaYlv42fwmdhnl5L+JxnG9NIF9RjbHHKAPA/4S/CWiGajFNyX9HLCvAUDOahG54Dp5LlaZxKRz8cJFITlB6wLPKkDyRNCkZy6bWuI9nqu2E5Uf0Tq8FN2mumRll0TldbvJH+4AMWOyqQSuO5pJ9nigho6sgpAiTNT9G7wHtdl4fu1x+KgzRLdO62xMrLJpNZZd5L/oJP5Zrq+pD4fSK4v83w6Ln7oe6Io68osm20pVOzvk2tw0TS25L1rGh9mK12kgFuI9cE6Qf0OgQLMiXJW/Uj2yQQjzH8lIO72EVymcwvcyIWj5MFmZ1Xv34uh66o9JlW7tdoWDthCXUDWO89RLg9KEBcRWbspiCWf4U9oPsvnqbExtwUYVNnihOfYTRDjCc3+tak5nR4mWDsdIfyc= ceph-b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:19.367 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD40TnPsEeaPB9l8EeYjLx2RgeyYp3c4Q6++gh1nDIazMo1yiu4GLjST6SXraSmO/uv4xyQyCzdEkJ2w2BhAbGzc7/8/Ypin2dRBIK8/o36qMGaYlv42fwmdhnl5L+JxnG9NIF9RjbHHKAPA/4S/CWiGajFNyX9HLCvAUDOahG54Dp5LlaZxKRz8cJFITlB6wLPKkDyRNCkZy6bWuI9nqu2E5Uf0Tq8FN2mumRll0TldbvJH+4AMWOyqQSuO5pJ9nigho6sgpAiTNT9G7wHtdl4fu1x+KgzRLdO62xMrLJpNZZd5L/oJP5Zrq+pD4fSK4v83w6Ln7oe6Io68osm20pVOzvk2tw0TS25L1rGh9mK12kgFuI9cE6Qf0OgQLMiXJW/Uj2yQQjzH8lIO72EVymcwvcyIWj5MFmZ1Xv34uh66o9JlW7tdoWDthCXUDWO89RLg9KEBcRWbspiCWf4U9oPsvnqbExtwUYVNnihOfYTRDjCc3+tak5nR4mWDsdIfyc= ceph-b7a98308-6667-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T05:52:19.442 INFO:teuthology.orchestra.run.smithi150.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD40TnPsEeaPB9l8EeYjLx2RgeyYp3c4Q6++gh1nDIazMo1yiu4GLjST6SXraSmO/uv4xyQyCzdEkJ2w2BhAbGzc7/8/Ypin2dRBIK8/o36qMGaYlv42fwmdhnl5L+JxnG9NIF9RjbHHKAPA/4S/CWiGajFNyX9HLCvAUDOahG54Dp5LlaZxKRz8cJFITlB6wLPKkDyRNCkZy6bWuI9nqu2E5Uf0Tq8FN2mumRll0TldbvJH+4AMWOyqQSuO5pJ9nigho6sgpAiTNT9G7wHtdl4fu1x+KgzRLdO62xMrLJpNZZd5L/oJP5Zrq+pD4fSK4v83w6Ln7oe6Io68osm20pVOzvk2tw0TS25L1rGh9mK12kgFuI9cE6Qf0OgQLMiXJW/Uj2yQQjzH8lIO72EVymcwvcyIWj5MFmZ1Xv34uh66o9JlW7tdoWDthCXUDWO89RLg9KEBcRWbspiCWf4U9oPsvnqbExtwUYVNnihOfYTRDjCc3+tak5nR4mWDsdIfyc= ceph-b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:19.454 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T05:52:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:19 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2728057570' entity='client.admin' 2023-10-09T05:52:19.800 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:22.546 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T05:52:22.546 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T05:52:22.572 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:22.572 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:22.572 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi107", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/221592434' entity='client.admin' 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:22.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:23.321 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:25.621 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi150 2023-10-09T05:52:25.621 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:52:25.621 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/etc/ceph/ceph.conf 2023-10-09T05:52:25.642 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:52:25.642 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:25.701 INFO:tasks.cephadm:Adding host smithi150 to orchestrator... 2023-10-09T05:52:25.702 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi150 2023-10-09T05:52:26.061 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:26.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:52:26.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:26.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:26.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:28 smithi107 ceph-mon[101102]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi150", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:29.541 INFO:teuthology.orchestra.run.smithi107.stdout:Added host 'smithi150' with addr '172.21.15.150' 2023-10-09T05:52:29.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:29 smithi107 ceph-mon[101102]: Deploying cephadm binary to smithi150 2023-10-09T05:52:29.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:29 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:29.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:29 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:30.297 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T05:52:30.634 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:31.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:30 smithi107 ceph-mon[101102]: Added host smithi150 2023-10-09T05:52:31.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:31.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:31 smithi107 ceph-mon[101102]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:31.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:32.277 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:52:32.277 INFO:teuthology.orchestra.run.smithi107.stdout:[{"addr": "172.21.15.107", "hostname": "smithi107", "labels": [], "status": ""}, {"addr": "172.21.15.150", "hostname": "smithi150", "labels": [], "status": ""}] 2023-10-09T05:52:32.961 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T05:52:32.962 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T05:52:33.293 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:34.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:33 smithi107 ceph-mon[101102]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:52:34.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:33 smithi107 ceph-mon[101102]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi150", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2264448061' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:35 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:35.809 INFO:teuthology.orchestra.run.smithi107.stderr:adjusted tunables profile to default 2023-10-09T05:52:36.540 INFO:tasks.cephadm:Adding mon.a on smithi107 2023-10-09T05:52:36.540 INFO:tasks.cephadm:Adding mon.c on smithi107 2023-10-09T05:52:36.540 INFO:tasks.cephadm:Adding mon.b on smithi150 2023-10-09T05:52:36.541 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b' 2023-10-09T05:52:36.908 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:37.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:36 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:52:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:36 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2264448061' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T05:52:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:36 smithi107 ceph-mon[101102]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:37 smithi107 ceph-mon[101102]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:38.549 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled mon update... 2023-10-09T05:52:39.192 DEBUG:teuthology.orchestra.run.smithi107:mon.c> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T05:52:39.195 DEBUG:teuthology.orchestra.run.smithi150:mon.b> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T05:52:39.197 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:52:39.197 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:52:39.229 INFO:journalctl@ceph.mon.b.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:52:39.231 INFO:journalctl@ceph.mon.c.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: Saving service mon spec with placement smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b;count:3 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:39.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:39.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:39.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:39 smithi107 ceph-mon[101102]: Deploying daemon mon.b on smithi150 2023-10-09T05:52:40.225 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:41.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:41 smithi107 ceph-mon[101102]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:41 smithi150 systemd[1]: Starting Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:52:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:52:42.497 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":1,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:51:14.581098Z","created":"2023-10-09T05:51:14.581098Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:3300","nonce":0},{"type":"v1","addr":"172.21.15.107:6789","nonce":0}]},"addr":"172.21.15.107:6789/0","public_addr":"172.21.15.107:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T05:52:42.499 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 1 2023-10-09T05:52:42.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:52:42.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T05:52:42.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: pidfile_write: ignore empty --pid-file 2023-10-09T05:52:42.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: load: jerasure load: lrc 2023-10-09T05:52:42.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: DB SUMMARY 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: DB Session ID: QXYNOO3CY897OPAZ37NS 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T05:52:42.801 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.env: 0x55df2e7bba80 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.fs: Posix File System 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.info_log: 0x55df2f1fc5a0 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.statistics: (nil) 2023-10-09T05:52:42.802 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.use_fsync: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 bash[105860]: 0a878f23149465e5d975db267aa084b2a1b166f9f6d3c833402f731276140fa0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:52:42.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.db_log_dir: 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.write_buffer_manager: 0x55df2ff6e900 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:52:42.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.unordered_write: 0 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:52:42.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.row_cache: None 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.wal_filter: None 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:52:42.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:52:42.807 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_open_files: -1 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Compression algorithms supported: 2023-10-09T05:52:42.808 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kZSTD supported: 0 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.merge_operator: 2023-10-09T05:52:42.809 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_filter: None 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55df2f24efb8) 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_type: 0 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_index_type: 0 2023-10-09T05:52:42.810 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_shortening: 1 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: hash_index_allow_collision: 1 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: checksum: 1 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: no_block_cache: 0 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache: 0x55df2f281090 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_options: 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: capacity : 536870912 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_shard_bits : 4 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: strict_capacity_limit : 0 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_compressed: (nil) 2023-10-09T05:52:42.811 INFO:journalctl@ceph.mon.b.smithi150.stdout: persistent_cache: (nil) 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size: 4096 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size_deviation: 10 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_restart_interval: 16 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_block_restart_interval: 1 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: metadata_block_size: 4096 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: partition_filters: 0 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_delta_encoding: 1 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: whole_key_filtering: 1 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: verify_compression: 0 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: format_version: 4 2023-10-09T05:52:42.812 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_index_compression: 1 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_align: 0 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression: NoCompression 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.num_levels: 7 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:52:42.813 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:52:42.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:52:42.815 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:52:42.816 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:52:42.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:52:42.818 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:52:42.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.ttl: 2592000 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:52:42.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:52:42.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:52:42.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T05:52:42.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T05:52:42.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830762515495, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T05:52:42.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T05:52:42.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:52:42.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830762515783, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696830762, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "954e5817-a921-4b9f-8b24-a68cf5f29f93", "db_session_id": "QXYNOO3CY897OPAZ37NS"}} 2023-10-09T05:52:42.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T05:52:42.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830762516098, "job": 1, "event": "recovery_finished"} 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 systemd[1]: Started Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55df2f215180 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: DB pointer 0x55df2f2ea000 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:52:42.823 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** DB Stats ** 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:42.824 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.825 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:42.826 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:42.827 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.00 GB write, 0.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:42.828 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:42.829 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.5 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.830 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: using public_addr v2:172.21.15.150:0/0 -> [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: starting mon.b rank -1 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(???) e0 preinit fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).mds e1 new map 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: e1 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: legacy client fscid: -1 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T05:52:42.831 INFO:journalctl@ceph.mon.b.smithi150.stdout: No filesystems configured 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:42.832 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: Saving service mon spec with placement smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b;count:3 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: Deploying daemon mon.b on smithi150 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:42.833 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:52:42.834 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:42 smithi150 ceph-mon[105942]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-09T05:52:44.296 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:52:44.297 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:52:44.638 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: mon.a calling monitor election 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: mon.b calling monitor election 2023-10-09T05:52:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: monmap e2: 2 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0]} removed_ranks: {} 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: fsmap 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: mgrmap e13: y(active, since 37s) 2023-10-09T05:52:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: overall HEALTH_OK 2023-10-09T05:52:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: mon.a calling monitor election 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: mon.b calling monitor election 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: monmap e2: 2 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0]} removed_ranks: {} 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: fsmap 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: mgrmap e13: y(active, since 37s) 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: overall HEALTH_OK 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:47.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:47.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:48 smithi107 ceph-mon[101102]: Deploying daemon mon.c on smithi107 2023-10-09T05:52:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:48 smithi150 ceph-mon[105942]: Deploying daemon mon.c on smithi107 2023-10-09T05:52:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:49.248 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:52:49.249 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":2,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:52:42.538048Z","created":"2023-10-09T05:51:14.581098Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:3300","nonce":0},{"type":"v1","addr":"172.21.15.107:6789","nonce":0}]},"addr":"172.21.15.107:6789/0","public_addr":"172.21.15.107:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T05:52:49.250 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 2 2023-10-09T05:52:50.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:49 smithi107 ceph-mon[101102]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:50.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/2819646731' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:50.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:49 smithi150 ceph-mon[105942]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/2819646731' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:50.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 systemd[1]: Starting Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:52:50.854 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T05:52:50.854 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T05:52:51.193 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pidfile_write: ignore empty --pid-file 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: load: jerasure load: lrc 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: RocksDB version: 6.15.5 2023-10-09T05:52:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Compile date Oct 8 2023 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: DB SUMMARY 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: DB Session ID: TELGK9VJ2RAGCBLRZNV4 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: CURRENT file: CURRENT 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-10-09T05:52:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.error_if_exists: 0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.create_if_missing: 0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.paranoid_checks: 1 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.env: 0x55668d02ca80 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.fs: Posix File System 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.info_log: 0x55668ecfa5a0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.statistics: (nil) 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.use_fsync: 0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_log_file_size: 0 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T05:52:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_fallocate: 1 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.use_direct_reads: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.db_log_dir: 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T05:52:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.write_buffer_manager: 0x55668fa6c930 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T05:52:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.unordered_write: 0 2023-10-09T05:52:51.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.row_cache: None 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.wal_filter: None 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.preserve_deletes: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.two_write_queues: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.atomic_flush: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T05:52:51.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.log_readahead_size: 0 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_background_jobs: 2 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_background_compactions: -1 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_subcompactions: 1 2023-10-09T05:52:51.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_open_files: -1 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T05:52:51.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_background_flushes: -1 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Compression algorithms supported: 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kZSTD supported: 0 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kXpressCompression supported: 0 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kZlibCompression supported: 1 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kSnappyCompression supported: 1 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kLZ4Compression supported: 1 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: kBZip2Compression supported: 0 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-10-09T05:52:51.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.merge_operator: 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_filter: None 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55668ed4cfb8) 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout: cache_index_and_filter_blocks: 1 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T05:52:51.411 INFO:journalctl@ceph.mon.c.smithi107.stdout: pin_top_level_index_and_filter: 1 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_type: 0 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: data_block_index_type: 0 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_shortening: 1 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: hash_index_allow_collision: 1 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: checksum: 1 2023-10-09T05:52:51.413 INFO:journalctl@ceph.mon.c.smithi107.stdout: no_block_cache: 0 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache: 0x55668ed7f090 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_name: BinnedLRUCache 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_options: 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: capacity : 536870912 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: num_shard_bits : 4 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: strict_capacity_limit : 0 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: high_pri_pool_ratio: 0.000 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_compressed: (nil) 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: persistent_cache: (nil) 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_size: 4096 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_size_deviation: 10 2023-10-09T05:52:51.414 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_restart_interval: 16 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_block_restart_interval: 1 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: metadata_block_size: 4096 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: partition_filters: 0 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: use_delta_encoding: 1 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: whole_key_filtering: 1 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: verify_compression: 0 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: read_amp_bytes_per_bit: 0 2023-10-09T05:52:51.415 INFO:journalctl@ceph.mon.c.smithi107.stdout: format_version: 4 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout: enable_index_compression: 1 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_align: 0 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression: NoCompression 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.num_levels: 7 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T05:52:51.416 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T05:52:51.417 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T05:52:51.418 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T05:52:51.419 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T05:52:51.420 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T05:52:51.421 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.table_properties_collectors: 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.inplace_update_support: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.bloom_locality: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.max_successive_merges: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T05:52:51.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.ttl: 2592000 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.enable_blob_files: false 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.min_blob_size: 0 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T05:52:51.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830770970317, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830770970670, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696830770, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3c1baea1-e910-4ba2-ac0b-ff6de1d33536", "db_session_id": "TELGK9VJ2RAGCBLRZNV4"}} 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696830770971089, "job": 1, "event": "recovery_finished"} 2023-10-09T05:52:51.424 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55668ed13180 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: DB pointer 0x55668ede8000 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** DB Stats ** 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:51.425 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:51.426 INFO:journalctl@ceph.mon.c.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:51.427 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T05:52:51.428 INFO:journalctl@ceph.mon.c.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative compaction: 0.00 GB write, 0.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T05:52:51.429 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: using public_addrv [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: starting mon.c rank -1 at public addrs [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] at bind addrs [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(???) e0 preinit fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 bash[118835]: c53cec84dedf147ec9773ea41e35a9f309e76085977281b3b77e429aa82c65e9 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).mds e1 new map 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout: e1 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout: legacy client fscid: -1 2023-10-09T05:52:51.430 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout: No filesystems configured 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 systemd[1]: Started Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: Saving service mon spec with placement smithi107:172.21.15.107=a;smithi107:[v2:172.21.15.107:3301,v1:172.21.15.107:6790]=c;smithi150:172.21.15.150=b;count:3 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:51.432 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: Deploying daemon mon.b on smithi150 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.a calling monitor election 2023-10-09T05:52:51.433 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.b calling monitor election 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: monmap e2: 2 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0]} removed_ranks: {} 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: fsmap 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mgrmap e13: y(active, since 37s) 2023-10-09T05:52:51.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: overall HEALTH_OK 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:51.436 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: Deploying daemon mon.c on smithi107 2023-10-09T05:52:51.437 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:51.437 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:51.437 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/2819646731' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:51.437 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T05:52:51.437 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:50 smithi107 ceph-mon[118900]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-09T05:52:56.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: mon.a calling monitor election 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: mon.b calling monitor election 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: mon.c calling monitor election 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: fsmap 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: mgrmap e13: y(active, since 45s) 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: overall HEALTH_OK 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: mon.a calling monitor election 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: mon.b calling monitor election 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: mon.c calling monitor election 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: fsmap 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: mgrmap e13: y(active, since 45s) 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: overall HEALTH_OK 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:56 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: mon.a calling monitor election 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: mon.b calling monitor election 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: mon.c calling monitor election 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: fsmap 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T05:52:56.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: mgrmap e13: y(active, since 45s) 2023-10-09T05:52:56.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: overall HEALTH_OK 2023-10-09T05:52:56.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:56 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:56.421 INFO:teuthology.orchestra.run.smithi150.stdout: 2023-10-09T05:52:56.421 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":3,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","modified":"2023-10-09T05:52:50.995078Z","created":"2023-10-09T05:51:14.581098Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:3300","nonce":0},{"type":"v1","addr":"172.21.15.107:6789","nonce":0}]},"addr":"172.21.15.107:6789/0","public_addr":"172.21.15.107:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:3301","nonce":0},{"type":"v1","addr":"172.21.15.107:6790","nonce":0}]},"addr":"172.21.15.107:6790/0","public_addr":"172.21.15.107:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T05:52:56.423 INFO:teuthology.orchestra.run.smithi150.stderr:dumped monmap epoch 3 2023-10-09T05:52:56.991 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T05:52:56.991 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T05:52:57.341 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:52:57.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/722788152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/722788152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:57.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/722788152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:52:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:52:58.599 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:58.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:58.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:52:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:52:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.180 INFO:teuthology.orchestra.run.smithi107.stdout:# minimal ceph.conf for b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:59.180 INFO:teuthology.orchestra.run.smithi107.stdout:[global] 2023-10-09T05:52:59.181 INFO:teuthology.orchestra.run.smithi107.stdout: fsid = b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T05:52:59.181 INFO:teuthology.orchestra.run.smithi107.stdout: mon_host = [v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] 2023-10-09T05:52:59.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[101102]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[101102]: Reconfiguring daemon mon.a on smithi107 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[101102]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2744560220' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[118900]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[118900]: Reconfiguring daemon mon.a on smithi107 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[118900]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:52:59 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2744560220' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:52:59.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:59 smithi150 ceph-mon[105942]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T05:52:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:59 smithi150 ceph-mon[105942]: Reconfiguring daemon mon.a on smithi107 2023-10-09T05:52:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:59 smithi150 ceph-mon[105942]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:52:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:52:59 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2744560220' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:00.250 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T05:53:00.250 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:53:00.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:53:00.279 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:53:00.279 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:53:00.346 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:53:00.346 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T05:53:00.382 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:53:00.382 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:53:00.454 INFO:tasks.cephadm:Adding mgr.y on smithi107 2023-10-09T05:53:00.454 INFO:tasks.cephadm:Adding mgr.x on smithi150 2023-10-09T05:53:00.454 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi107=y;smithi150=x' 2023-10-09T05:53:00.825 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:53:01.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: Reconfiguring daemon mon.c on smithi107 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:01.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: Reconfiguring daemon mon.c on smithi107 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: Reconfiguring mon.c (monmap changed)... 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: Reconfiguring daemon mon.c on smithi107 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.443 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled mgr update... 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: Reconfiguring daemon mon.b on smithi150 2023-10-09T05:53:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: Reconfiguring daemon mon.b on smithi150 2023-10-09T05:53:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: Reconfiguring mon.b (monmap changed)... 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: Reconfiguring daemon mon.b on smithi150 2023-10-09T05:53:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:03.311 DEBUG:teuthology.orchestra.run.smithi150:mgr.x> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T05:53:03.313 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T05:53:03.313 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:53:03.313 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:53:03.334 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:53:03.334 DEBUG:teuthology.orchestra.run.smithi107:> stat /dev/vg_nvme/lv_1 2023-10-09T05:53:03.342 INFO:journalctl@ceph.mgr.x.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:53:03.394 INFO:teuthology.orchestra.run.smithi107.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:53:03.394 INFO:teuthology.orchestra.run.smithi107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.394 INFO:teuthology.orchestra.run.smithi107.stdout:Device: 6h/6d Inode: 396355 Links: 1 2023-10-09T05:53:03.394 INFO:teuthology.orchestra.run.smithi107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.395 INFO:teuthology.orchestra.run.smithi107.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:03.395 INFO:teuthology.orchestra.run.smithi107.stdout:Access: 2023-10-09 05:52:50.384414289 +0000 2023-10-09T05:53:03.395 INFO:teuthology.orchestra.run.smithi107.stdout:Modify: 2023-10-09 05:43:16.615216873 +0000 2023-10-09T05:53:03.395 INFO:teuthology.orchestra.run.smithi107.stdout:Change: 2023-10-09 05:43:16.615216873 +0000 2023-10-09T05:53:03.395 INFO:teuthology.orchestra.run.smithi107.stdout: Birth: - 2023-10-09T05:53:03.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:53:03.467 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records in 2023-10-09T05:53:03.467 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records out 2023-10-09T05:53:03.467 INFO:teuthology.orchestra.run.smithi107.stderr:512 bytes copied, 0.000250812 s, 2.0 MB/s 2023-10-09T05:53:03.468 DEBUG:teuthology.orchestra.run.smithi107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:53:03.529 DEBUG:teuthology.orchestra.run.smithi107:> stat /dev/vg_nvme/lv_2 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Device: 6h/6d Inode: 395072 Links: 1 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Access: 2023-10-09 05:52:50.384414289 +0000 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Modify: 2023-10-09 05:43:17.161208072 +0000 2023-10-09T05:53:03.589 INFO:teuthology.orchestra.run.smithi107.stdout:Change: 2023-10-09 05:43:17.161208072 +0000 2023-10-09T05:53:03.590 INFO:teuthology.orchestra.run.smithi107.stdout: Birth: - 2023-10-09T05:53:03.590 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:53:03.658 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records in 2023-10-09T05:53:03.658 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records out 2023-10-09T05:53:03.658 INFO:teuthology.orchestra.run.smithi107.stderr:512 bytes copied, 0.000239839 s, 2.1 MB/s 2023-10-09T05:53:03.659 DEBUG:teuthology.orchestra.run.smithi107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:53:03.719 DEBUG:teuthology.orchestra.run.smithi107:> stat /dev/vg_nvme/lv_3 2023-10-09T05:53:03.778 INFO:teuthology.orchestra.run.smithi107.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:53:03.778 INFO:teuthology.orchestra.run.smithi107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.778 INFO:teuthology.orchestra.run.smithi107.stdout:Device: 6h/6d Inode: 393996 Links: 1 2023-10-09T05:53:03.778 INFO:teuthology.orchestra.run.smithi107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.778 INFO:teuthology.orchestra.run.smithi107.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:03.779 INFO:teuthology.orchestra.run.smithi107.stdout:Access: 2023-10-09 05:52:50.384414289 +0000 2023-10-09T05:53:03.779 INFO:teuthology.orchestra.run.smithi107.stdout:Modify: 2023-10-09 05:43:17.667199917 +0000 2023-10-09T05:53:03.779 INFO:teuthology.orchestra.run.smithi107.stdout:Change: 2023-10-09 05:43:17.667199917 +0000 2023-10-09T05:53:03.779 INFO:teuthology.orchestra.run.smithi107.stdout: Birth: - 2023-10-09T05:53:03.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:53:03.846 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records in 2023-10-09T05:53:03.846 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records out 2023-10-09T05:53:03.846 INFO:teuthology.orchestra.run.smithi107.stderr:512 bytes copied, 0.000218388 s, 2.3 MB/s 2023-10-09T05:53:03.847 DEBUG:teuthology.orchestra.run.smithi107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:53:03.906 DEBUG:teuthology.orchestra.run.smithi107:> stat /dev/vg_nvme/lv_4 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Device: 6h/6d Inode: 396590 Links: 1 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Access: 2023-10-09 05:52:50.384414289 +0000 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Modify: 2023-10-09 05:43:18.178191681 +0000 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout:Change: 2023-10-09 05:43:18.178191681 +0000 2023-10-09T05:53:03.965 INFO:teuthology.orchestra.run.smithi107.stdout: Birth: - 2023-10-09T05:53:03.966 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:53:04.033 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records in 2023-10-09T05:53:04.033 INFO:teuthology.orchestra.run.smithi107.stderr:1+0 records out 2023-10-09T05:53:04.033 INFO:teuthology.orchestra.run.smithi107.stderr:512 bytes copied, 0.00027342 s, 1.9 MB/s 2023-10-09T05:53:04.034 DEBUG:teuthology.orchestra.run.smithi107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi107=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: Saving service mgr spec with placement smithi107=y;smithi150=x;count:2 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:04.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:04.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:03 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.093 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:53:04.093 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T05:53:04.110 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T05:53:04.110 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_1 2023-10-09T05:53:04.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi107=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: Saving service mgr spec with placement smithi107=y;smithi150=x;count:2 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi107=y;smithi150=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: Saving service mgr spec with placement smithi107=y;smithi150=x;count:2 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:03 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:04.169 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 6h/6d Inode: 393298 Links: 1 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2023-10-09 05:52:41.754211969 +0000 2023-10-09T05:53:04.170 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2023-10-09 05:43:16.605842493 +0000 2023-10-09T05:53:04.171 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2023-10-09 05:43:16.605842493 +0000 2023-10-09T05:53:04.171 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: - 2023-10-09T05:53:04.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T05:53:04.237 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2023-10-09T05:53:04.237 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2023-10-09T05:53:04.237 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000211515 s, 2.4 MB/s 2023-10-09T05:53:04.239 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T05:53:04.297 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_2 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 6h/6d Inode: 396350 Links: 1 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2023-10-09 05:52:41.754211969 +0000 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2023-10-09 05:43:17.141831550 +0000 2023-10-09T05:53:04.354 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2023-10-09 05:43:17.141831550 +0000 2023-10-09T05:53:04.355 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: - 2023-10-09T05:53:04.355 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T05:53:04.419 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2023-10-09T05:53:04.420 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2023-10-09T05:53:04.420 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000282082 s, 1.8 MB/s 2023-10-09T05:53:04.421 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T05:53:04.478 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_3 2023-10-09T05:53:04.537 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T05:53:04.537 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.537 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 6h/6d Inode: 393437 Links: 1 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2023-10-09 05:53:04.528747472 +0000 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2023-10-09 05:43:17.641821343 +0000 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2023-10-09 05:43:17.641821343 +0000 2023-10-09T05:53:04.538 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: - 2023-10-09T05:53:04.539 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T05:53:04.605 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2023-10-09T05:53:04.606 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2023-10-09T05:53:04.606 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000239183 s, 2.1 MB/s 2023-10-09T05:53:04.607 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T05:53:04.664 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_4 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 6h/6d Inode: 391508 Links: 1 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T05:53:04.724 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2023-10-09 05:53:04.657744841 +0000 2023-10-09T05:53:04.725 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2023-10-09 05:43:18.177810402 +0000 2023-10-09T05:53:04.725 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2023-10-09 05:43:18.177810402 +0000 2023-10-09T05:53:04.725 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: - 2023-10-09T05:53:04.725 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T05:53:04.794 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2023-10-09T05:53:04.794 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2023-10-09T05:53:04.794 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000215655 s, 2.4 MB/s 2023-10-09T05:53:04.795 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T05:53:04.853 INFO:tasks.cephadm:Deploying osd.0 on smithi107 with /dev/vg_nvme/lv_4... 2023-10-09T05:53:04.853 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:53:05.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:04 smithi107 ceph-mon[101102]: Deploying daemon mgr.x on smithi150 2023-10-09T05:53:05.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:04 smithi107 ceph-mon[118900]: Deploying daemon mgr.x on smithi150 2023-10-09T05:53:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:04 smithi150 ceph-mon[105942]: Deploying daemon mgr.x on smithi150 2023-10-09T05:53:05.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:04 smithi150 systemd[1]: Starting Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:53:05.197 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mgr[109204]: -- 172.21.15.150:0/3220570603 <== mon.1 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dbce5209c0 con 0x55dbce4f8800 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 bash[109139]: 4c9c42dc3a0f2bbc7065586f1934d43546e28877716d748120d09a798327bf5c 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 systemd[1]: Started Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 conmon[109180]: 2023-10-09T05:53:05.405+0000 7f99c789a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 conmon[109180]: 2023-10-09T05:53:05.485+0000 7f99c789a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:53:05.577 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 conmon[109180]: 2023-10-09T05:53:05.576+0000 7f99c789a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:53:05.865 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 conmon[109180]: 2023-10-09T05:53:05.672+0000 7f99c789a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:53:05.865 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:05.865 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:05.865 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:05.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:05.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:05.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:06.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:05 smithi150 conmon[109180]: 2023-10-09T05:53:05.863+0000 7f99c789a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:53:06.541 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:06 smithi150 conmon[109180]: 2023-10-09T05:53:06.216+0000 7f99c789a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:53:06.541 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:06 smithi150 conmon[109180]: 2023-10-09T05:53:06.307+0000 7f99c789a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:53:06.541 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:06 smithi150 conmon[109180]: 2023-10-09T05:53:06.446+0000 7f99c789a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:53:06.541 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:06 smithi150 conmon[109180]: 2023-10-09T05:53:06.540+0000 7f99c789a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:53:07.047 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:53:07.047 INFO:teuthology.orchestra.run.smithi107.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 10+0 records in 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi107.stderr:10+0 records out 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi107.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128104 s, 819 MB/s 2023-10-09T05:53:07.048 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping successful for: 2023-10-09T05:53:07.405 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:06 smithi150 conmon[109180]: 2023-10-09T05:53:06.958+0000 7f99c789a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:07 smithi150 conmon[109180]: 2023-10-09T05:53:07.523+0000 7f99c789a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:07.692 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.693 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:07.693 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:07.693 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:07.693 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.693 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:07 smithi150 ceph-mon[105942]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T05:53:07.881 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.881 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.881 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.881 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[101102]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:07.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:07 smithi107 ceph-mon[118900]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T05:53:08.118 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi107:vg_nvme/lv_4 2023-10-09T05:53:08.153 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:07 smithi150 conmon[109180]: 2023-10-09T05:53:07.691+0000 7f99c789a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:53:08.509 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:08.691 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:08 smithi150 conmon[109180]: 2023-10-09T05:53:08.437+0000 7f99c789a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:53:08.692 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:08 smithi150 conmon[109180]: 2023-10-09T05:53:08.526+0000 7f99c789a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:53:08.959 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:08 smithi150 conmon[109180]: 2023-10-09T05:53:08.691+0000 7f99c789a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:53:09.327 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:08 smithi150 conmon[109180]: 2023-10-09T05:53:08.959+0000 7f99c789a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:53:09.327 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:09 smithi150 conmon[109180]: 2023-10-09T05:53:09.044+0000 7f99c789a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:53:09.578 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:09 smithi150 conmon[109180]: 2023-10-09T05:53:09.327+0000 7f99c789a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:09.579 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:09.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:09.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:09.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:09.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:09.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:09.903 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:09 smithi150 conmon[109180]: 2023-10-09T05:53:09.578+0000 7f99c789a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:53:10.594 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:10 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:10.595 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:10 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:10.595 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:10 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:10.595 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:10 smithi150 conmon[109180]: 2023-10-09T05:53:10.280+0000 7f99c789a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:53:10.595 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:10 smithi150 conmon[109180]: 2023-10-09T05:53:10.410+0000 7f99c789a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:53:10.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:10.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:10 smithi150 conmon[109180]: 2023-10-09T05:53:10.594+0000 7f99c789a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:53:10.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:10 smithi150 conmon[109180]: 2023-10-09T05:53:10.707+0000 7f99c789a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:53:11.396 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:10 smithi150 conmon[109180]: 2023-10-09T05:53:10.917+0000 7f99c789a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[101102]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[101102]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[118900]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[118900]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:11 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:11 smithi150 ceph-mon[105942]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:11 smithi150 ceph-mon[105942]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:11 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:12.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:53:11 smithi150 conmon[109180]: 2023-10-09T05:53:11.759+0000 7f99c789a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[101102]: Standby manager daemon x started 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T05:53:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:12 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:12 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T05:53:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:12 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:53:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:12 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:53:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:12 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:53:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:12 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/3335456435' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: mgrmap e14: y(active, since 62s), standbys: x 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/772555792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]': finished 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3665181095' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: mgrmap e14: y(active, since 62s), standbys: x 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/772555792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]': finished 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:13 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3665181095' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: mgrmap e14: y(active, since 62s), standbys: x 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/772555792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]: dispatch 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "61782add-fd90-440a-b489-da0bd6cdb972"}]': finished 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:13 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3665181095' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:15.721 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:15 smithi107 ceph-mon[101102]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:15.721 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:15 smithi107 ceph-mon[118900]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:15 smithi150 ceph-mon[105942]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.414 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:17 smithi107 ceph-mon[101102]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:17 smithi107 ceph-mon[118900]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:17 smithi150 ceph-mon[105942]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[118900]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[101102]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:19 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:19 smithi150 ceph-mon[105942]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:19 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T05:53:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:19 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:20.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:20 smithi107 ceph-mon[101102]: Deploying daemon osd.0 on smithi107 2023-10-09T05:53:20.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:20 smithi107 ceph-mon[118900]: Deploying daemon osd.0 on smithi107 2023-10-09T05:53:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:20 smithi150 ceph-mon[105942]: Deploying daemon osd.0 on smithi107 2023-10-09T05:53:21.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:21 smithi107 ceph-mon[118900]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:21 smithi107 ceph-mon[101102]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:22.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:21 smithi150 ceph-mon[105942]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:24.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:23 smithi107 ceph-mon[101102]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:23 smithi107 ceph-mon[118900]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:23 smithi150 ceph-mon[105942]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[101102]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:24 smithi107 ceph-mon[118900]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:24 smithi150 ceph-mon[105942]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:27.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:27.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:27.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:27.174 INFO:teuthology.orchestra.run.smithi107.stdout:Created osd(s) 0 on host 'smithi107' 2023-10-09T05:53:27.879 DEBUG:teuthology.orchestra.run.smithi107:osd.0> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T05:53:27.882 INFO:tasks.cephadm:Deploying osd.1 on smithi107 with /dev/vg_nvme/lv_3... 2023-10-09T05:53:27.882 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:53:27.913 INFO:journalctl@ceph.osd.0.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:53:28.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:28.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.128 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.129 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.129 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:28.949 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:29.151 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:53:28 smithi107 conmon[125135]: 2023-10-09T05:53:28.813+0000 7eff1b0c9700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T05:53:29.151 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:53:28 smithi107 conmon[125135]: 2023-10-09T05:53:28.815+0000 7eff166c5700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:53:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[101102]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[101102]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[101102]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[101102]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[118900]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[118900]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[118900]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:28 smithi107 ceph-mon[118900]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:29.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:28 smithi150 ceph-mon[105942]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T05:53:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:28 smithi150 ceph-mon[105942]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T05:53:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:28 smithi150 ceph-mon[105942]: from='osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:28 smithi150 ceph-mon[105942]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[101102]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[101102]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[101102]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[118900]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:30.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[118900]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:30.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[118900]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:30.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:29 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:29 smithi150 ceph-mon[105942]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:29 smithi150 ceph-mon[105942]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:29 smithi150 ceph-mon[105942]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T05:53:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:29 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:29 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:53:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:53:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648] boot 2023-10-09T05:53:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648] boot 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:30 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: osd.0 [v2:172.21.15.107:6802/2908141648,v1:172.21.15.107:6803/2908141648] boot 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:30 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:53:31.404 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:53:31.404 INFO:teuthology.orchestra.run.smithi107.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:53:31.405 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 10+0 records in 2023-10-09T05:53:31.405 INFO:teuthology.orchestra.run.smithi107.stderr:10+0 records out 2023-10-09T05:53:31.405 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135099 s, 776 MB/s 2023-10-09T05:53:31.405 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping successful for: 2023-10-09T05:53:31.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:31 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:32.048 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi107:vg_nvme/lv_3 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:31 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:32.382 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:32 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi107 2023-10-09T05:53:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:32 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi107 to 16144M 2023-10-09T05:53:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:32 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi107 2023-10-09T05:53:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:32 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi107 to 16144M 2023-10-09T05:53:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:32 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi107 2023-10-09T05:53:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:32 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi107 to 16144M 2023-10-09T05:53:34.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:33 smithi107 ceph-mon[101102]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:34.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:33 smithi107 ceph-mon[118900]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:33 smithi150 ceph-mon[105942]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:35.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:35.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:35.134 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:35.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:35.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:35.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:34 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:34 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:34 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:34 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:35.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:35 smithi107 ceph-mon[101102]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:35.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:35 smithi107 ceph-mon[101102]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:35.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:35 smithi107 ceph-mon[118900]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:35.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:35 smithi107 ceph-mon[118900]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:35 smithi150 ceph-mon[105942]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:35 smithi150 ceph-mon[105942]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]: dispatch 2023-10-09T05:53:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]': finished 2023-10-09T05:53:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[101102]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:53:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]: dispatch 2023-10-09T05:53:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]': finished 2023-10-09T05:53:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[118900]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:53:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:36 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:37.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:36 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]: dispatch 2023-10-09T05:53:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:36 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3930073478' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c947d5-3ca8-468c-9f6d-2859ed2275b2"}]': finished 2023-10-09T05:53:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:36 smithi150 ceph-mon[105942]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T05:53:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:36 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:37 smithi107 ceph-mon[101102]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:37 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3268582101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:38.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:37 smithi107 ceph-mon[118900]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:38.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:37 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3268582101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:37 smithi150 ceph-mon[105942]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:37 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3268582101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:53:40.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:39 smithi107 ceph-mon[101102]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:40.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:39 smithi107 ceph-mon[118900]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:40.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:39 smithi150 ceph-mon[105942]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:41 smithi107 ceph-mon[101102]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:42.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:41 smithi107 ceph-mon[118900]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:41 smithi150 ceph-mon[105942]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:44.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[101102]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:44.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:53:44.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:44.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[118900]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:44.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:53:44.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:43 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:44.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:43 smithi150 ceph-mon[105942]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:44.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:43 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T05:53:44.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:43 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:45.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:44 smithi107 ceph-mon[101102]: Deploying daemon osd.1 on smithi107 2023-10-09T05:53:45.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:44 smithi107 ceph-mon[118900]: Deploying daemon osd.1 on smithi107 2023-10-09T05:53:45.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:44 smithi150 ceph-mon[105942]: Deploying daemon osd.1 on smithi107 2023-10-09T05:53:46.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:45 smithi107 ceph-mon[101102]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:46.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:45 smithi107 ceph-mon[118900]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:46.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:45 smithi150 ceph-mon[105942]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:48.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:47 smithi107 ceph-mon[101102]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:48.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:47 smithi107 ceph-mon[118900]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:48.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:47 smithi150 ceph-mon[105942]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[101102]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:49.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:48 smithi107 ceph-mon[118900]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:49.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:48 smithi150 ceph-mon[105942]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:50.878 INFO:teuthology.orchestra.run.smithi107.stdout:Created osd(s) 1 on host 'smithi107' 2023-10-09T05:53:51.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.096 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:53:51.615 DEBUG:teuthology.orchestra.run.smithi107:osd.1> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T05:53:51.617 INFO:tasks.cephadm:Deploying osd.2 on smithi107 with /dev/vg_nvme/lv_2... 2023-10-09T05:53:51.617 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:53:51.648 INFO:journalctl@ceph.osd.1.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:53:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:52 smithi107 ceph-mon[101102]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:53:52.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:52 smithi107 ceph-mon[118900]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:53:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:52 smithi150 ceph-mon[105942]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T05:53:52.841 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:53.150 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:53:53 smithi107 conmon[134831]: 2023-10-09T05:53:53.013+0000 7f946ba7d700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T05:53:53.150 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:53:53 smithi107 conmon[134831]: 2023-10-09T05:53:53.017+0000 7f9466878700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:53:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[101102]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:53:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[101102]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:53:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[101102]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[101102]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[118900]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[118900]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[118900]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:53.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:53 smithi107 ceph-mon[118900]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:53 smithi150 ceph-mon[105942]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T05:53:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:53 smithi150 ceph-mon[105942]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T05:53:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:53 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:53 smithi150 ceph-mon[105942]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:53:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:53 smithi150 ceph-mon[105942]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[101102]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[101102]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[118900]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:54.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[118900]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:53:54.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:54 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:54 smithi150 ceph-mon[105942]: from='osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:53:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:54 smithi150 ceph-mon[105942]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T05:53:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:54 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:54 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:54.897 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:53:54.897 INFO:teuthology.orchestra.run.smithi107.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:53:54.897 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 10+0 records in 2023-10-09T05:53:54.897 INFO:teuthology.orchestra.run.smithi107.stderr:10+0 records out 2023-10-09T05:53:54.897 INFO:teuthology.orchestra.run.smithi107.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0143976 s, 728 MB/s 2023-10-09T05:53:54.898 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping successful for: 2023-10-09T05:53:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:53:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:53:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147] boot 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[101102]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147] boot 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:55 smithi107 ceph-mon[118900]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: osd.1 [v2:172.21.15.107:6810/420392147,v1:172.21.15.107:6811/420392147] boot 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:53:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:55 smithi150 ceph-mon[105942]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T05:53:55.974 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi107:vg_nvme/lv_2 2023-10-09T05:53:56.333 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:53:56.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:56 smithi107 ceph-mon[101102]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:53:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:56 smithi107 ceph-mon[118900]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:53:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:56 smithi150 ceph-mon[105942]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi107 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi107 to 8072M 2023-10-09T05:53:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[101102]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi107 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi107 to 8072M 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:57 smithi107 ceph-mon[118900]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi107 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi107 to 8072M 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:53:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:57 smithi150 ceph-mon[105942]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:58 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:53:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:53:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:58 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:53:59.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:59 smithi107 ceph-mon[101102]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:59.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:53:59 smithi107 ceph-mon[101102]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:53:59.961 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:59 smithi107 ceph-mon[118900]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:53:59.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:53:59 smithi107 ceph-mon[118900]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:59 smithi150 ceph-mon[105942]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:53:59 smithi150 ceph-mon[105942]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/4102002393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]': finished 2023-10-09T05:54:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[101102]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/4102002393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]': finished 2023-10-09T05:54:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[118900]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:00 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:00 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/4102002393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:00 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]: dispatch 2023-10-09T05:54:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:00 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ba0a04dd-49a5-498f-a016-aff57d88ff33"}]': finished 2023-10-09T05:54:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:00 smithi150 ceph-mon[105942]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T05:54:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:00 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:01 smithi107 ceph-mon[101102]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:01 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3072086089' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:01 smithi107 ceph-mon[118900]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:01 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3072086089' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:02.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:01 smithi150 ceph-mon[105942]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:01 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3072086089' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:03.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:03 smithi107 ceph-mon[101102]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:03.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:03 smithi107 ceph-mon[118900]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:03 smithi150 ceph-mon[105942]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:06.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:05 smithi107 ceph-mon[101102]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:06.034 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:05 smithi107 ceph-mon[118900]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:05 smithi150 ceph-mon[105942]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:08.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[101102]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:08.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:54:08.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:08.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[118900]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:08.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:54:08.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:07 smithi150 ceph-mon[105942]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T05:54:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:08 smithi107 ceph-mon[101102]: Deploying daemon osd.2 on smithi107 2023-10-09T05:54:09.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:08 smithi107 ceph-mon[118900]: Deploying daemon osd.2 on smithi107 2023-10-09T05:54:09.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:08 smithi150 ceph-mon[105942]: Deploying daemon osd.2 on smithi107 2023-10-09T05:54:10.085 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:09 smithi107 ceph-mon[101102]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:10.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:09 smithi107 ceph-mon[118900]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:09 smithi150 ceph-mon[105942]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:12.097 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:11 smithi107 ceph-mon[101102]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:12.098 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:11 smithi107 ceph-mon[118900]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:12.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:11 smithi150 ceph-mon[105942]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[101102]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:13 smithi107 ceph-mon[118900]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:13 smithi150 ceph-mon[105942]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:15.212 INFO:teuthology.orchestra.run.smithi107.stdout:Created osd(s) 2 on host 'smithi107' 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:15 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:15 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:15.896 DEBUG:teuthology.orchestra.run.smithi107:osd.2> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T05:54:15.901 INFO:tasks.cephadm:Deploying osd.3 on smithi107 with /dev/vg_nvme/lv_1... 2023-10-09T05:54:15.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:54:15.928 INFO:journalctl@ceph.osd.2.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:54:16.616 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:16 smithi107 ceph-mon[101102]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:16.617 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:16 smithi107 ceph-mon[101102]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:16.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:16 smithi107 ceph-mon[118900]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:16.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:16 smithi107 ceph-mon[118900]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:16 smithi150 ceph-mon[105942]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:16 smithi150 ceph-mon[105942]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T05:54:17.086 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:17.401 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:17 smithi107 conmon[143874]: 2023-10-09T05:54:17.236+0000 7fc632b41700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T05:54:17.780 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:17 smithi107 conmon[143874]: 2023-10-09T05:54:17.240+0000 7fc62e13d700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:54:17.780 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:54:17.780 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[118900]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:17 smithi107 ceph-mon[101102]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: from='osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:17.782 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:17 smithi150 ceph-mon[105942]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:18.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[118900]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:18.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[118900]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:54:18.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:18.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:18.589 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[101102]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:18.589 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[101102]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:54:18.589 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:18.589 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:18 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:18.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:18 smithi150 ceph-mon[105942]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:18 smithi150 ceph-mon[105942]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T05:54:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:18 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:18 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.164 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:54:19.164 INFO:teuthology.orchestra.run.smithi107.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:54:19.164 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 10+0 records in 2023-10-09T05:54:19.165 INFO:teuthology.orchestra.run.smithi107.stderr:10+0 records out 2023-10-09T05:54:19.165 INFO:teuthology.orchestra.run.smithi107.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124795 s, 840 MB/s 2023-10-09T05:54:19.165 INFO:teuthology.orchestra.run.smithi107.stderr: stderr: 2023-10-09T05:54:19.165 INFO:teuthology.orchestra.run.smithi107.stderr:--> Zapping successful for: 2023-10-09T05:54:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:54:19.526 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.527 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815] boot 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815] boot 2023-10-09T05:54:19.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:54:19.529 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.529 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:19.529 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:19 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: osd.2 [v2:172.21.15.107:6818/1709812815,v1:172.21.15.107:6819/1709812815] boot 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T05:54:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:19 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T05:54:20.284 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi107:vg_nvme/lv_1 2023-10-09T05:54:20.627 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:54:20.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:54:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[101102]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:54:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:54:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:54:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[118900]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:54:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:20 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:54:20.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:20 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T05:54:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:20 smithi150 ceph-mon[105942]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T05:54:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:20 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:54:21.381 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149733]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:21.381 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149733]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:21.381 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149733]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:21.381 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149733]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:21.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:54:21.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:54:21.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi107 2023-10-09T05:54:21.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi107 to 5381M 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[101102]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi107 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi107 to 5381M 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:21.384 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:21.384 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.384 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:21 smithi107 ceph-mon[118900]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:21.643 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149769]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:21.643 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149769]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:21.643 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149769]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:21.643 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149769]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi107 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi107 to 5381M 2023-10-09T05:54:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:21.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:21.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:21.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:21 smithi150 ceph-mon[105942]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:21.900 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149772]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:21.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149772]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:21.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149772]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:21.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149772]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:22.159 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149830]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:22.160 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149830]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:22.160 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149830]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:22.160 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 05:54:21 smithi107 sudo[149830]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:22.160 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149853]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:22.423 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[101102]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:54:22.424 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[101102]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:54:22.424 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:22.424 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149853]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:22.424 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149853]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:22.424 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149853]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:22.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[118900]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:54:22.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[118900]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:54:22.425 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:22 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:22 smithi150 ceph-mon[105942]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T05:54:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:22 smithi150 ceph-mon[105942]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T05:54:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:22 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:22.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149889]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:22.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149889]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:22.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149889]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:22.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149889]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:22.684 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149885]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:22.685 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149885]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:22.685 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149885]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:22.685 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149885]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:23.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149892]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149892]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149892]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:22 smithi107 sudo[149892]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:23.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110080]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:23.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110080]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:23.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110080]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:23.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110080]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:23.698 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:23.698 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:23.699 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.700 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.701 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.701 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:23.701 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110083]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110083]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110083]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 sudo[110083]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi107:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:23.966 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150140]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T05:54:23.967 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150140]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:23.967 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150140]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:23.967 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150140]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:24.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150195]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T05:54:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150195]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T05:54:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150195]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T05:54:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:23 smithi107 sudo[150195]: pam_unix(sudo:session): session closed for user root 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:24.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[101102]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:24.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:24 smithi107 ceph-mon[118900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:54:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:24 smithi150 ceph-mon[105942]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T05:54:25.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:25 smithi107 ceph-mon[101102]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:25.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:25 smithi107 ceph-mon[118900]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:25.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:25 smithi150 ceph-mon[105942]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:26.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2783410324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]': finished 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1640083924' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:26.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2783410324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]': finished 2023-10-09T05:54:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:54:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:26 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1640083924' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2783410324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]: dispatch 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ca913f-7262-4d0a-8f15-07e70fe94239"}]': finished 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:26 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1640083924' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:27.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:27 smithi107 ceph-mon[101102]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:27 smithi107 ceph-mon[118900]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:27.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:27 smithi150 ceph-mon[105942]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:29.243 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:28 smithi107 ceph-mon[118900]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:29.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:28 smithi107 ceph-mon[101102]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:28 smithi150 ceph-mon[105942]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:31 smithi107 ceph-mon[101102]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:31 smithi107 ceph-mon[118900]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:31 smithi150 ceph-mon[105942]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:32.862 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:54:32.862 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:32.862 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[101102]: Deploying daemon osd.3 on smithi107 2023-10-09T05:54:32.862 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:54:32.863 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:32.863 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:32 smithi107 ceph-mon[118900]: Deploying daemon osd.3 on smithi107 2023-10-09T05:54:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:32 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T05:54:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:32 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:32 smithi150 ceph-mon[105942]: Deploying daemon osd.3 on smithi107 2023-10-09T05:54:33.613 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:33 smithi107 ceph-mon[101102]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:33.613 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:33 smithi107 ceph-mon[118900]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:33 smithi150 ceph-mon[105942]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:35 smithi107 ceph-mon[101102]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:35.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:35 smithi107 ceph-mon[118900]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:35 smithi150 ceph-mon[105942]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:37.694 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[101102]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:37.694 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:37.694 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:37.695 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:37.695 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[118900]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:37.695 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:37.695 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:37.695 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:37 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:37.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:37 smithi150 ceph-mon[105942]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:37 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:37 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:37 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:39.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:38 smithi107 ceph-mon[101102]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:39.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:38 smithi107 ceph-mon[118900]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:38 smithi150 ceph-mon[105942]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:40.277 INFO:teuthology.orchestra.run.smithi107.stdout:Created osd(s) 3 on host 'smithi107' 2023-10-09T05:54:40.460 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.460 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.460 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:40.461 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:40 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:40 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:40 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:40 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:40 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:40 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:40.973 DEBUG:teuthology.orchestra.run.smithi107:osd.3> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T05:54:40.975 INFO:tasks.cephadm:Deploying osd.4 on smithi150 with /dev/vg_nvme/lv_4... 2023-10-09T05:54:40.976 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T05:54:41.004 INFO:journalctl@ceph.osd.3.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:41.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[101102]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:41.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:41 smithi107 ceph-mon[118900]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:41.322 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:54:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:41 smithi150 ceph-mon[105942]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:42.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[101102]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:54:42.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[101102]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:54:42.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:42.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[101102]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[101102]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[118900]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:54:42.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[118900]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:54:42.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:42.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[118900]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:42 smithi107 ceph-mon[118900]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.316 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 05:54:42 smithi107 conmon[152901]: 2023-10-09T05:54:42.194+0000 7f0fc8170700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T05:54:42.316 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 05:54:42 smithi107 conmon[152901]: 2023-10-09T05:54:42.198+0000 7f0fc0f67700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:54:42.467 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:42 smithi150 ceph-mon[105942]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T05:54:42.467 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:42 smithi150 ceph-mon[105942]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T05:54:42.467 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:42 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:42.467 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:42 smithi150 ceph-mon[105942]: from='osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.468 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:42 smithi150 ceph-mon[105942]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T05:54:42.949 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T05:54:42.949 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T05:54:42.949 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2023-10-09T05:54:42.949 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2023-10-09T05:54:42.950 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0145459 s, 721 MB/s 2023-10-09T05:54:42.950 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2023-10-09T05:54:43.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[101102]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[101102]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[101102]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[118900]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[118900]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:43 smithi107 ceph-mon[118900]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:43 smithi150 ceph-mon[105942]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]': finished 2023-10-09T05:54:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:43 smithi150 ceph-mon[105942]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T05:54:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:43 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:43 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:43 smithi150 ceph-mon[105942]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T05:54:43.513 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi150:vg_nvme/lv_4 2023-10-09T05:54:43.847 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:54:44.331 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:54:44.331 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415] boot 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:44.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:44.333 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415] boot 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: osd.3 [v2:172.21.15.107:6826/4077493415,v1:172.21.15.107:6827/4077493415] boot 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:54:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi107 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi107 to 4036M 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[101102]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[101102]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi107 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi107 to 4036M 2023-10-09T05:54:45.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[118900]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:45.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:45 smithi107 ceph-mon[118900]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:54:45.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:45 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi107 2023-10-09T05:54:45.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:45 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi107 to 4036M 2023-10-09T05:54:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:45 smithi150 ceph-mon[105942]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:45 smithi150 ceph-mon[105942]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T05:54:46.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[101102]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:46.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:46.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:46.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:46.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[118900]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:46.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:46.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:46.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:46 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:46 smithi150 ceph-mon[105942]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:54:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:46 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:54:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:46 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:54:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:46 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:47 smithi150 ceph-mon[105942]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:47 smithi107 ceph-mon[101102]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:47 smithi107 ceph-mon[118900]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/641395471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]': finished 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[101102]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/641395471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]': finished 2023-10-09T05:54:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[118900]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:54:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:48 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:54:48.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:48 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/641395471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:48 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]: dispatch 2023-10-09T05:54:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:48 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c43fad0c-5200-44d6-b725-36b66a11482e"}]': finished 2023-10-09T05:54:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:48 smithi150 ceph-mon[105942]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T05:54:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:54:49.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:49 smithi107 ceph-mon[118900]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:49.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/3859519787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:49 smithi107 ceph-mon[101102]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/3859519787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:49.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:49 smithi150 ceph-mon[105942]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/3859519787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:54:51.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:51 smithi107 ceph-mon[101102]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:51 smithi107 ceph-mon[118900]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:51 smithi150 ceph-mon[105942]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:53.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:53 smithi107 ceph-mon[101102]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:53 smithi107 ceph-mon[118900]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:53 smithi150 ceph-mon[105942]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:54.675 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:54 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:54:54.675 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:54 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:54.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:54 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:54:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:54 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:54.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:54 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T05:54:54.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:54 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:54:55.731 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:55 smithi150 ceph-mon[105942]: Deploying daemon osd.4 on smithi150 2023-10-09T05:54:55.731 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:55 smithi150 ceph-mon[105942]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:55.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:55 smithi107 ceph-mon[101102]: Deploying daemon osd.4 on smithi150 2023-10-09T05:54:55.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:55 smithi107 ceph-mon[101102]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:55.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:55 smithi107 ceph-mon[118900]: Deploying daemon osd.4 on smithi150 2023-10-09T05:54:55.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:55 smithi107 ceph-mon[118900]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:57 smithi150 ceph-mon[105942]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:57.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:57 smithi107 ceph-mon[101102]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:57 smithi107 ceph-mon[118900]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:59.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:54:58 smithi107 ceph-mon[101102]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:59.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:54:58 smithi107 ceph-mon[118900]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:54:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:54:58 smithi150 ceph-mon[105942]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:00 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:00 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:00 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:00.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:00 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:02.017 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 4 on host 'smithi150' 2023-10-09T05:55:02.043 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:02.044 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.044 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.044 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:02.044 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:02.044 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:02.559 DEBUG:teuthology.orchestra.run.smithi150:osd.4> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T05:55:02.561 INFO:tasks.cephadm:Deploying osd.5 on smithi150 with /dev/vg_nvme/lv_3... 2023-10-09T05:55:02.561 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T05:55:02.592 INFO:journalctl@ceph.osd.4.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:02 smithi150 ceph-mon[105942]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:03.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[101102]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:55:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:02 smithi107 ceph-mon[118900]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:03.592 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:04 smithi150 ceph-mon[105942]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:55:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:04 smithi150 ceph-mon[105942]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:55:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:04 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:04 smithi150 ceph-mon[105942]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:04 smithi150 ceph-mon[105942]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:04.155 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 05:55:04 smithi150 conmon[114006]: 2023-10-09T05:55:04.004+0000 7f0408c99700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T05:55:04.156 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 05:55:04 smithi150 conmon[114006]: 2023-10-09T05:55:04.008+0000 7f0402291700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[101102]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[101102]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[101102]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[101102]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[118900]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[118900]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T05:55:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[118900]: from='osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:04 smithi107 ceph-mon[118900]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:05.306 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864] boot 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:55:05.307 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:05 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864] boot 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:55:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: osd.4 [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864] boot 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T05:55:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:05 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110402 s, 950 MB/s 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 2023-10-09T05:55:05.884 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2023-10-09T05:55:06.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi150 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 17168M 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[101102]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:55:06.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi150 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 17168M 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:06 smithi107 ceph-mon[118900]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi150 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 17168M 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:06.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:06.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:06.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:06 smithi150 ceph-mon[105942]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T05:55:06.462 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi150:vg_nvme/lv_3 2023-10-09T05:55:06.798 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:07.328 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:07 smithi150 ceph-mon[105942]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:07.328 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:07 smithi150 ceph-mon[105942]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:55:07.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:07 smithi107 ceph-mon[101102]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:07 smithi107 ceph-mon[101102]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:55:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:07 smithi107 ceph-mon[118900]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:07 smithi107 ceph-mon[118900]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:08 smithi150 ceph-mon[105942]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:55:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:09.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:09.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[101102]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T05:55:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:09.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:09.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:09.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:09.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:08 smithi107 ceph-mon[118900]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:09 smithi150 ceph-mon[105942]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:55:10.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:09 smithi107 ceph-mon[101102]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:55:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:09 smithi107 ceph-mon[118900]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T05:55:11.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 132 KiB/s, 0 objects/s recovering 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/757430319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]': finished 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 132 KiB/s, 0 objects/s recovering 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/757430319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]': finished 2023-10-09T05:55:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:55:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:10 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 132 KiB/s, 0 objects/s recovering 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/757430319' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]: dispatch 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98904b1f-c066-4710-974a-288f9323551d"}]': finished 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T05:55:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:10 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:11 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/1475478404' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:12.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:11 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/1475478404' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:12.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:11 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/1475478404' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:13.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:12 smithi107 ceph-mon[101102]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T05:55:13.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:12 smithi107 ceph-mon[118900]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T05:55:13.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:12 smithi150 ceph-mon[105942]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 107 KiB/s, 0 objects/s recovering 2023-10-09T05:55:15.678 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:15 smithi150 ceph-mon[105942]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:55:15.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:15 smithi107 ceph-mon[101102]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:55:15.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:15 smithi107 ceph-mon[118900]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:55:17.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:17 smithi150 ceph-mon[105942]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:55:17.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:17 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:55:17.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:17 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:17.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:17 smithi150 ceph-mon[105942]: Deploying daemon osd.5 on smithi150 2023-10-09T05:55:17.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[101102]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[101102]: Deploying daemon osd.5 on smithi150 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[118900]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:17 smithi107 ceph-mon[118900]: Deploying daemon osd.5 on smithi150 2023-10-09T05:55:19.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:18 smithi150 ceph-mon[105942]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:55:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:18 smithi107 ceph-mon[101102]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:55:19.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:18 smithi107 ceph-mon[118900]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:55:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:21 smithi150 ceph-mon[105942]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:21 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:21.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[101102]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[118900]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:21 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:22 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:22.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:22 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:22.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:22 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:23 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:23.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:23.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:23.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:23 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:24.254 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 5 on host 'smithi150' 2023-10-09T05:55:24.790 DEBUG:teuthology.orchestra.run.smithi150:osd.5> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T05:55:24.795 INFO:tasks.cephadm:Deploying osd.6 on smithi150 with /dev/vg_nvme/lv_2... 2023-10-09T05:55:24.796 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T05:55:24.826 INFO:journalctl@ceph.osd.5.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:55:25.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:25.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:25 smithi150 ceph-mon[105942]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[101102]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T05:55:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:25 smithi107 ceph-mon[118900]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:25.706 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[101102]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:55:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[101102]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:55:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[101102]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[101102]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[118900]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:55:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[118900]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:55:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[118900]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:26 smithi107 ceph-mon[118900]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:26 smithi150 ceph-mon[105942]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T05:55:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:26 smithi150 ceph-mon[105942]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T05:55:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:26 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:26 smithi150 ceph-mon[105942]: from='osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:26 smithi150 ceph-mon[105942]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:26.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 05:55:26 smithi150 conmon[122886]: 2023-10-09T05:55:26.254+0000 7f72e2bff700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T05:55:26.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 05:55:26 smithi150 conmon[122886]: 2023-10-09T05:55:26.259+0000 7f72df9fe700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:55:27.318 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:27 smithi150 ceph-mon[105942]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:27.318 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:27 smithi150 ceph-mon[105942]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:55:27.318 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.318 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.319 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:27 smithi150 ceph-mon[105942]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:27.588 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T05:55:27.589 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T05:55:27.589 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2023-10-09T05:55:27.589 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2023-10-09T05:55:27.589 INFO:teuthology.orchestra.run.smithi150.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127941 s, 820 MB/s 2023-10-09T05:55:27.589 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[101102]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[101102]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[101102]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[118900]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[118900]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T05:55:27.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:27.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:27 smithi107 ceph-mon[118900]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T05:55:28.182 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi150:vg_nvme/lv_2 2023-10-09T05:55:28.519 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682] boot 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682] boot 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:28.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:28 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: osd.5 [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682] boot 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:28 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:29 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi150 2023-10-09T05:55:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:29 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 8584M 2023-10-09T05:55:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:29 smithi150 ceph-mon[105942]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:29 smithi150 ceph-mon[105942]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi150 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 8584M 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[101102]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[101102]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi150 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 8584M 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[118900]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:29.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:29 smithi107 ceph-mon[118900]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T05:55:31.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[101102]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:55:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:30 smithi107 ceph-mon[118900]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:30 smithi150 ceph-mon[105942]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:32.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:31 smithi107 ceph-mon[101102]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:55:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:31 smithi107 ceph-mon[118900]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:55:32.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:31 smithi150 ceph-mon[105942]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/1367159525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]': finished 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:33.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:33.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/1367159525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]': finished 2023-10-09T05:55:33.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:55:33.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:32 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/1367159525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]: dispatch 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d40d76a2-9d6d-41d9-91ed-dfa6f8228130"}]': finished 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T05:55:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:32 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:34.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:33 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/1529608990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:33 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/1529608990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:34.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:33 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/1529608990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:35.364 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:34 smithi150 ceph-mon[105942]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:35.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:34 smithi107 ceph-mon[101102]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:34 smithi107 ceph-mon[118900]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:37.575 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:37 smithi150 ceph-mon[105942]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:37.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:37 smithi107 ceph-mon[101102]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:37 smithi107 ceph-mon[118900]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:39.207 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:38 smithi150 ceph-mon[105942]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:39.207 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:55:39.207 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:39.208 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:38 smithi150 ceph-mon[105942]: Deploying daemon osd.6 on smithi150 2023-10-09T05:55:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[101102]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:55:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:39.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[101102]: Deploying daemon osd.6 on smithi150 2023-10-09T05:55:39.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[118900]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:39.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T05:55:39.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:39.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:38 smithi107 ceph-mon[118900]: Deploying daemon osd.6 on smithi150 2023-10-09T05:55:41.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:41 smithi150 ceph-mon[105942]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:41.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:41 smithi107 ceph-mon[101102]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:41.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:41 smithi107 ceph-mon[118900]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:43.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:43 smithi107 ceph-mon[101102]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:43.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:43 smithi107 ceph-mon[118900]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:43.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:43 smithi150 ceph-mon[105942]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:44.799 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.799 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.799 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:44 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:44.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:44 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:45.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:45.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:45.870 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:45.870 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:45 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:45.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:45 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:46.430 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 6 on host 'smithi150' 2023-10-09T05:55:47.061 DEBUG:teuthology.orchestra.run.smithi150:osd.6> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T05:55:47.063 INFO:tasks.cephadm:Deploying osd.7 on smithi150 with /dev/vg_nvme/lv_1... 2023-10-09T05:55:47.064 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T05:55:47.093 INFO:journalctl@ceph.osd.6.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:55:47.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[101102]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:47 smithi107 ceph-mon[118900]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:47 smithi150 ceph-mon[105942]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T05:55:48.153 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[101102]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[101102]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[101102]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[101102]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[118900]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:55:48.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[118900]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:55:48.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:48.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[118900]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:48 smithi107 ceph-mon[118900]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:48 smithi150 ceph-mon[105942]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T05:55:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:48 smithi150 ceph-mon[105942]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T05:55:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:48 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:48 smithi150 ceph-mon[105942]: from='osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:48 smithi150 ceph-mon[105942]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:55:48.905 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 05:55:48 smithi150 conmon[132350]: 2023-10-09T05:55:48.627+0000 7f64ebfab700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T05:55:48.905 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 05:55:48 smithi150 conmon[132350]: 2023-10-09T05:55:48.634+0000 7f64e7da8700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[101102]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[101102]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[101102]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[118900]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[118900]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[118900]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:55:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:49 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:49 smithi150 ceph-mon[105942]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:49 smithi150 ceph-mon[105942]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:55:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:49 smithi150 ceph-mon[105942]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T05:55:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:49 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:49 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0109081 s, 961 MB/s 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 2023-10-09T05:55:50.243 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2023-10-09T05:55:50.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495] boot 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495] boot 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:50.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:50 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: osd.6 [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495] boot 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:55:50.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:50.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:55:50.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:50 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:55:50.972 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi150:vg_nvme/lv_1 2023-10-09T05:55:51.310 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:55:51.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:51 smithi150 ceph-mon[105942]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:51.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:51 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi150 2023-10-09T05:55:51.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:51 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 5722M 2023-10-09T05:55:51.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:51 smithi150 ceph-mon[105942]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:55:51.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[101102]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:51.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi150 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 5722M 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[101102]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[118900]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi150 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 5722M 2023-10-09T05:55:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:51 smithi107 ceph-mon[118900]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T05:55:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:52 smithi150 ceph-mon[105942]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:55:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:52 smithi107 ceph-mon[101102]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:55:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:52 smithi107 ceph-mon[118900]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T05:55:53.855 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:53 smithi150 ceph-mon[105942]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:53.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:53 smithi150 ceph-mon[105942]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:55:53.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:53 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:53.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:53 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:53.856 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:53 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[101102]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[101102]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[118900]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[118900]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T05:55:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T05:55:54.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T05:55:54.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:53 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:55:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:54 smithi150 ceph-mon[105942]: from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:54.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:54 smithi150 ceph-mon[105942]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:55:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:54 smithi107 ceph-mon[101102]: from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:54 smithi107 ceph-mon[101102]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:55:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:54 smithi107 ceph-mon[118900]: from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:55:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:54 smithi107 ceph-mon[118900]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T05:55:56.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:56.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/1160826016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]': finished 2023-10-09T05:55:56.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/1160826016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]': finished 2023-10-09T05:55:56.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:55:56.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:55 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:55:56.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:55:56.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/1160826016' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]: dispatch 2023-10-09T05:55:56.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4772509b-9d82-4565-86d2-557b830afc74"}]': finished 2023-10-09T05:55:56.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T05:55:56.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:55 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:55:57.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:56 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/2085281466' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:57.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:56 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/2085281466' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:56 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/2085281466' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T05:55:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:57 smithi107 ceph-mon[101102]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 122 KiB/s, 0 objects/s recovering 2023-10-09T05:55:58.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:57 smithi107 ceph-mon[118900]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 122 KiB/s, 0 objects/s recovering 2023-10-09T05:55:58.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:57 smithi150 ceph-mon[105942]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 122 KiB/s, 0 objects/s recovering 2023-10-09T05:55:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:55:58 smithi150 ceph-mon[105942]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:55:59.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:55:58 smithi107 ceph-mon[101102]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:55:59.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:55:58 smithi107 ceph-mon[118900]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T05:56:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:01 smithi150 ceph-mon[105942]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-10-09T05:56:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:56:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:01 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:01.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[101102]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-10-09T05:56:01.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:56:01.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:01.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[118900]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-10-09T05:56:01.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T05:56:01.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:01 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:02 smithi150 ceph-mon[105942]: Deploying daemon osd.7 on smithi150 2023-10-09T05:56:02.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:02 smithi107 ceph-mon[118900]: Deploying daemon osd.7 on smithi150 2023-10-09T05:56:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:02 smithi107 ceph-mon[101102]: Deploying daemon osd.7 on smithi150 2023-10-09T05:56:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:03 smithi150 ceph-mon[105942]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T05:56:03.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:03 smithi107 ceph-mon[101102]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T05:56:03.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:03 smithi107 ceph-mon[118900]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T05:56:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:05 smithi107 ceph-mon[101102]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:56:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:05 smithi107 ceph-mon[118900]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:56:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:05 smithi150 ceph-mon[105942]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[101102]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:07.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:07 smithi107 ceph-mon[118900]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:07 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:07 smithi150 ceph-mon[105942]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-10-09T05:56:09.251 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 7 on host 'smithi150' 2023-10-09T05:56:09.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:09.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.822 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.823 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:09 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:09.848 DEBUG:teuthology.orchestra.run.smithi150:osd.7> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T05:56:09.849 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T05:56:09.850 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:56:09.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:09.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.883 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:09.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:09 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:09.884 INFO:journalctl@ceph.osd.7.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:56:10.202 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:10.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:10 smithi107 ceph-mon[101102]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:10 smithi107 ceph-mon[101102]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:10 smithi107 ceph-mon[118900]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:10 smithi107 ceph-mon[118900]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:10.894 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:10 smithi150 ceph-mon[105942]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:10.895 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:10 smithi150 ceph-mon[105942]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[101102]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:11.567 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:56:11.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:56:11.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:11.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:11 smithi107 ceph-mon[118900]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T05:56:11.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T05:56:11.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:11.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: from='osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:11 smithi150 ceph-mon[105942]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2023-10-09T05:56:11.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 05:56:11 smithi150 conmon[141446]: 2023-10-09T05:56:11.463+0000 7fc7d8a94700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T05:56:11.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 05:56:11 smithi150 conmon[141446]: 2023-10-09T05:56:11.471+0000 7fc7d208c700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:56:11.949 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:56:12.687 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1696830949,"num_in_osds":8,"osd_in_since":1696830955,"num_remapped_pgs":0} 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: purged_snaps scrub starts 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: purged_snaps scrub ok 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1017976542' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784] boot 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:56:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: purged_snaps scrub starts 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: purged_snaps scrub ok 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1017976542' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784] boot 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:56:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:12 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: purged_snaps scrub starts 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: purged_snaps scrub ok 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1017976542' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: osd.7 [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784] boot 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T05:56:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:12 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:13.689 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: Detected new or changed devices on smithi150 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 4292M 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[101102]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: Detected new or changed devices on smithi150 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 4292M 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:13.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:13 smithi107 ceph-mon[118900]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:56:13.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T05:56:13.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: Detected new or changed devices on smithi150 2023-10-09T05:56:13.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 4292M 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:13.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:13.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:13.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:13 smithi150 ceph-mon[105942]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T05:56:14.039 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:15.342 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:14 smithi107 ceph-mon[101102]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:56:15.342 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:14 smithi107 ceph-mon[101102]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:15.342 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:14 smithi107 ceph-mon[118900]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:56:15.342 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:14 smithi107 ceph-mon[118900]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:14 smithi150 ceph-mon[105942]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T05:56:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:14 smithi150 ceph-mon[105942]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:15.793 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:56:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:15 smithi107 ceph-mon[101102]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:56:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:15 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3684439268' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:15 smithi107 ceph-mon[118900]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:56:16.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:15 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3684439268' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:15 smithi150 ceph-mon[105942]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T05:56:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:15 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3684439268' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T05:56:16.572 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1696830972,"num_in_osds":8,"osd_in_since":1696830955,"num_remapped_pgs":0} 2023-10-09T05:56:16.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:56:16.910 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:17.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:16 smithi107 ceph-mon[101102]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:56:17.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:16 smithi107 ceph-mon[101102]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:17.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:16 smithi107 ceph-mon[118900]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:56:17.330 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:16 smithi107 ceph-mon[118900]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:16 smithi150 ceph-mon[105942]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T05:56:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:16 smithi150 ceph-mon[105942]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:18.657 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:56:18.657 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":58,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:56:15.957318+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:56:19.347 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:18 smithi107 ceph-mon[118900]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s, 0 objects/s recovering 2023-10-09T05:56:19.347 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:18 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2663799614' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:19.347 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:18 smithi107 ceph-mon[101102]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s, 0 objects/s recovering 2023-10-09T05:56:19.347 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:18 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2663799614' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:19.362 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T05:54:18.659893+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T05:56:19.362 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T05:56:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:18 smithi150 ceph-mon[105942]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s, 0 objects/s recovering 2023-10-09T05:56:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:18 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2663799614' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:19.698 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:21.508 INFO:teuthology.orchestra.run.smithi107.stdout:pg_num: 1 2023-10-09T05:56:21.704 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:21 smithi107 ceph-mon[101102]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:56:21.705 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:21 smithi107 ceph-mon[118900]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:56:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:21 smithi150 ceph-mon[105942]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s, 0 objects/s recovering 2023-10-09T05:56:22.202 INFO:tasks.cephadm:Adding prometheus.a on smithi150 2023-10-09T05:56:22.203 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi150=a' 2023-10-09T05:56:22.439 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:22 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1685055586' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:56:22.558 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:22 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1685055586' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:56:22.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:22 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1685055586' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T05:56:23.770 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:23 smithi150 ceph-mon[105942]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:56:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:23 smithi107 ceph-mon[101102]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:56:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:23 smithi107 ceph-mon[118900]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T05:56:24.180 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled prometheus update... 2023-10-09T05:56:24.905 DEBUG:teuthology.orchestra.run.smithi150:prometheus.a> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T05:56:24.907 INFO:tasks.cephadm:Adding node-exporter.a on smithi107 2023-10-09T05:56:24.907 INFO:tasks.cephadm:Adding node-exporter.b on smithi150 2023-10-09T05:56:24.907 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi107=a;smithi150=b' 2023-10-09T05:56:24.938 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:56:25.273 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:25.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:25.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: Saving service prometheus spec with placement smithi150=a;count:1 2023-10-09T05:56:25.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.436 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:25.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:25.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:25.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: Deploying daemon prometheus.a on smithi150 2023-10-09T05:56:25.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:25 smithi150 ceph-mon[105942]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: Saving service prometheus spec with placement smithi150=a;count:1 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: Deploying daemon prometheus.a on smithi150 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[101102]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: Saving service prometheus spec with placement smithi150=a;count:1 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: Deploying daemon prometheus.a on smithi150 2023-10-09T05:56:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:25 smithi107 ceph-mon[118900]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-09T05:56:27.271 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled node-exporter update... 2023-10-09T05:56:27.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:27 smithi150 ceph-mon[105942]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:56:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:27 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:27.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:27 smithi107 ceph-mon[101102]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:56:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:27 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:27 smithi107 ceph-mon[118900]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T05:56:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:27 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:28.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:28 smithi107 ceph-mon[101102]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi107=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:28 smithi107 ceph-mon[101102]: Saving service node-exporter spec with placement smithi107=a;smithi150=b;count:2 2023-10-09T05:56:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:28 smithi107 ceph-mon[118900]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi107=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:28 smithi107 ceph-mon[118900]: Saving service node-exporter spec with placement smithi107=a;smithi150=b;count:2 2023-10-09T05:56:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:28 smithi150 ceph-mon[105942]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi107=a;smithi150=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:28 smithi150 ceph-mon[105942]: Saving service node-exporter spec with placement smithi107=a;smithi150=b;count:2 2023-10-09T05:56:28.929 DEBUG:teuthology.orchestra.run.smithi107:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T05:56:28.932 DEBUG:teuthology.orchestra.run.smithi150:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T05:56:28.935 INFO:tasks.cephadm:Adding alertmanager.a on smithi107 2023-10-09T05:56:28.935 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi107=a' 2023-10-09T05:56:28.967 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:56:28.970 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:56:29.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:29 smithi107 ceph-mon[101102]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:56:29.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:29 smithi107 ceph-mon[118900]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:56:29.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:29 smithi150 ceph-mon[105942]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T05:56:31.332 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:31 smithi107 ceph-mon[101102]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:31 smithi107 ceph-mon[118900]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:31 smithi150 ceph-mon[105942]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:33.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:33 smithi107 ceph-mon[101102]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:33 smithi107 ceph-mon[118900]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:33.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:33 smithi150 ceph-mon[105942]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:35.594 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:35 smithi150 ceph-mon[105942]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:35 smithi107 ceph-mon[101102]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:35.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:35 smithi107 ceph-mon[118900]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:36.275 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled alertmanager update... 2023-10-09T05:56:37.051 DEBUG:teuthology.orchestra.run.smithi107:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T05:56:37.054 INFO:tasks.cephadm:Adding grafana.a on smithi150 2023-10-09T05:56:37.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi150=a' 2023-10-09T05:56:37.085 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:-- Logs begin at Mon 2023-10-09 05:33:35 UTC. -- 2023-10-09T05:56:37.280 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 systemd[1]: Starting Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[101102]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi107=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[101102]: Saving service alertmanager spec with placement smithi107=a;count:1 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[101102]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[118900]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi107=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:37.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[118900]: Saving service alertmanager spec with placement smithi107=a;count:1 2023-10-09T05:56:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:37 smithi107 ceph-mon[118900]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:37.582 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:37 smithi150 ceph-mon[105942]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi107=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:56:37.582 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:37 smithi150 ceph-mon[105942]: Saving service alertmanager spec with placement smithi107=a;count:1 2023-10-09T05:56:37.582 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:37 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:37.582 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:37 smithi150 ceph-mon[105942]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:37.731 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:37.975 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.721Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:56:37.975 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.722Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:56:37.975 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.722Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi150 (none))" 2023-10-09T05:56:37.975 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.722Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:56:37.975 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.722Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.723Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.723Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 bash[148731]: bcc6466d2bff56865e92382306f2158cef1ae775900c5c82d2549fbf555f73a9 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.725Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.725Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 systemd[1]: Started Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.727Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.728Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.136µs 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.728Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.728Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.728Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.928µs wal_replay_duration=344.906µs wbl_replay_duration=117ns total_replay_duration=382.478µs 2023-10-09T05:56:37.976 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.729Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:56:37.977 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.729Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:56:37.977 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.729Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:56:37.977 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.748Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.13232ms db_storage=852ns remote_storage=1.156µs web_handler=499ns query_engine=620ns scrape=417.508µs scrape_sd=19.109µs notify=687ns notify_sd=1.654µs rules=18.50376ms tracing=7.707µs 2023-10-09T05:56:37.977 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.748Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:56:37.977 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:56:37 smithi150 conmon[148851]: ts=2023-10-09T05:56:37.748Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:56:39.078 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:38 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:38 smithi150 conmon[109180]: ignoring --setuser ceph since I am not root 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:38 smithi150 conmon[109180]: ignoring --setgroup ceph since I am not root 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:38 smithi150 ceph-mgr[109204]: -- 172.21.15.150:0/883697555 <== mon.1 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5579df57e9c0 con 0x5579df556c00 2023-10-09T05:56:39.079 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.018+0000 7ff276bf01c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:56:39.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:38 smithi107 conmon[101524]: ignoring --setuser ceph since I am not root 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:38 smithi107 conmon[101524]: ignoring --setgroup ceph since I am not root 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.014+0000 7fa74e8331c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.091+0000 7fa74e8331c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' 2023-10-09T05:56:39.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:38 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T05:56:39.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.179+0000 7fa74e8331c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:56:39.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.268+0000 7fa74e8331c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:56:39.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.099+0000 7ff276bf01c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T05:56:39.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.192+0000 7ff276bf01c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T05:56:39.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.289+0000 7ff276bf01c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T05:56:39.738 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.435+0000 7fa74e8331c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:56:39.764 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.460+0000 7ff276bf01c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T05:56:40.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:39 smithi107 ceph-mon[101102]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:56:40.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:39 smithi107 ceph-mon[101102]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:56:40.046 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.737+0000 7fa74e8331c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:56:40.046 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.824+0000 7fa74e8331c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:56:40.046 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:39 smithi107 conmon[101524]: 2023-10-09T05:56:39.953+0000 7fa74e8331c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:56:40.046 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:39 smithi107 ceph-mon[118900]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:56:40.047 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:39 smithi107 ceph-mon[118900]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:56:40.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:39 smithi150 ceph-mon[105942]: from='mgr.14152 172.21.15.107:0/2609008058' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T05:56:40.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:39 smithi150 ceph-mon[105942]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T05:56:40.069 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.763+0000 7ff276bf01c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T05:56:40.069 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.850+0000 7ff276bf01c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T05:56:40.070 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:39 smithi150 conmon[109180]: 2023-10-09T05:56:39.977+0000 7ff276bf01c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T05:56:40.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:40 smithi107 conmon[101524]: 2023-10-09T05:56:40.044+0000 7fa74e8331c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:56:40.403 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:40 smithi150 conmon[109180]: 2023-10-09T05:56:40.068+0000 7ff276bf01c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T05:56:40.900 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:40 smithi107 conmon[101524]: 2023-10-09T05:56:40.460+0000 7fa74e8331c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:56:40.903 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:40 smithi150 conmon[109180]: 2023-10-09T05:56:40.482+0000 7ff276bf01c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T05:56:41.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:41 smithi107 conmon[101524]: 2023-10-09T05:56:41.016+0000 7fa74e8331c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:56:41.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:41 smithi107 conmon[101524]: 2023-10-09T05:56:41.183+0000 7fa74e8331c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:56:41.403 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:41 smithi150 conmon[109180]: 2023-10-09T05:56:41.037+0000 7ff276bf01c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T05:56:41.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:41 smithi150 conmon[109180]: 2023-10-09T05:56:41.204+0000 7ff276bf01c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T05:56:42.195 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:41 smithi107 conmon[101524]: 2023-10-09T05:56:41.937+0000 7fa74e8331c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:56:42.195 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:42 smithi107 conmon[101524]: 2023-10-09T05:56:42.026+0000 7fa74e8331c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:56:42.212 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:41 smithi150 conmon[109180]: 2023-10-09T05:56:41.954+0000 7ff276bf01c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:56:42.212 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:42 smithi150 conmon[109180]: 2023-10-09T05:56:42.044+0000 7ff276bf01c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T05:56:42.468 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:42 smithi107 conmon[101524]: 2023-10-09T05:56:42.194+0000 7fa74e8331c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:56:42.485 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:42 smithi150 conmon[109180]: 2023-10-09T05:56:42.211+0000 7ff276bf01c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T05:56:42.837 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:42 smithi107 conmon[101524]: 2023-10-09T05:56:42.467+0000 7fa74e8331c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:56:42.837 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:42 smithi107 conmon[101524]: 2023-10-09T05:56:42.553+0000 7fa74e8331c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:56:42.859 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:42 smithi150 conmon[109180]: 2023-10-09T05:56:42.484+0000 7ff276bf01c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T05:56:42.859 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:42 smithi150 conmon[109180]: 2023-10-09T05:56:42.571+0000 7ff276bf01c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T05:56:43.096 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:42 smithi107 conmon[101524]: 2023-10-09T05:56:42.836+0000 7fa74e8331c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:56:43.117 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:42 smithi150 conmon[109180]: 2023-10-09T05:56:42.858+0000 7ff276bf01c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T05:56:43.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:43 smithi107 conmon[101524]: 2023-10-09T05:56:43.095+0000 7fa74e8331c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:56:43.403 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:43 smithi150 conmon[109180]: 2023-10-09T05:56:43.116+0000 7ff276bf01c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T05:56:44.117 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:43 smithi107 conmon[101524]: 2023-10-09T05:56:43.801+0000 7fa74e8331c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:56:44.117 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:43 smithi107 conmon[101524]: 2023-10-09T05:56:43.931+0000 7fa74e8331c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:56:44.129 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:43 smithi150 conmon[109180]: 2023-10-09T05:56:43.814+0000 7ff276bf01c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T05:56:44.129 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:43 smithi150 conmon[109180]: 2023-10-09T05:56:43.944+0000 7ff276bf01c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T05:56:44.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:44 smithi107 conmon[101524]: 2023-10-09T05:56:44.116+0000 7fa74e8331c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:56:44.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:44 smithi107 conmon[101524]: 2023-10-09T05:56:44.231+0000 7fa74e8331c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:56:44.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:44 smithi150 conmon[109180]: 2023-10-09T05:56:44.128+0000 7ff276bf01c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T05:56:44.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:44 smithi150 conmon[109180]: 2023-10-09T05:56:44.242+0000 7ff276bf01c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T05:56:44.900 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:44 smithi107 conmon[101524]: 2023-10-09T05:56:44.445+0000 7fa74e8331c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:56:44.903 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:44 smithi150 conmon[109180]: 2023-10-09T05:56:44.454+0000 7ff276bf01c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T05:56:45.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: 2023-10-09T05:56:45.291+0000 7fa74e8331c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:56:45.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: [09/Oct/2023:05:56:45] ENGINE Bus STARTING 2023-10-09T05:56:45.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: CherryPy Checker: 2023-10-09T05:56:45.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: The Application mounted at '' has an empty config. 2023-10-09T05:56:45.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: 2023-10-09T05:56:45.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: [09/Oct/2023:05:56:45] ENGINE Serving on http://:::9283 2023-10-09T05:56:45.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:56:45 smithi107 conmon[101524]: [09/Oct/2023:05:56:45] ENGINE Bus STARTED 2023-10-09T05:56:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: Active manager daemon y restarted 2023-10-09T05:56:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: Activating manager daemon y 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: mgrmap e17: y(active, starting, since 0.00675638s), standbys: x 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: Standby manager daemon x restarted 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: Standby manager daemon x started 2023-10-09T05:56:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:56:45.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:56:45.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: Manager daemon y is now available 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: Active manager daemon y restarted 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: Activating manager daemon y 2023-10-09T05:56:45.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: mgrmap e17: y(active, starting, since 0.00675638s), standbys: x 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: Standby manager daemon x restarted 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:56:45.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:56:45.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:56:45.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:56:45.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:56:45.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:56:45.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: Manager daemon y is now available 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:45.660 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: 2023-10-09T05:56:45.297+0000 7ff276bf01c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: [09/Oct/2023:05:56:45] ENGINE Bus STARTING 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: CherryPy Checker: 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: The Application mounted at '' has an empty config. 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: [09/Oct/2023:05:56:45] ENGINE Serving on http://:::9283 2023-10-09T05:56:45.661 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:45 smithi150 conmon[109180]: [09/Oct/2023:05:56:45] ENGINE Bus STARTED 2023-10-09T05:56:45.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: Active manager daemon y restarted 2023-10-09T05:56:45.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: Activating manager daemon y 2023-10-09T05:56:45.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T05:56:45.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: mgrmap e17: y(active, starting, since 0.00675638s), standbys: x 2023-10-09T05:56:45.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: Standby manager daemon x restarted 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T05:56:45.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2079387373' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T05:56:45.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T05:56:45.665 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: Manager daemon y is now available 2023-10-09T05:56:45.665 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:46.329 INFO:teuthology.orchestra.run.smithi150.stdout:Scheduled grafana update... 2023-10-09T05:56:46.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:46.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:46.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:46.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: mgrmap e18: y(active, since 1.00956s), standbys: x 2023-10-09T05:56:46.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: mgrmap e18: y(active, since 1.00956s), standbys: x 2023-10-09T05:56:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: mgrmap e18: y(active, since 1.00956s), standbys: x 2023-10-09T05:56:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:47.119 DEBUG:teuthology.orchestra.run.smithi150:grafana.a> sudo journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T05:56:47.123 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T05:56:47.124 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:56:47.156 INFO:journalctl@ceph.grafana.a.smithi150.stdout:-- Logs begin at Mon 2023-10-09 05:33:43 UTC. -- 2023-10-09T05:56:47.489 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:48.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: Saving service grafana spec with placement smithi150=a;count:1 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.058 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: Saving service grafana spec with placement smithi150=a;count:1 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: Saving service grafana spec with placement smithi150=a;count:1 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:48.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:48 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:56:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: [09/Oct/2023:05:56:47] ENGINE Bus STARTING 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: [09/Oct/2023:05:56:47] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: [09/Oct/2023:05:56:47] ENGINE Bus STARTED 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:56:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:49.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:48 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: [09/Oct/2023:05:56:47] ENGINE Bus STARTING 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: [09/Oct/2023:05:56:47] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: [09/Oct/2023:05:56:47] ENGINE Bus STARTED 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: [09/Oct/2023:05:56:47] ENGINE Bus STARTING 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: [09/Oct/2023:05:56:47] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: [09/Oct/2023:05:56:47] ENGINE Bus STARTED 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: mgrmap e19: y(active, since 3s), standbys: x 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:56:49.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:48 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:56:49.532 INFO:teuthology.orchestra.run.smithi107.stdout:[client.0] 2023-10-09T05:56:49.532 INFO:teuthology.orchestra.run.smithi107.stdout: key = AQAhliNlXqR9HxAAhJKkg0fzvUAyeXim5+QxRQ== 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 4036M 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.257 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 4036M 2023-10-09T05:56:50.258 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.275 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T05:56:50.276 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T05:56:50.276 DEBUG:teuthology.orchestra.run.smithi107:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T05:56:50.319 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 4036M 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/746490375' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:50.350 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:50.672 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:50 smithi150 ceph-mon[105942]: Deploying daemon alertmanager.a on smithi107 2023-10-09T05:56:51.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[101102]: Deploying daemon alertmanager.a on smithi107 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T05:56:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:50 smithi107 ceph-mon[118900]: Deploying daemon alertmanager.a on smithi107 2023-10-09T05:56:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:51 smithi150 ceph-mon[105942]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:52.338 INFO:teuthology.orchestra.run.smithi150.stdout:[client.1] 2023-10-09T05:56:52.338 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQAkliNl0X7XExAAMapcc/nVPxcs2axBK9qXHg== 2023-10-09T05:56:52.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:51 smithi107 ceph-mon[101102]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:51 smithi107 ceph-mon[118900]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:52.897 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2023-10-09T05:56:52.898 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T05:56:52.898 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T05:56:52.981 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T05:56:52.982 DEBUG:teuthology.orchestra.run.smithi107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T05:56:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:52 smithi150 ceph-mon[105942]: from='client.? 172.21.15.150:0/1841370904' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:52 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:52 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:53.318 INFO:teuthology.misc.health.smithi107.stdout: 2023-10-09T05:56:53.319 INFO:teuthology.misc.health.smithi107.stdout:{"epoch":59,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:56:45.306076+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:56:53.331 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T05:56:53.331 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T05:56:53.332 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T05:56:53.355 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[101102]: from='client.? 172.21.15.150:0/1841370904' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.355 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.356 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:53.356 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[118900]: from='client.? 172.21.15.150:0/1841370904' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.356 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T05:56:53.356 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:52 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T05:56:53.719 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:54.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:53 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3408821521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:54.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:53 smithi107 ceph-mon[101102]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:54.223 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:53 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3408821521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:54.223 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:53 smithi107 ceph-mon[118900]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:53 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3408821521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:56:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:53 smithi150 ceph-mon[105942]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:56 smithi107 ceph-mon[101102]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:56 smithi107 ceph-mon[118900]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:56 smithi150 ceph-mon[105942]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:57.390 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'rbd' created 2023-10-09T05:56:57.555 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:57 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1368471649' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:57.555 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:57 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:57.556 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:57 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1368471649' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:57.556 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:57 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:57 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1368471649' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:57 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T05:56:58.095 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T05:56:58.646 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[101102]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:58.646 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:56:58.646 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[101102]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:56:58.646 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[118900]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:58.646 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:56:58.647 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:58 smithi107 ceph-mon[118900]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:56:58.647 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 conmon[163776]: ts=2023-10-09T05:56:58.629Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:56:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:58 smithi150 ceph-mon[105942]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:56:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:58 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T05:56:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:58 smithi150 ceph-mon[105942]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T05:56:58.716 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:56:58.900 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 bash[163715]: 3dce8e40d11b0bfb6d71c195cac2d3f32ca6cec4a7de2458ae87f363271d79f7 2023-10-09T05:56:58.901 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 systemd[1]: Started Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:56:58.901 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 conmon[163776]: ts=2023-10-09T05:56:58.655Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:56:58.901 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 conmon[163776]: ts=2023-10-09T05:56:58.656Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:56:58.901 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 conmon[163776]: ts=2023-10-09T05:56:58.659Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T05:56:58.901 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:56:58 smithi107 conmon[163776]: ts=2023-10-09T05:56:58.659Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T05:56:59.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:56:58 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:56:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:56:59.651 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:56:59 smithi107 systemd[1]: Starting Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:56:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:56:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[101102]: Deploying daemon node-exporter.a on smithi107 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:56:59 smithi107 ceph-mon[118900]: Deploying daemon node-exporter.a on smithi107 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:56:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:56:59 smithi150 ceph-mon[105942]: Deploying daemon node-exporter.a on smithi107 2023-10-09T05:57:00.401 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:00 smithi107 bash[164468]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T05:57:00.650 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:00 smithi107 bash[164468]: Getting image source signatures 2023-10-09T05:57:00.651 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:00 smithi107 conmon[163776]: ts=2023-10-09T05:57:00.630Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000821577s 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:00 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: pgmap v12: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:00 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:01.150 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:00 smithi107 bash[164468]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T05:57:01.151 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:00 smithi107 bash[164468]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T05:57:01.151 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:00 smithi107 bash[164468]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T05:57:01.414 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T05:57:01.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:01 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2880477658' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:01 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:01 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2880477658' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:01 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:01.651 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:01 smithi107 bash[164468]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T05:57:01.651 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:01 smithi107 bash[164468]: Writing manifest to image destination 2023-10-09T05:57:01.651 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:01 smithi107 bash[164468]: Storing signatures 2023-10-09T05:57:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:01 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2880477658' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:01 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:57:02.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[101102]: pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:57:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[101102]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:57:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[118900]: pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:57:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:02 smithi107 ceph-mon[118900]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:57:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:02 smithi150 ceph-mon[105942]: pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:02 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:57:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:02 smithi150 ceph-mon[105942]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T05:57:03.211 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:57:03.213 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:57:03.214 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:57:03.737 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.646Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.646Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.647Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.647Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.647Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.647Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:57:03.901 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:57:03.902 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:57:03.903 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:57:03.904 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 conmon[164683]: ts=2023-10-09T05:57:03.648Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 bash[164468]: da46b9f4dd3d39a31e5eb467084f9f5c9a02f61ed1953a47b4fdc7b53e421b36 2023-10-09T05:57:03.905 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 05:57:03 smithi107 systemd[1]: Started Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:04.654 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:04 smithi150 systemd[1]: Starting Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: Deploying daemon node-exporter.b on smithi150 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:57:05.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[101102]: Cluster is now healthy 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: Deploying daemon node-exporter.b on smithi150 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:57:05.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:04 smithi107 ceph-mon[118900]: Cluster is now healthy 2023-10-09T05:57:05.151 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:04 smithi150 bash[151879]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: Deploying daemon node-exporter.b on smithi150 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:57:05.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:04 smithi150 ceph-mon[105942]: Cluster is now healthy 2023-10-09T05:57:05.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Getting image source signatures 2023-10-09T05:57:05.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T05:57:05.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T05:57:05.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T05:57:05.593 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:05.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:05 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2510368256' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:57:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:05 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2510368256' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:57:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:05 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2510368256' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:57:05.904 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T05:57:05.904 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Writing manifest to image destination 2023-10-09T05:57:05.904 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:05 smithi150 bash[151879]: Storing signatures 2023-10-09T05:57:06.226 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":19,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6800","nonce":2525813986},{"type":"v1","addr":"172.21.15.107:6801","nonce":2525813986}]},"active_addr":"172.21.15.107:6801/2525813986","active_change":"2023-10-09T05:56:45.306192+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24353,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":3067456200}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":3684818042}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2888496999}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2930305484}]}]} 2023-10-09T05:57:06.231 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:57:06.231 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:57:06.232 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:57:06.568 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:07.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:06 smithi107 ceph-mon[101102]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:07.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:06 smithi107 ceph-mon[118900]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:07.046 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:06 smithi150 ceph-mon[105942]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:07.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.147Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T05:57:07.405 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T05:57:07.406 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T05:57:07.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T05:57:07.408 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 conmon[151955]: ts=2023-10-09T05:57:07.148Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 bash[151879]: 3ead6ee07e6f1295a28fd8c279e614e6c30e4c097027ae88e163730022f90f9f 2023-10-09T05:57:07.409 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 05:57:07 smithi150 systemd[1]: Started Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE Bus STOPPING 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE Bus STOPPED 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE Bus STARTING 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE Serving on http://:::9283 2023-10-09T05:57:07.889 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:07 smithi107 conmon[101524]: [09/Oct/2023:05:57:07] ENGINE Bus STARTED 2023-10-09T05:57:08.275 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:08.275 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":63,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:57:01.400889+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:57:08.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[101102]: Deploying daemon grafana.a on smithi150 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.505 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.505 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.505 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:08 smithi107 ceph-mon[118900]: Deploying daemon grafana.a on smithi150 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T05:57:08.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:08.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:08 smithi150 ceph-mon[105942]: Deploying daemon grafana.a on smithi150 2023-10-09T05:57:08.900 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:08 smithi107 conmon[163776]: ts=2023-10-09T05:57:08.633Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00407758s 2023-10-09T05:57:09.010 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:57:09.011 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:57:09.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:08 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:09.354 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:09.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:09 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2105584295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:09.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:09 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2105584295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:09 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2105584295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:10 smithi107 ceph-mon[101102]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:10 smithi107 ceph-mon[118900]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:10.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:10 smithi150 ceph-mon[105942]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:11.049 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:11.049 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":63,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:57:01.400889+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:57:11.625 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:57:11.626 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:57:11.626 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:57:11.627 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:57:11.627 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:57:11.628 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:57:11.628 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:57:11.628 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:57:11.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:11 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:11.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:11 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1756110417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:11 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:11 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1756110417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:11.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:11 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:11 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1756110417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:57:12.201 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.319 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.382 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.452 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.505 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:12 smithi107 ceph-mon[101102]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:12 smithi107 ceph-mon[118900]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:12.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:12 smithi150 ceph-mon[105942]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:12.836 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:12.837 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:14.284 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:13 smithi107 ceph-mon[101102]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:14.284 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:13 smithi107 ceph-mon[118900]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:13 smithi150 ceph-mon[105942]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:15.731 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[101102]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[118900]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:16 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:16.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:16 smithi150 ceph-mon[105942]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:16 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:16 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:16 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:18.491 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:18 smithi107 ceph-mon[101102]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:18.492 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:18 smithi107 ceph-mon[118900]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:18 smithi150 ceph-mon[105942]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:19.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:18 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:20.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:20 smithi107 ceph-mon[101102]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:20 smithi107 ceph-mon[118900]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:20 smithi150 ceph-mon[105942]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:20.674 INFO:teuthology.orchestra.run.smithi107.stdout:163208757273 2023-10-09T05:57:20.674 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:57:21.020 INFO:teuthology.orchestra.run.smithi107.stdout:55834574891 2023-10-09T05:57:21.021 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:57:21.116 INFO:teuthology.orchestra.run.smithi107.stdout:128849018908 2023-10-09T05:57:21.117 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:57:21.268 INFO:teuthology.orchestra.run.smithi107.stdout:231928233999 2023-10-09T05:57:21.268 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:57:22.430 INFO:teuthology.orchestra.run.smithi107.stdout:34359738416 2023-10-09T05:57:22.430 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:57:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:22 smithi107 ceph-mon[101102]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:22.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:22 smithi107 ceph-mon[118900]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:22 smithi150 ceph-mon[105942]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:22.664 INFO:teuthology.orchestra.run.smithi107.stdout:197568495636 2023-10-09T05:57:22.664 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:57:22.940 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:22.987 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:23.509 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:23.991 INFO:teuthology.orchestra.run.smithi107.stdout:77309411366 2023-10-09T05:57:23.991 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:57:24.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:23 smithi107 ceph-mon[101102]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:24.180 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:23 smithi107 ceph-mon[118900]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:23 smithi150 ceph-mon[105942]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:25.181 INFO:teuthology.orchestra.run.smithi107.stdout:107374182433 2023-10-09T05:57:25.181 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:57:26.139 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:26.162 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:26.379 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:26 smithi150 systemd[1]: Starting Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:26.560 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:26.560 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:26.633 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:26 smithi107 ceph-mon[101102]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:26.633 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:26 smithi107 ceph-mon[118900]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:26 smithi150 ceph-mon[105942]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:27.201 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:26 smithi150 bash[152624]: eaf4772a342e6dae5240371e1e42448adaf05fa001bfcde0fa9062a3b50607c0 2023-10-09T05:57:27.201 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:26 smithi150 systemd[1]: Started Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079126165Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079328953Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079347826Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079363573Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079369636Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.0793754Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079383006Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079388719Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079394555Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079400134Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T05:57:27.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079406Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079411581Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079417123Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=settings t=2023-10-09T05:57:27.079422712Z level=info msg="App mode production" 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=sqlstore t=2023-10-09T05:57:27.079474911Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=sqlstore t=2023-10-09T05:57:27.079522798Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.080393492Z level=info msg="Starting DB migrations" 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.080470874Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.109390416Z level=info msg="Executing migration" id="create user table" 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.159845597Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T05:57:27.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.201715528Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T05:57:27.232 INFO:teuthology.orchestra.run.smithi107.stdout:163208757274 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.260959342Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.285960839Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.311173946Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.353137478Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.428388753Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T05:57:27.470 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.470370914Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T05:57:27.750 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.512583298Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.529288674Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.562841999Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.579787505Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.596665343Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.630172534Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.647350496Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.663957622Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.697470692Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T05:57:27.751 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.751245651Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T05:57:28.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.823478271Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T05:57:28.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.874403978Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T05:57:28.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:27 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:27.982730795Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T05:57:28.267 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.051321942Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T05:57:28.267 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.143043181Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T05:57:28.267 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.192293464Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T05:57:28.268 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.23416207Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3935409100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:57:28.268 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:28 smithi150 ceph-mon[105942]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:28.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3935409100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[101102]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3935409100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:57:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:28 smithi107 ceph-mon[118900]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.267817624Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.320446722Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.345585649Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.413921275Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.4461622Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.46300626Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.47983504Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T05:57:28.530 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.496694452Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T05:57:28.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.513576086Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T05:57:28.531 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.530442432Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T05:57:28.790 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.555818812Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.572574336Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.589474415Z level=info msg="Executing migration" id="create star table" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.631450934Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.648384495Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.715270992Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.765797064Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T05:57:28.791 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.79102125Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T05:57:29.096 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.841762837Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T05:57:29.096 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:28 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:28.95887428Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T05:57:29.096 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.017926878Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T05:57:29.096 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.096233669Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.126313932Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.151495859Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.193385182Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.218589429Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.243813985Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.26899203Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.325976442Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T05:57:29.384 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.384169974Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T05:57:29.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.487196648Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T05:57:29.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.546006803Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T05:57:29.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.609363528Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T05:57:29.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.663117836Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T05:57:29.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.697001426Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.722174968Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.764091916Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.789263292Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.814501647Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.839659946Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.884252927Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T05:57:29.993 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:29 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:29.993061299Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T05:57:30.156 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757274 for osd.5 2023-10-09T05:57:30.157 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:30.308 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.015978943Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.040924085Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.09148686Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.150038904Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.1752771Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.192072003Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.242824371Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T05:57:30.309 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.309248139Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T05:57:30.313 INFO:teuthology.orchestra.run.smithi107.stdout:55834574893 2023-10-09T05:57:30.379 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[118900]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:30.379 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3806356151' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:57:30.379 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.379 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.561 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:30 smithi150 ceph-mon[105942]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:30.561 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:30 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3806356151' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:57:30.561 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:30 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.561 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:30 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.368804073Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.402038348Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.462129992Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.486003595Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.52780442Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.544733854Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T05:57:30.562 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.561664924Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T05:57:30.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[101102]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:30.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3806356151' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:57:30.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:30 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:30.821 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.604161568Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.645966725Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.687866126Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.713244824Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.738213316Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.763395569Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.788603194Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T05:57:30.822 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.822183693Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T05:57:31.079 INFO:teuthology.orchestra.run.smithi107.stdout:231928234001 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.872696892Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.914744002Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:30 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:30.989965482Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.006918122Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.02363518Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.048912653Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.065754627Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T05:57:31.082 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.082632765Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T05:57:31.359 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.107952782Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.124930386Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.158495398Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.175408117Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.192187612Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.209143564Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.225923607Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.250979978Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.292967938Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.318056832Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T05:57:31.360 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.360243427Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T05:57:31.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:31.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:31.381 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/838469843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:57:31.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:31.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:31.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/838469843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:57:31.487 INFO:teuthology.orchestra.run.smithi107.stdout:197568495639 2023-10-09T05:57:31.618 INFO:teuthology.orchestra.run.smithi107.stdout:34359738418 2023-10-09T05:57:31.627 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:31 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:31.628 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:31 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:31.628 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/838469843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.410656371Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.494333886Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.519389229Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.544565923Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.586557741Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T05:57:31.628 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.628423059Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.670871991Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.696216271Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.721243008Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.746362922Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.788448315Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T05:57:31.880 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.813485856Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T05:57:31.881 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.838731004Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T05:57:31.881 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.880689311Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T05:57:32.118 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:32.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.906037864Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:31 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:31.974159034Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.006614628Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.048386776Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.073536634Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.098795637Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.124117242Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T05:57:32.149 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.149224979Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T05:57:32.400 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.199714349Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.224850971Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.250129393Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.275304245Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.300321838Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.325628306Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.367454181Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.38445612Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T05:57:32.401 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.401240392Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T05:57:32.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:32 smithi150 ceph-mon[105942]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:32.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2317032366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:57:32.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3925754923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:57:32.653 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.434787547Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T05:57:32.653 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.501924738Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T05:57:32.653 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.518720733Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.535602966Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.56921113Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.586149227Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.602834804Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.636364285Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T05:57:32.654 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.653468991Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T05:57:32.704 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[101102]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:32.704 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2317032366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:57:32.704 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3925754923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:57:32.705 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[118900]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:32.705 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2317032366' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:57:32.705 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3925754923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:57:32.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.712730658Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T05:57:32.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.729590409Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T05:57:32.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.746284934Z level=info msg="Executing migration" id="create session table" 2023-10-09T05:57:32.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.763195693Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T05:57:32.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.805007711Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T05:57:32.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.847107897Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T05:57:32.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.872077876Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T05:57:32.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.897268532Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T05:57:33.190 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:32 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:32.939373562Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.014679176Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.039887421Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.065044131Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.09014844Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.115370074Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.140476438Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.165798552Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T05:57:33.191 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.191041491Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T05:57:33.492 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.224740699Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T05:57:33.492 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.249795366Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.274950129Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.300233146Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.325317874Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.35074367Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.392680459Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.41786473Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T05:57:33.493 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.493073929Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T05:57:33.536 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234001 for osd.7 2023-10-09T05:57:33.536 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.526731062Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.568581833Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.610614088Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.652494829Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.67774238Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T05:57:33.761 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.711216301Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T05:57:33.762 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.736281344Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T05:57:33.762 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.761535654Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T05:57:34.012 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:33 smithi150 ceph-mon[105942]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.81210591Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.837171182Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.887743975Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.913035831Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:33 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:33.979913789Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T05:57:34.013 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.013278823Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T05:57:34.137 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:33 smithi107 ceph-mon[101102]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:34.137 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:33 smithi107 ceph-mon[118900]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:34.193 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574893 for osd.1 2023-10-09T05:57:34.194 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.055659422Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.080916247Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.106212357Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.131260748Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.156316852Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.181517351Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.206837433Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.231876453Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T05:57:34.282 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.25722467Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T05:57:34.283 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.282344494Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T05:57:34.542 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.32430238Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.341163117Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.358087337Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.375077079Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.392036193Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.408726954Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.425659244Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.442385825Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.509474918Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.526364539Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T05:57:34.543 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.543232046Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.593522088Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.610502783Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.643949327Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.660884524Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.69446071Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T05:57:34.812 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.728106957Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T05:57:34.813 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.744941474Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T05:57:34.813 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.786963561Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T05:57:34.813 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.812045758Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T05:57:34.905 INFO:teuthology.orchestra.run.smithi107.stdout:77309411368 2023-10-09T05:57:35.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:34 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3463014075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:57:35.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:34 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2461357525' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.854653126Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.88824273Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.90513507Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:34 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:34.938523787Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.013953931Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T05:57:35.064 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.039079576Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T05:57:35.065 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.064211024Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T05:57:35.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:34 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3463014075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:57:35.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:34 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2461357525' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:57:35.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:34 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3463014075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:57:35.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:34 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2461357525' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:57:35.349 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.114814965Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T05:57:35.349 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.156732986Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T05:57:35.349 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.181989292Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T05:57:35.349 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.207228563Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T05:57:35.349 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.232236674Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T05:57:35.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.257436868Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T05:57:35.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.282574691Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T05:57:35.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.307921957Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T05:57:35.350 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.349730177Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T05:57:35.372 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495636 got 197568495639 for osd.6 2023-10-09T05:57:35.372 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:35.495 INFO:teuthology.orchestra.run.smithi107.stdout:128849018911 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.392020165Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.417188265Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.458968203Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.526176767Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.551186539Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.576422251Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T05:57:35.618 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.618269547Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.660421945Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.685587209Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.727348159Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.769289874Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.794366001Z level=info msg="Executing migration" id="create team table" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.836313227Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T05:57:35.878 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.878457646Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T05:57:36.129 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:35 smithi150 ceph-mon[105942]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.912222063Z level=info msg="Executing migration" id="create team member table" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:35 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:35.953954191Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.029623124Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.054637976Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.079837875Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.105067362Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T05:57:36.130 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.130168257Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T05:57:36.382 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.155687956Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T05:57:36.382 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.172312507Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T05:57:36.382 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.189248837Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.206059463Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.223145067Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.239810007Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.273334065Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.290248037Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.307243259Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.340734758Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.357692147Z level=info msg="Executing migration" id="create tag table" 2023-10-09T05:57:36.383 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.382868743Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T05:57:36.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:35 smithi107 ceph-mon[101102]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:35 smithi107 ceph-mon[118900]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:36.624 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411368 for osd.2 2023-10-09T05:57:36.624 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:36.650 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.425030546Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.466889434Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.533897419Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.55902604Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.584111242Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.609213061Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T05:57:36.651 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.651207253Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T05:57:36.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2023-10-09T05:57:36.661 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:36.747 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2023-10-09T05:57:36.747 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.701923741Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.751964739Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.785514736Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.819167568Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.844182135Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T05:57:36.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.869222955Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.911081784Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.936427905Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:36 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:36.961604544Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.028511925Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.070363065Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.095756787Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T05:57:37.171 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.129577257Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T05:57:37.172 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.171522876Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T05:57:37.212 INFO:teuthology.orchestra.run.smithi107.stdout:107374182435 2023-10-09T05:57:37.448 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:37 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/909262828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:57:37.448 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.196952671Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T05:57:37.448 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.222038569Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.280645863Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.305938414Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.347744748Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.364498477Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.381527066Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.398198395Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T05:57:37.449 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.41505972Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T05:57:37.450 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.448789247Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T05:57:37.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:37 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/909262828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:57:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:37 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/909262828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:57:37.708 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.513415759Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.532844107Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.566252475Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.591658187Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.633548135Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.65882943Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.683836122Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T05:57:37.709 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.709067678Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T05:57:38.016 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.751107283Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T05:57:38.016 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.793178622Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T05:57:38.016 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.818294396Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T05:57:38.016 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.843573028Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T05:57:38.017 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.868692399Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T05:57:38.017 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.910698002Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T05:57:38.017 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:37 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:37.952544842Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T05:57:38.017 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.01682822Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T05:57:38.131 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182433 got 107374182435 for osd.3 2023-10-09T05:57:38.131 DEBUG:teuthology.parallel:result is None 2023-10-09T05:57:38.132 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:57:38.132 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:57:38.270 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.044967467Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T05:57:38.270 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.086810133Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T05:57:38.270 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.111982533Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T05:57:38.270 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.128760352Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T05:57:38.271 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.145807843Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T05:57:38.271 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.162569466Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T05:57:38.271 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.187985683Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T05:57:38.273 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.21315987Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T05:57:38.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.230063161Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T05:57:38.274 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.24697312Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T05:57:38.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:38 smithi107 ceph-mon[101102]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:38 smithi107 ceph-mon[118900]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:38.549 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:38 smithi150 ceph-mon[105942]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:38.549 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.289080172Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T05:57:38.549 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.314226723Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T05:57:38.549 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.331213592Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T05:57:38.549 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.34796327Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.364770889Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.381651737Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.398613799Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.432200896Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.448978033Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.482623396Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T05:57:38.550 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.549607584Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T05:57:38.556 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:38.801 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.574804282Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.591667771Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.608607392Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.642132164Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.684141943Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.709358022Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.734503332Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.776568704Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T05:57:38.802 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.801914679Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T05:57:39.078 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.844049618Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T05:57:39.078 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.885943692Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T05:57:39.078 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.911025574Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T05:57:39.078 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.936157861Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T05:57:39.079 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:38 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:38.961368448Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T05:57:39.079 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.003216519Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T05:57:39.079 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.078721666Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T05:57:39.355 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.1123906Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T05:57:39.355 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.137427473Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.162769338Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.188107553Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.21323335Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.238480351Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.263805108Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.28890269Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.314212352Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T05:57:39.356 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.355999084Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.406565368Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.431621524Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.473549546Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.543560603Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.574047661Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.599327497Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.599470727Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T05:57:39.641 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.641115445Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T05:57:39.892 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.666319883Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T05:57:39.892 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.691437516Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T05:57:39.892 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.691634792Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T05:57:39.892 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.691833607Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.716744825Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.758684294Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.80063847Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.842350494Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.867624955Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T05:57:39.893 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.892680761Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T05:57:40.154 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.943228747Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T05:57:40.154 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:39 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:39.985207635Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T05:57:40.154 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.06022908Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T05:57:40.154 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.102167177Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T05:57:40.154 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.127319968Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T05:57:40.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:40.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:40.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:40.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:40 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.430 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.178972189Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T05:57:40.430 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.194800642Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T05:57:40.430 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.211482851Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T05:57:40.430 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.23672856Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.262095569Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.287253534Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.312244246Z level=info msg="Executing migration" id="create permission table" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.337399936Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.379309708Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T05:57:40.431 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.429941063Z level=info msg="Executing migration" id="create role table" 2023-10-09T05:57:40.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:40 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.489032322Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.564134953Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.589358039Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.614555388Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.65648402Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T05:57:40.681 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.681607818Z level=info msg="Executing migration" id="create team role table" 2023-10-09T05:57:40.700 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:40.702 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T05:57:40.942 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.706948495Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T05:57:40.942 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.732069011Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T05:57:40.942 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.774115469Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T05:57:40.942 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.790889057Z level=info msg="Executing migration" id="create user role table" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.807736569Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.824657096Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.858192872Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.875063206Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.891934154Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.908835158Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T05:57:40.943 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.942380494Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:40 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:40.984641108Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.001463003Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.076576134Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.101790817Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.126928341Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.152282154Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T05:57:41.202 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.177212425Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T05:57:41.203 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.202476499Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T05:57:41.479 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:41 smithi150 ceph-mon[105942]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:57:41.479 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:41 smithi150 ceph-mon[105942]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T05:57:41.479 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:41 smithi150 ceph-mon[105942]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:41.479 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.252995493Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T05:57:41.479 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.278255419Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.303224321Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.345329793Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.370268563Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.412311141Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.437401428Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T05:57:41.480 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.479316774Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[101102]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[101102]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[101102]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[118900]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[118900]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T05:57:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:41 smithi107 ceph-mon[118900]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.521771487Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.572001821Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.597124575Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.639074969Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.664277555Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.689424354Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T05:57:41.731 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.731417605Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T05:57:41.991 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.756908009Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T05:57:41.991 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.798816109Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T05:57:41.991 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.840657278Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:57:41.991 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.840816778Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:57:41.991 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.865901428Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:57:41.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.866039463Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:57:41.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.90768929Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T05:57:41.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.949567513Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T05:57:41.992 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:41 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:41.991658679Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T05:57:42.074 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2023-10-09T05:57:39.326484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384956,"kb_used_data":6652,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576260,"statfs":{"total":767960285184,"available":765518090240,"internally_reserved":0,"allocated":6811648,"data_stored":3875712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002219"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.426860+0000","last_change":"2023-10-09T05:56:58.401504+0000","last_active":"2023-10-09T05:56:59.426860+0000","last_peered":"2023-10-09T05:56:59.426860+0000","last_clean":"2023-10-09T05:56:59.426860+0000","last_became_active":"2023-10-09T05:56:58.401182+0000","last_became_peered":"2023-10-09T05:56:58.401182+0000","last_unstale":"2023-10-09T05:56:59.426860+0000","last_undegraded":"2023-10-09T05:56:59.426860+0000","last_fullsized":"2023-10-09T05:56:59.426860+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.219090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395857+0000","last_change":"2023-10-09T05:56:58.672381+0000","last_active":"2023-10-09T05:56:59.395857+0000","last_peered":"2023-10-09T05:56:59.395857+0000","last_clean":"2023-10-09T05:56:59.395857+0000","last_became_active":"2023-10-09T05:56:58.672020+0000","last_became_peered":"2023-10-09T05:56:58.672020+0000","last_unstale":"2023-10-09T05:56:59.395857+0000","last_undegraded":"2023-10-09T05:56:59.395857+0000","last_fullsized":"2023-10-09T05:56:59.395857+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:25.352710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395428+0000","last_change":"2023-10-09T05:56:58.672203+0000","last_active":"2023-10-09T05:56:59.395428+0000","last_peered":"2023-10-09T05:56:59.395428+0000","last_clean":"2023-10-09T05:56:59.395428+0000","last_became_active":"2023-10-09T05:56:58.671871+0000","last_became_peered":"2023-10-09T05:56:58.671871+0000","last_unstale":"2023-10-09T05:56:59.395428+0000","last_undegraded":"2023-10-09T05:56:59.395428+0000","last_fullsized":"2023-10-09T05:56:59.395428+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:35.636544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395416+0000","last_change":"2023-10-09T05:56:58.401311+0000","last_active":"2023-10-09T05:56:59.395416+0000","last_peered":"2023-10-09T05:56:59.395416+0000","last_clean":"2023-10-09T05:56:59.395416+0000","last_became_active":"2023-10-09T05:56:58.400912+0000","last_became_peered":"2023-10-09T05:56:58.400912+0000","last_unstale":"2023-10-09T05:56:59.395416+0000","last_undegraded":"2023-10-09T05:56:59.395416+0000","last_fullsized":"2023-10-09T05:56:59.395416+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:19.221754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395416+0000","last_change":"2023-10-09T05:56:15.968136+0000","last_active":"2023-10-09T05:56:59.395416+0000","last_peered":"2023-10-09T05:56:59.395416+0000","last_clean":"2023-10-09T05:56:59.395416+0000","last_became_active":"2023-10-09T05:56:15.967691+0000","last_became_peered":"2023-10-09T05:56:15.967691+0000","last_unstale":"2023-10-09T05:56:59.395416+0000","last_undegraded":"2023-10-09T05:56:59.395416+0000","last_fullsized":"2023-10-09T05:56:59.395416+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:30.631087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:57:00.057707+0000","last_change":"2023-10-09T05:56:58.400169+0000","last_active":"2023-10-09T05:57:00.057707+0000","last_peered":"2023-10-09T05:57:00.057707+0000","last_clean":"2023-10-09T05:57:00.057707+0000","last_became_active":"2023-10-09T05:56:58.399784+0000","last_became_peered":"2023-10-09T05:56:58.399784+0000","last_unstale":"2023-10-09T05:57:00.057707+0000","last_undegraded":"2023-10-09T05:57:00.057707+0000","last_fullsized":"2023-10-09T05:57:00.057707+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:07.486985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.396000+0000","last_change":"2023-10-09T05:56:58.751442+0000","last_active":"2023-10-09T05:56:59.396000+0000","last_peered":"2023-10-09T05:56:59.396000+0000","last_clean":"2023-10-09T05:56:59.396000+0000","last_became_active":"2023-10-09T05:56:58.751121+0000","last_became_peered":"2023-10-09T05:56:58.751121+0000","last_unstale":"2023-10-09T05:56:59.396000+0000","last_undegraded":"2023-10-09T05:56:59.396000+0000","last_fullsized":"2023-10-09T05:56:59.396000+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:25.340194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:57:00.057698+0000","last_change":"2023-10-09T05:56:58.401280+0000","last_active":"2023-10-09T05:57:00.057698+0000","last_peered":"2023-10-09T05:57:00.057698+0000","last_clean":"2023-10-09T05:57:00.057698+0000","last_became_active":"2023-10-09T05:56:58.400923+0000","last_became_peered":"2023-10-09T05:56:58.400923+0000","last_unstale":"2023-10-09T05:57:00.057698+0000","last_undegraded":"2023-10-09T05:57:00.057698+0000","last_fullsized":"2023-10-09T05:57:00.057698+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:13.215907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395955+0000","last_change":"2023-10-09T05:56:58.400200+0000","last_active":"2023-10-09T05:56:59.395955+0000","last_peered":"2023-10-09T05:56:59.395955+0000","last_clean":"2023-10-09T05:56:59.395955+0000","last_became_active":"2023-10-09T05:56:58.399933+0000","last_became_peered":"2023-10-09T05:56:58.399933+0000","last_unstale":"2023-10-09T05:56:59.395955+0000","last_undegraded":"2023-10-09T05:56:59.395955+0000","last_fullsized":"2023-10-09T05:56:59.395955+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:57.074449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.373},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.5299999999999998,"5min":2.5299999999999998,"15min":2.5299999999999998},"last":0.53400000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.70399999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.747}]},{"osd":1,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.81999999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.69199999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.503},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.71799999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.80300000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.32900000000000001,"15min":0.29899999999999999},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54700000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33400000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.56499999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.496}]},{"osd":2,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40100000000000002,"15min":0.39100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52800000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.53500000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:56:47 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.59199999999999997,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.59799999999999998,"15min":0.60599999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82999999999999996,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.64200000000000002,"15min":0.65000000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82399999999999995,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.69199999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.61099999999999999,"15min":0.624},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85499999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.628},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.84499999999999997,"5min":1.04,"15min":1.04},"last":0.67900000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.73199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.751}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.34999999999999998,"15min":0.33600000000000002},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.379,"5min":0.35399999999999998,"15min":0.33600000000000002},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.36099999999999999,"15min":0.35399999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.627,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.59899999999999998,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.441}]},{"osd":4,"last update":"Mon Oct 9 05:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73899999999999999,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.505},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.75,"5min":0.879,"15min":0.879},"last":0.41999999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.60499999999999998,"15min":0.61399999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.78200000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72199999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48599999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.436},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.51600000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":2.9020000000000001,"5min":2.9020000000000001,"15min":2.9020000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.9390000000000001,"5min":2.9390000000000001,"15min":2.9390000000000001},"last":0.42199999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.42299999999999999,"15min":0.39900000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.9319999999999999,"5min":2.9319999999999999,"15min":2.9319999999999999},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46600000000000003,"15min":0.46899999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.47399999999999998,"15min":0.48399999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52000000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.60199999999999998,"15min":0.59899999999999998},"min":{"1min":0.47399999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79000000000000004,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.46400000000000002,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.60899999999999999,"15min":0.59599999999999997},"min":{"1min":0.46200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":2.2989999999999999,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.65500000000000003,"15min":0.64500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.831,"5min":2.831,"15min":2.831},"last":0.76000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.73299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":2.8719999999999999,"5min":2.8719999999999999,"15min":2.8719999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.747,"5min":0.747,"15min":0.747},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":3.0190000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.66400000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.42199999999999999,"15min":0.42999999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.54000000000000004,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.217},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.33400000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.46500000000000002,"15min":0.47199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.48299999999999998,"15min":0.49399999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.496,"15min":0.505},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:57:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.58999999999999997,"15min":0.59899999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81599999999999995,"5min":1.077,"15min":1.077},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.65800000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.61099999999999999,"15min":0.626},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83599999999999997,"5min":1,"15min":1},"last":0.63200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.68899999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.67100000000000004}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53800000000000003,"15min":0.54800000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.67300000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.499},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53900000000000003,"15min":0.54100000000000004},"min":{"1min":0.41999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.74099999999999999,"5min":0.753,"15min":0.753},"last":0.51700000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.66800000000000004,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.55800000000000005,"15min":0.56699999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71899999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.58699999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56699999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81499999999999995,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68700000000000006,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.627}]},{"osd":3,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69999999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.72399999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.73499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.443,"15min":0.435},"min":{"1min":0.31,"5min":0.245,"15min":0.245},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.71999999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.85399999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.51200000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.48899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55800000000000005,"15min":0.55200000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.443,"15min":0.43099999999999999},"min":{"1min":0.39600000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.628}]},{"osd":6,"last update":"Mon Oct 9 05:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.60899999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.68100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:57:42.077 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:57:42.259 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.075387225Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T05:57:42.259 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.100509093Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T05:57:42.259 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.125534113Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T05:57:42.259 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.167566334Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T05:57:42.260 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.192800162Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T05:57:42.260 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.21796427Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T05:57:42.260 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.259867252Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T05:57:42.519 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:42 smithi150 ceph-mon[105942]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.285074512Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.310159627Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.335225115Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.37712655Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.41910988Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.444167652Z level=info msg="Executing migration" id="add share column" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.469296282Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.494635119Z level=info msg="Executing migration" id="create file table" 2023-10-09T05:57:42.520 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.519883009Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T05:57:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:42 smithi107 ceph-mon[101102]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:42 smithi107 ceph-mon[118900]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:42.779 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.603612817Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T05:57:42.779 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.645406054Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T05:57:42.779 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.670593593Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T05:57:42.779 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.712541901Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T05:57:42.779 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.737681026Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T05:57:42.789 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:43.089 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.779572284Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T05:57:43.089 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.813548882Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.838845459Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.880589223Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.905793588Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.956019409Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:42 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:42.997954061Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.023220606Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T05:57:43.090 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.089984038Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T05:57:43.357 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.132007815Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T05:57:43.358 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.165700008Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T05:57:43.358 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.198922644Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T05:57:43.358 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.257731198Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T05:57:43.358 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.30783201Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T05:57:43.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.358251523Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T05:57:43.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.383311113Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T05:57:43.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.408366379Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T05:57:43.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.433611958Z level=info msg="Executing migration" id="create folder table" 2023-10-09T05:57:43.609 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.475574873Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T05:57:43.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.500670337Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T05:57:43.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.54258774Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T05:57:43.610 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.609607877Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=migrator t=2023-10-09T05:57:43.66010319Z level=info msg="migrations completed" performed=485 skipped=0 duration=16.579651195s 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=sqlstore t=2023-10-09T05:57:43.660700727Z level=info msg="Created default organization" 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=plugin.loader t=2023-10-09T05:57:43.759141277Z level=info msg="Plugin registered" pluginID=input 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=plugin.loader t=2023-10-09T05:57:43.786242878Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=plugin.loader t=2023-10-09T05:57:43.786265179Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T05:57:43.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=secrets t=2023-10-09T05:57:43.78629826Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T05:57:43.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=query_data t=2023-10-09T05:57:43.78808236Z level=info msg="Query Service initialization" 2023-10-09T05:57:43.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:43 smithi150 conmon[152665]: logger=live.push_http t=2023-10-09T05:57:43.79071764Z level=info msg="Live Push Gateway initialization" 2023-10-09T05:57:43.975 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:43 smithi107 systemd[1]: Stopping Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:43.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:43 smithi107 ceph-mon[118900]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:43 smithi107 ceph-mon[101102]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:44.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:43 smithi150 ceph-mon[105942]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:44.872 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:44 smithi107 conmon[163776]: ts=2023-10-09T05:57:44.617Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:57:44.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=infra.usagestats.collector t=2023-10-09T05:57:44.56540184Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T05:57:44.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=provisioning.datasources t=2023-10-09T05:57:44.565831617Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T05:57:44.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=provisioning.datasources t=2023-10-09T05:57:44.598317868Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T05:57:44.904 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=provisioning.datasources t=2023-10-09T05:57:44.671241924Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=provisioning.alerting t=2023-10-09T05:57:44.719785531Z level=info msg="starting to provision alerting" 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=provisioning.alerting t=2023-10-09T05:57:44.719823308Z level=info msg="finished to provision alerting" 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=ngalert.state.manager t=2023-10-09T05:57:44.719897513Z level=info msg="Warming state cache for startup" 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=grafanaStorageLogger t=2023-10-09T05:57:44.719924102Z level=info msg="storage starting" 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=http.server t=2023-10-09T05:57:44.723861564Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=sqlstore.transactions t=2023-10-09T05:57:44.731251352Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=ngalert.state.manager t=2023-10-09T05:57:44.763447421Z level=info msg="State cache has been initialized" states=0 duration=43.548794ms 2023-10-09T05:57:44.905 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=ticker t=2023-10-09T05:57:44.763522617Z level=info msg=starting first_tick=2023-10-09T05:57:50Z 2023-10-09T05:57:44.906 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 05:57:44 smithi150 conmon[152665]: logger=ngalert.multiorg.alertmanager t=2023-10-09T05:57:44.763512662Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T05:57:45.129 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:44 smithi107 bash[174477]: 3dce8e40d11b0bfb6d71c195cac2d3f32ca6cec4a7de2458ae87f363271d79f7 2023-10-09T05:57:45.265 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:45.265 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T05:57:46.276 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2023-10-09T05:57:43.327138+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384956,"kb_used_data":6652,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576260,"statfs":{"total":767960285184,"available":765518090240,"internally_reserved":0,"allocated":6811648,"data_stored":3875712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002214"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.426860+0000","last_change":"2023-10-09T05:56:58.401504+0000","last_active":"2023-10-09T05:56:59.426860+0000","last_peered":"2023-10-09T05:56:59.426860+0000","last_clean":"2023-10-09T05:56:59.426860+0000","last_became_active":"2023-10-09T05:56:58.401182+0000","last_became_peered":"2023-10-09T05:56:58.401182+0000","last_unstale":"2023-10-09T05:56:59.426860+0000","last_undegraded":"2023-10-09T05:56:59.426860+0000","last_fullsized":"2023-10-09T05:56:59.426860+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.219090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395857+0000","last_change":"2023-10-09T05:56:58.672381+0000","last_active":"2023-10-09T05:56:59.395857+0000","last_peered":"2023-10-09T05:56:59.395857+0000","last_clean":"2023-10-09T05:56:59.395857+0000","last_became_active":"2023-10-09T05:56:58.672020+0000","last_became_peered":"2023-10-09T05:56:58.672020+0000","last_unstale":"2023-10-09T05:56:59.395857+0000","last_undegraded":"2023-10-09T05:56:59.395857+0000","last_fullsized":"2023-10-09T05:56:59.395857+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:25.352710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395428+0000","last_change":"2023-10-09T05:56:58.672203+0000","last_active":"2023-10-09T05:56:59.395428+0000","last_peered":"2023-10-09T05:56:59.395428+0000","last_clean":"2023-10-09T05:56:59.395428+0000","last_became_active":"2023-10-09T05:56:58.671871+0000","last_became_peered":"2023-10-09T05:56:58.671871+0000","last_unstale":"2023-10-09T05:56:59.395428+0000","last_undegraded":"2023-10-09T05:56:59.395428+0000","last_fullsized":"2023-10-09T05:56:59.395428+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:35.636544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395416+0000","last_change":"2023-10-09T05:56:58.401311+0000","last_active":"2023-10-09T05:56:59.395416+0000","last_peered":"2023-10-09T05:56:59.395416+0000","last_clean":"2023-10-09T05:56:59.395416+0000","last_became_active":"2023-10-09T05:56:58.400912+0000","last_became_peered":"2023-10-09T05:56:58.400912+0000","last_unstale":"2023-10-09T05:56:59.395416+0000","last_undegraded":"2023-10-09T05:56:59.395416+0000","last_fullsized":"2023-10-09T05:56:59.395416+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:19.221754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395416+0000","last_change":"2023-10-09T05:56:15.968136+0000","last_active":"2023-10-09T05:56:59.395416+0000","last_peered":"2023-10-09T05:56:59.395416+0000","last_clean":"2023-10-09T05:56:59.395416+0000","last_became_active":"2023-10-09T05:56:15.967691+0000","last_became_peered":"2023-10-09T05:56:15.967691+0000","last_unstale":"2023-10-09T05:56:59.395416+0000","last_undegraded":"2023-10-09T05:56:59.395416+0000","last_fullsized":"2023-10-09T05:56:59.395416+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:30.631087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:57:00.057707+0000","last_change":"2023-10-09T05:56:58.400169+0000","last_active":"2023-10-09T05:57:00.057707+0000","last_peered":"2023-10-09T05:57:00.057707+0000","last_clean":"2023-10-09T05:57:00.057707+0000","last_became_active":"2023-10-09T05:56:58.399784+0000","last_became_peered":"2023-10-09T05:56:58.399784+0000","last_unstale":"2023-10-09T05:57:00.057707+0000","last_undegraded":"2023-10-09T05:57:00.057707+0000","last_fullsized":"2023-10-09T05:57:00.057707+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:07.486985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.396000+0000","last_change":"2023-10-09T05:56:58.751442+0000","last_active":"2023-10-09T05:56:59.396000+0000","last_peered":"2023-10-09T05:56:59.396000+0000","last_clean":"2023-10-09T05:56:59.396000+0000","last_became_active":"2023-10-09T05:56:58.751121+0000","last_became_peered":"2023-10-09T05:56:58.751121+0000","last_unstale":"2023-10-09T05:56:59.396000+0000","last_undegraded":"2023-10-09T05:56:59.396000+0000","last_fullsized":"2023-10-09T05:56:59.396000+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:25.340194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:57:00.057698+0000","last_change":"2023-10-09T05:56:58.401280+0000","last_active":"2023-10-09T05:57:00.057698+0000","last_peered":"2023-10-09T05:57:00.057698+0000","last_clean":"2023-10-09T05:57:00.057698+0000","last_became_active":"2023-10-09T05:56:58.400923+0000","last_became_peered":"2023-10-09T05:56:58.400923+0000","last_unstale":"2023-10-09T05:57:00.057698+0000","last_undegraded":"2023-10-09T05:57:00.057698+0000","last_fullsized":"2023-10-09T05:57:00.057698+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:13.215907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T05:56:59.395955+0000","last_change":"2023-10-09T05:56:58.400200+0000","last_active":"2023-10-09T05:56:59.395955+0000","last_peered":"2023-10-09T05:56:59.395955+0000","last_clean":"2023-10-09T05:56:59.395955+0000","last_became_active":"2023-10-09T05:56:58.399933+0000","last_became_peered":"2023-10-09T05:56:58.399933+0000","last_unstale":"2023-10-09T05:56:59.395955+0000","last_undegraded":"2023-10-09T05:56:59.395955+0000","last_fullsized":"2023-10-09T05:56:59.395955+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:57.074449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53900000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.41799999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.448,"5min":2.448,"15min":2.448},"last":0.432},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.5299999999999998,"5min":2.5299999999999998,"15min":2.5299999999999998},"last":0.373}]},{"osd":4,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.495},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.58599999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298476,"kb_used_data":1188,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446676,"statfs":{"total":95995035648,"available":95689396224,"internally_reserved":0,"allocated":1216512,"data_stored":853003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.625},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53300000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.57699999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.66900000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 05:56:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 05:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.32900000000000001,"15min":0.29899999999999999},"min":{"1min":0.25600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54700000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.311},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.35899999999999999,"15min":0.33400000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.56499999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.26300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40100000000000002,"15min":0.39100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52800000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:56:47 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.42799999999999999,"15min":0.42999999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.59199999999999997,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.372},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47899999999999998,"15min":0.48099999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.29699999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.59799999999999998,"15min":0.60599999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82999999999999996,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.64200000000000002,"15min":0.65000000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82399999999999995,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.61099999999999999,"15min":0.624},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85499999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.84499999999999997,"5min":1.04,"15min":1.04},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.54700000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58399999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.34999999999999998,"15min":0.33600000000000002},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.379,"5min":0.35399999999999998,"15min":0.33600000000000002},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.28399999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:57:25 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.36099999999999999,"15min":0.35399999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.627,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.40500000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.46200000000000002,"15min":0.46600000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.59899999999999998,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.29799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73899999999999999,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.75,"5min":0.879,"15min":0.879},"last":0.66900000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.60499999999999998,"15min":0.61399999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.78200000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.59699999999999998,"15min":0.60599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72199999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.65200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.64100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.68899999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":2.9020000000000001,"5min":2.9020000000000001,"15min":2.9020000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.9390000000000001,"5min":2.9390000000000001,"15min":2.9390000000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:57:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.42299999999999999,"15min":0.39900000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.9319999999999999,"5min":2.9319999999999999,"15min":2.9319999999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46600000000000003,"15min":0.46899999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.375},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.47399999999999998,"15min":0.48399999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52000000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.41399999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:57:06 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.60199999999999998,"15min":0.59899999999999998},"min":{"1min":0.47399999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79000000000000004,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.46400000000000002,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.60899999999999999,"15min":0.59599999999999997},"min":{"1min":0.46200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":2.2989999999999999,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.65500000000000003,"15min":0.64500000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.831,"5min":2.831,"15min":2.831},"last":0.70399999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.66900000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":2.8719999999999999,"5min":2.8719999999999999,"15min":2.8719999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.747,"5min":0.747,"15min":0.747},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":3.0190000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.76400000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.42199999999999999,"15min":0.42999999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.54000000000000004,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.46500000000000002,"15min":0.47199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.374},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.48299999999999998,"15min":0.49399999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 05:56:50 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.496},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.496,"15min":0.505},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.46700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:57:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.58999999999999997,"15min":0.59899999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81599999999999995,"5min":1.077,"15min":1.077},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84399999999999997,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.52400000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.61099999999999999,"15min":0.626},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83599999999999997,"5min":1,"15min":1},"last":0.53600000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.54700000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.45900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.53800000000000003,"15min":0.54800000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.67300000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53900000000000003,"15min":0.54100000000000004},"min":{"1min":0.41999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.74099999999999999,"5min":0.753,"15min":0.753},"last":0.52200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.66800000000000004,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.55800000000000005,"15min":0.56699999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71899999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.66100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56699999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81499999999999995,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68700000000000006,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69999999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.72399999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.80200000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:57:31 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.443,"15min":0.435},"min":{"1min":0.31,"5min":0.245,"15min":0.245},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.75900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.74299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:57:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.872}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297900,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447252,"statfs":{"total":95995035648,"available":95689986048,"internally_reserved":0,"allocated":626688,"data_stored":262900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53800000000000003,"15min":0.54300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.51200000000000001,"15min":0.51400000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.46100000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.56699999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54800000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55800000000000005,"15min":0.55200000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.65600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:57:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.443,"15min":0.43099999999999999},"min":{"1min":0.39600000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.439,"15min":0.42899999999999999},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:56:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.72399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:57:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.77000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:57:46.279 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:57:46.279 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:57:46.279 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:57:46.279 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:57:46.308 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:57:46.308 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 systemd[1]: Stopped Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:46.308 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 systemd[1]: Starting Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:46.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[101102]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[101102]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[118900]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[118900]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:46 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:46 smithi150 ceph-mon[105942]: from='client.24536 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:57:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:46 smithi150 ceph-mon[105942]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:46 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:57:46.967 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:47.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.887Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T05:57:47.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.887Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T05:57:47.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.888Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.107 port=9094 2023-10-09T05:57:47.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.889Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T05:57:47.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 bash[174983]: 2b73cb66110c15baf49df58fd17ccb85aaa3cc99dd69d97df51239f558576a79 2023-10-09T05:57:47.152 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 systemd[1]: Started Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:47.152 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.916Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:57:47.152 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.916Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T05:57:47.152 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.919Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T05:57:47.152 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:46 smithi107 conmon[175101]: ts=2023-10-09T05:57:46.919Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T05:57:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:47 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:47 smithi150 ceph-mon[105942]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:57:48.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:47 smithi150 ceph-mon[105942]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T05:57:48.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:47 smithi150 ceph-mon[105942]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[101102]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:57:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[101102]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T05:57:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[101102]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[118900]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T05:57:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[118900]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T05:57:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:47 smithi107 ceph-mon[118900]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:48.874 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:57:48.874 INFO:teuthology.orchestra.run.smithi107.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:57:48.974 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:48 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:48 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/935320946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:57:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:48 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/935320946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:57:49.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:48 smithi107 conmon[175101]: ts=2023-10-09T05:57:48.889Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000009231s 2023-10-09T05:57:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:48 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/935320946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:57:49.668 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:57:49.668 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T05:57:49.669 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T05:57:49.678 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi107.front.sepia.ceph.com 2023-10-09T05:57:49.679 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T05:57:49.750 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 systemd[1]: Stopping Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:50.056 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:49 smithi150 ceph-mon[105942]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:50.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:57:50.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.750Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.751Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.751Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:57:50.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.751Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:57:50.156 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:49 smithi150 conmon[148851]: ts=2023-10-09T05:57:49.751Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:57:50.251 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:49 smithi107 ceph-mon[101102]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:50.251 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:49 smithi107 ceph-mon[118900]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:50.654 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:50 smithi150 bash[154206]: bcc6466d2bff56865e92382306f2158cef1ae775900c5c82d2549fbf555f73a9 2023-10-09T05:57:50.654 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:50 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T05:57:50.654 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:50 smithi150 systemd[1]: Stopped Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:50.654 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:50 smithi150 systemd[1]: Starting Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T05:57:51.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPING 2023-10-09T05:57:51.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:57:51.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPED 2023-10-09T05:57:51.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTING 2023-10-09T05:57:51.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Serving on http://:::9283 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTED 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPING 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPED 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTING 2023-10-09T05:57:51.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Serving on http://:::9283 2023-10-09T05:57:51.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTED 2023-10-09T05:57:51.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPING 2023-10-09T05:57:51.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:57:51.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STOPPED 2023-10-09T05:57:51.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTING 2023-10-09T05:57:51.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.044Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T05:57:51.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.044Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T05:57:51.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.044Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi150 (none))" 2023-10-09T05:57:51.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.044Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T05:57:51.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.045Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.046Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.046Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 bash[154534]: d2a78d53b2412858e9ddb4c1bad17c69167c69baeb73cc34bc4d9a2f4b6a3540 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.049Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.049Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.050Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.050Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.649µs 2023-10-09T05:57:51.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.050Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.050Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.050Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.051Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.237µs wal_replay_duration=438.082µs wbl_replay_duration=108ns total_replay_duration=476.172µs 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.051Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.051Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.051Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T05:57:51.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 systemd[1]: Started Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T05:57:51.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.067Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.59443ms db_storage=734ns remote_storage=1.203µs web_handler=296ns query_engine=488ns scrape=578.28µs scrape_sd=27.429µs notify=19.592µs notify_sd=5.698µs rules=14.678094ms tracing=6.413µs 2023-10-09T05:57:51.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.067Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T05:57:51.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 05:57:51 smithi150 conmon[154574]: ts=2023-10-09T05:57:51.067Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T05:57:51.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Serving on http://:::9283 2023-10-09T05:57:51.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:51 smithi107 conmon[101524]: [09/Oct/2023:05:57:51] ENGINE Bus STARTED 2023-10-09T05:57:52.464 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.464 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.464 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.465 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[101102]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.466 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.467 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:52.468 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:52 smithi107 ceph-mon[118900]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi107.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T05:57:52.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:52.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:57:52.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:52 smithi150 ceph-mon[105942]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:53.501 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:57:53.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:57:53.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:53.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:53.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:53 smithi107 conmon[101077]: 2023-10-09T05:57:53.252+0000 7f547d741700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='client.24548 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.328 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:57:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: fsmap foo:0 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: Saving service mds.foo spec with placement count:2 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: fsmap foo:0 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: Saving service mds.foo spec with placement count:2 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:54 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T05:57:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: fsmap foo:0 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: Saving service mds.foo spec with placement count:2 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:54 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:54.689 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[101102]: Deploying daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:55 smithi107 ceph-mon[118900]: Deploying daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:55 smithi150 ceph-mon[105942]: Deploying daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T05:57:56.627 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:57:56 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:56] "GET /metrics HTTP/1.1" 200 203277 "" "Prometheus/2.43.0" 2023-10-09T05:57:56.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:56 smithi107 ceph-mon[101102]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:57:56.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:56 smithi107 ceph-mon[101102]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:56.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:56 smithi107 ceph-mon[118900]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:57:56.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:56 smithi107 ceph-mon[118900]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:56 smithi150 ceph-mon[105942]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T05:57:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:56 smithi150 ceph-mon[105942]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:57.150 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 05:57:56 smithi107 conmon[175101]: ts=2023-10-09T05:57:56.892Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003189316s 2023-10-09T05:57:57.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T05:57:58.059 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:57:58.624 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:58 smithi107 ceph-mon[101102]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:58.624 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:58 smithi107 ceph-mon[118900]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:58 smithi150 ceph-mon[105942]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:57:59.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:57:58 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:57:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:57:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: Deploying daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:boot 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: daemon mds.foo.smithi107.dqtsqc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: Cluster is now healthy 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: fsmap foo:0 1 up:standby 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:creating} 2023-10-09T05:57:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:57:59 smithi150 ceph-mon[105942]: daemon mds.foo.smithi107.dqtsqc is now active in filesystem foo as rank 0 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: Deploying daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:boot 2023-10-09T05:58:00.108 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: daemon mds.foo.smithi107.dqtsqc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: Cluster is now healthy 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: fsmap foo:0 1 up:standby 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:creating} 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[101102]: daemon mds.foo.smithi107.dqtsqc is now active in filesystem foo as rank 0 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: Deploying daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:boot 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: daemon mds.foo.smithi107.dqtsqc assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T05:58:00.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: Cluster is now healthy 2023-10-09T05:58:00.111 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: fsmap foo:0 1 up:standby 2023-10-09T05:58:00.111 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:creating} 2023-10-09T05:58:00.111 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:57:59 smithi107 ceph-mon[118900]: daemon mds.foo.smithi107.dqtsqc is now active in filesystem foo as rank 0 2023-10-09T05:58:00.874 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T05:58:00.883 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T05:58:00.883 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T05:58:00.884 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T05:58:00.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:00 smithi150 ceph-mon[105942]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:00.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:00 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:00.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:00 smithi150 ceph-mon[105942]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:00.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:00 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 2023-10-09T05:58:01.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[101102]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[101102]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[118900]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[118900]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:00 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 2023-10-09T05:58:01.231 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:02.618 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.618 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.618 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:boot 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:02.619 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:boot 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:02.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:02 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:boot 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:02 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:03.127 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:03.913 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":19,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6800","nonce":2525813986},{"type":"v1","addr":"172.21.15.107:6801","nonce":2525813986}]},"active_addr":"172.21.15.107:6801/2525813986","active_change":"2023-10-09T05:56:45.306192+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24353,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":3067456200}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":3684818042}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2888496999}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2930305484}]}]} 2023-10-09T05:58:03.918 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T05:58:03.918 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:58:03.919 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:58:04.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:03 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1762592726' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:58:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:03 smithi107 ceph-mon[101102]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-10-09T05:58:04.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:03 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1762592726' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:58:04.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:03 smithi107 ceph-mon[118900]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-10-09T05:58:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:03 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1762592726' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T05:58:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:03 smithi150 ceph-mon[105942]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-10-09T05:58:04.496 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[101102]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[118900]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:05.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:05.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:05.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:04 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:04 smithi150 ceph-mon[105942]: mds.? [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] up:active 2023-10-09T05:58:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:04 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:04 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:04 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:06 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:06] "GET /metrics HTTP/1.1" 200 238188 "" "Prometheus/2.43.0" 2023-10-09T05:58:06.606 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:06.607 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":68,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:57:55.261506+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:06.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:standby 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:06.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.892 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:standby 2023-10-09T05:58:06.892 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:06.892 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:06 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: mds.? [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] up:standby 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T05:58:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:06 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:07.561 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:58:07.562 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:58:07.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:07 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2449899602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:07.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:07 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2449899602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:07.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:07 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2449899602' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:07.940 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:08.706 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:08 smithi107 ceph-mon[101102]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:08.706 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:08 smithi107 ceph-mon[118900]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:08.758 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:08 smithi150 ceph-mon[105942]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:09.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:08 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:10.038 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:10.039 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":68,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T05:57:55.261506+0000","last_up_change":"2023-10-09T05:56:12.451691+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6803","nonce":2908141648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6805","nonce":2908141648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6809","nonce":2908141648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2908141648},{"type":"v1","addr":"172.21.15.107:6807","nonce":2908141648}]},"public_addr":"172.21.15.107:6803/2908141648","cluster_addr":"172.21.15.107:6805/2908141648","heartbeat_back_addr":"172.21.15.107:6809/2908141648","heartbeat_front_addr":"172.21.15.107:6807/2908141648","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6811","nonce":420392147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6813","nonce":420392147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6817","nonce":420392147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":420392147},{"type":"v1","addr":"172.21.15.107:6815","nonce":420392147}]},"public_addr":"172.21.15.107:6811/420392147","cluster_addr":"172.21.15.107:6813/420392147","heartbeat_back_addr":"172.21.15.107:6817/420392147","heartbeat_front_addr":"172.21.15.107:6815/420392147","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6819","nonce":1709812815}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6821","nonce":1709812815}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6825","nonce":1709812815}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":1709812815},{"type":"v1","addr":"172.21.15.107:6823","nonce":1709812815}]},"public_addr":"172.21.15.107:6819/1709812815","cluster_addr":"172.21.15.107:6821/1709812815","heartbeat_back_addr":"172.21.15.107:6825/1709812815","heartbeat_front_addr":"172.21.15.107:6823/1709812815","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6827","nonce":4077493415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6829","nonce":4077493415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6833","nonce":4077493415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":4077493415},{"type":"v1","addr":"172.21.15.107:6831","nonce":4077493415}]},"public_addr":"172.21.15.107:6827/4077493415","cluster_addr":"172.21.15.107:6829/4077493415","heartbeat_back_addr":"172.21.15.107:6833/4077493415","heartbeat_front_addr":"172.21.15.107:6831/4077493415","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[101102]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.256 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:09 smithi107 ceph-mon[118900]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:09 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:09 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:09 smithi150 ceph-mon[105942]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:10.864 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:58:10.864 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:58:10.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:58:10.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:58:10.866 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:58:10.866 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:58:10.866 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:58:10.867 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:58:11.160 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3007459008' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:11.161 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.169 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.169 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.169 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:11.170 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:11.170 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.170 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3007459008' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:11.170 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:11 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3007459008' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:58:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:11 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:11.834 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:12.139 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:12.191 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:12.306 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:12 smithi107 ceph-mon[101102]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:12.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:12 smithi107 ceph-mon[118900]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:12 smithi150 ceph-mon[105942]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T05:58:12.779 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:13.297 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:13.457 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:14.201 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:14.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:13 smithi107 ceph-mon[101102]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:14.244 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:13 smithi107 ceph-mon[118900]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:14.252 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:13 smithi150 ceph-mon[105942]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:15.692 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:15 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:15.692 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:15 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:15.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:15 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:16.307 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:16 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:16] "GET /metrics HTTP/1.1" 200 238772 "" "Prometheus/2.43.0" 2023-10-09T05:58:16.613 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:16 smithi107 ceph-mon[101102]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:16.613 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:16 smithi107 ceph-mon[118900]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:16 smithi150 ceph-mon[105942]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:18.546 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:18 smithi107 ceph-mon[101102]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:18.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:18 smithi107 ceph-mon[118900]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:18.757 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:18 smithi150 ceph-mon[105942]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T05:58:19.087 INFO:teuthology.orchestra.run.smithi107.stdout:55834574904 2023-10-09T05:58:19.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:58:19.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:18 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:19.957 INFO:teuthology.orchestra.run.smithi107.stdout:77309411378 2023-10-09T05:58:19.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:58:20.734 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:20 smithi107 ceph-mon[101102]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:20.735 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:20 smithi107 ceph-mon[118900]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:20 smithi150 ceph-mon[105942]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:22.476 INFO:teuthology.orchestra.run.smithi107.stdout:197568495649 2023-10-09T05:58:22.476 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:58:22.625 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:22 smithi107 ceph-mon[118900]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:22.625 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:22 smithi107 ceph-mon[101102]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:22.652 INFO:teuthology.orchestra.run.smithi107.stdout:128849018921 2023-10-09T05:58:22.653 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:58:22.735 INFO:teuthology.orchestra.run.smithi107.stdout:231928234012 2023-10-09T05:58:22.735 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:58:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:22 smithi150 ceph-mon[105942]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:23.792 INFO:teuthology.orchestra.run.smithi107.stdout:34359738429 2023-10-09T05:58:23.792 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:58:23.859 INFO:teuthology.orchestra.run.smithi107.stdout:163208757286 2023-10-09T05:58:23.859 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:58:24.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:23 smithi107 ceph-mon[101102]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:23 smithi107 ceph-mon[118900]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:23 smithi150 ceph-mon[105942]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:24.433 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:24.469 INFO:teuthology.orchestra.run.smithi107.stdout:107374182446 2023-10-09T05:58:24.469 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:58:24.824 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:24.875 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:24.988 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:24.989 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:25.053 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:25.553 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:25.954 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:26 smithi107 ceph-mon[101102]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:26.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:26 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:26] "GET /metrics HTTP/1.1" 200 238772 "" "Prometheus/2.43.0" 2023-10-09T05:58:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:26 smithi107 ceph-mon[118900]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:26 smithi150 ceph-mon[105942]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:28 smithi107 ceph-mon[101102]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:28.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:28 smithi107 ceph-mon[118900]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:28 smithi150 ceph-mon[105942]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:29.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:28 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:30.468 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:30 smithi107 ceph-mon[101102]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:30.468 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.468 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:30 smithi107 ceph-mon[118900]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:30.469 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:30 smithi150 ceph-mon[105942]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:30.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:30.816 INFO:teuthology.orchestra.run.smithi107.stdout:163208757288 2023-10-09T05:58:31.507 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/641553360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:58:31.507 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1801975171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:58:31.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/641553360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:58:31.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1801975171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:58:31.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/641553360' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:58:31.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1801975171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:58:31.991 INFO:teuthology.orchestra.run.smithi107.stdout:55834574906 2023-10-09T05:58:32.355 INFO:teuthology.orchestra.run.smithi107.stdout:197568495652 2023-10-09T05:58:32.561 INFO:teuthology.orchestra.run.smithi107.stdout:231928234015 2023-10-09T05:58:32.577 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:32 smithi107 ceph-mon[101102]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:32.577 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2679382411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:58:32.577 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:32 smithi107 ceph-mon[118900]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:32.577 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2679382411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:58:32.604 INFO:teuthology.orchestra.run.smithi107.stdout:77309411381 2023-10-09T05:58:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:32 smithi150 ceph-mon[105942]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2679382411' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:58:33.224 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757286 got 163208757288 for osd.5 2023-10-09T05:58:33.224 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:33.742 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:33 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3737994962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:58:33.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:33 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3828071381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:58:33.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:33 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3737994962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:58:33.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:33 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3828071381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:58:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:33 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3737994962' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:58:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:33 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3828071381' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:58:34.363 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234012 got 231928234015 for osd.7 2023-10-09T05:58:34.363 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:34.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:34 smithi107 ceph-mon[101102]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:34.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:34 smithi107 ceph-mon[118900]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:34.683 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574904 got 55834574906 for osd.1 2023-10-09T05:58:34.683 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:34 smithi150 ceph-mon[105942]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:34.924 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495649 got 197568495652 for osd.6 2023-10-09T05:58:34.924 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:35.265 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411378 got 77309411381 for osd.2 2023-10-09T05:58:35.266 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:35.268 INFO:teuthology.orchestra.run.smithi107.stdout:107374182448 2023-10-09T05:58:35.374 INFO:teuthology.orchestra.run.smithi107.stdout:34359738432 2023-10-09T05:58:35.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/51872138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:58:35.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/670853961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:58:35.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:35 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/51872138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:58:35.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:35 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/670853961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:58:35.576 INFO:teuthology.orchestra.run.smithi107.stdout:128849018924 2023-10-09T05:58:35.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:35 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/51872138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:58:35.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:35 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/670853961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:58:36.367 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:36 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:36] "GET /metrics HTTP/1.1" 200 238737 "" "Prometheus/2.43.0" 2023-10-09T05:58:36.589 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182446 got 107374182448 for osd.3 2023-10-09T05:58:36.589 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:36.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:36 smithi107 ceph-mon[101102]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:36 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2391345' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:58:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:36 smithi107 ceph-mon[118900]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:36 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2391345' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:58:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:36 smithi150 ceph-mon[105942]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:36 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2391345' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:58:36.926 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018921 got 128849018924 for osd.4 2023-10-09T05:58:36.926 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:36.956 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738429 got 34359738432 for osd.0 2023-10-09T05:58:36.956 DEBUG:teuthology.parallel:result is None 2023-10-09T05:58:36.956 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T05:58:36.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:37.345 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:38.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:38 smithi107 ceph-mon[101102]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:38.752 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:38 smithi107 ceph-mon[118900]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:38.757 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:38 smithi150 ceph-mon[105942]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:39.144 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:39.146 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T05:58:39.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:38 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:39.881 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2023-10-09T05:58:37.338569+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385532,"kb_used_data":7228,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575684,"statfs":{"total":767960285184,"available":765517500416,"internally_reserved":0,"allocated":7401472,"data_stored":4173302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002617"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569068+0000","last_change":"2023-10-09T05:57:53.535595+0000","last_active":"2023-10-09T05:57:54.569068+0000","last_peered":"2023-10-09T05:57:54.569068+0000","last_clean":"2023-10-09T05:57:54.569068+0000","last_became_active":"2023-10-09T05:57:53.535316+0000","last_became_peered":"2023-10-09T05:57:53.535316+0000","last_unstale":"2023-10-09T05:57:54.569068+0000","last_undegraded":"2023-10-09T05:57:54.569068+0000","last_fullsized":"2023-10-09T05:57:54.569068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:39.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401404+0000","last_change":"2023-10-09T05:57:53.256207+0000","last_active":"2023-10-09T05:57:59.401404+0000","last_peered":"2023-10-09T05:57:59.401404+0000","last_clean":"2023-10-09T05:57:59.401404+0000","last_became_active":"2023-10-09T05:57:53.255333+0000","last_became_peered":"2023-10-09T05:57:53.255333+0000","last_unstale":"2023-10-09T05:57:59.401404+0000","last_undegraded":"2023-10-09T05:57:59.401404+0000","last_fullsized":"2023-10-09T05:57:59.401404+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:31.463784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570133+0000","last_change":"2023-10-09T05:57:53.277868+0000","last_active":"2023-10-09T05:57:54.570133+0000","last_peered":"2023-10-09T05:57:54.570133+0000","last_clean":"2023-10-09T05:57:54.570133+0000","last_became_active":"2023-10-09T05:57:53.277550+0000","last_became_peered":"2023-10-09T05:57:53.277550+0000","last_unstale":"2023-10-09T05:57:54.570133+0000","last_undegraded":"2023-10-09T05:57:54.570133+0000","last_fullsized":"2023-10-09T05:57:54.570133+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:10.542115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833082+0000","last_change":"2023-10-09T05:57:53.253563+0000","last_active":"2023-10-09T05:57:55.833082+0000","last_peered":"2023-10-09T05:57:55.833082+0000","last_clean":"2023-10-09T05:57:55.833082+0000","last_became_active":"2023-10-09T05:57:53.253271+0000","last_became_peered":"2023-10-09T05:57:53.253271+0000","last_unstale":"2023-10-09T05:57:55.833082+0000","last_undegraded":"2023-10-09T05:57:55.833082+0000","last_fullsized":"2023-10-09T05:57:55.833082+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:52.240488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569101+0000","last_change":"2023-10-09T05:57:53.273184+0000","last_active":"2023-10-09T05:57:54.569101+0000","last_peered":"2023-10-09T05:57:54.569101+0000","last_clean":"2023-10-09T05:57:54.569101+0000","last_became_active":"2023-10-09T05:57:53.272901+0000","last_became_peered":"2023-10-09T05:57:53.272901+0000","last_unstale":"2023-10-09T05:57:54.569101+0000","last_undegraded":"2023-10-09T05:57:54.569101+0000","last_fullsized":"2023-10-09T05:57:54.569101+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:04.982281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401902+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T05:57:59.401902+0000","last_peered":"2023-10-09T05:57:59.401902+0000","last_clean":"2023-10-09T05:57:59.401902+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T05:57:59.401902+0000","last_undegraded":"2023-10-09T05:57:59.401902+0000","last_fullsized":"2023-10-09T05:57:59.401902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569378+0000","last_change":"2023-10-09T05:57:53.275894+0000","last_active":"2023-10-09T05:57:54.569378+0000","last_peered":"2023-10-09T05:57:54.569378+0000","last_clean":"2023-10-09T05:57:54.569378+0000","last_became_active":"2023-10-09T05:57:53.275595+0000","last_became_peered":"2023-10-09T05:57:53.275595+0000","last_unstale":"2023-10-09T05:57:54.569378+0000","last_undegraded":"2023-10-09T05:57:54.569378+0000","last_fullsized":"2023-10-09T05:57:54.569378+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:49.277619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401898+0000","last_change":"2023-10-09T05:57:53.261290+0000","last_active":"2023-10-09T05:57:59.401898+0000","last_peered":"2023-10-09T05:57:59.401898+0000","last_clean":"2023-10-09T05:57:59.401898+0000","last_became_active":"2023-10-09T05:57:53.260845+0000","last_became_peered":"2023-10-09T05:57:53.260845+0000","last_unstale":"2023-10-09T05:57:59.401898+0000","last_undegraded":"2023-10-09T05:57:59.401898+0000","last_fullsized":"2023-10-09T05:57:59.401898+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:25.123667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.267472+0000","last_change":"2023-10-09T05:57:53.276553+0000","last_active":"2023-10-09T05:57:54.267472+0000","last_peered":"2023-10-09T05:57:54.267472+0000","last_clean":"2023-10-09T05:57:54.267472+0000","last_became_active":"2023-10-09T05:57:53.276257+0000","last_became_peered":"2023-10-09T05:57:53.276257+0000","last_unstale":"2023-10-09T05:57:54.267472+0000","last_undegraded":"2023-10-09T05:57:54.267472+0000","last_fullsized":"2023-10-09T05:57:54.267472+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:18.708942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.832780+0000","last_change":"2023-10-09T05:57:53.258489+0000","last_active":"2023-10-09T05:57:55.832780+0000","last_peered":"2023-10-09T05:57:55.832780+0000","last_clean":"2023-10-09T05:57:55.832780+0000","last_became_active":"2023-10-09T05:57:53.255996+0000","last_became_peered":"2023-10-09T05:57:53.255996+0000","last_unstale":"2023-10-09T05:57:55.832780+0000","last_undegraded":"2023-10-09T05:57:55.832780+0000","last_fullsized":"2023-10-09T05:57:55.832780+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:41.352315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569211+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T05:57:54.569211+0000","last_peered":"2023-10-09T05:57:54.569211+0000","last_clean":"2023-10-09T05:57:54.569211+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T05:57:54.569211+0000","last_undegraded":"2023-10-09T05:57:54.569211+0000","last_fullsized":"2023-10-09T05:57:54.569211+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833012+0000","last_change":"2023-10-09T05:57:53.259443+0000","last_active":"2023-10-09T05:57:55.833012+0000","last_peered":"2023-10-09T05:57:55.833012+0000","last_clean":"2023-10-09T05:57:55.833012+0000","last_became_active":"2023-10-09T05:57:53.255956+0000","last_became_peered":"2023-10-09T05:57:53.255956+0000","last_unstale":"2023-10-09T05:57:55.833012+0000","last_undegraded":"2023-10-09T05:57:55.833012+0000","last_fullsized":"2023-10-09T05:57:55.833012+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:08.461863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.268101+0000","last_change":"2023-10-09T05:56:58.401504+0000","last_active":"2023-10-09T05:57:54.268101+0000","last_peered":"2023-10-09T05:57:54.268101+0000","last_clean":"2023-10-09T05:57:54.268101+0000","last_became_active":"2023-10-09T05:56:58.401182+0000","last_became_peered":"2023-10-09T05:56:58.401182+0000","last_unstale":"2023-10-09T05:57:54.268101+0000","last_undegraded":"2023-10-09T05:57:54.268101+0000","last_fullsized":"2023-10-09T05:57:54.268101+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.219090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570416+0000","last_change":"2023-10-09T05:57:53.535944+0000","last_active":"2023-10-09T05:57:54.570416+0000","last_peered":"2023-10-09T05:57:54.570416+0000","last_clean":"2023-10-09T05:57:54.570416+0000","last_became_active":"2023-10-09T05:57:53.535679+0000","last_became_peered":"2023-10-09T05:57:53.535679+0000","last_unstale":"2023-10-09T05:57:54.570416+0000","last_undegraded":"2023-10-09T05:57:54.570416+0000","last_fullsized":"2023-10-09T05:57:54.570416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:15.090698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265937+0000","last_change":"2023-10-09T05:56:58.672381+0000","last_active":"2023-10-09T05:57:54.265937+0000","last_peered":"2023-10-09T05:57:54.265937+0000","last_clean":"2023-10-09T05:57:54.265937+0000","last_became_active":"2023-10-09T05:56:58.672020+0000","last_became_peered":"2023-10-09T05:56:58.672020+0000","last_unstale":"2023-10-09T05:57:54.265937+0000","last_undegraded":"2023-10-09T05:57:54.265937+0000","last_fullsized":"2023-10-09T05:57:54.265937+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:25.352710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.399740+0000","last_change":"2023-10-09T05:57:53.258919+0000","last_active":"2023-10-09T05:57:59.399740+0000","last_peered":"2023-10-09T05:57:59.399740+0000","last_clean":"2023-10-09T05:57:59.399740+0000","last_became_active":"2023-10-09T05:57:53.258628+0000","last_became_peered":"2023-10-09T05:57:53.258628+0000","last_unstale":"2023-10-09T05:57:59.399740+0000","last_undegraded":"2023-10-09T05:57:59.399740+0000","last_fullsized":"2023-10-09T05:57:59.399740+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:10.747268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569816+0000","last_change":"2023-10-09T05:57:53.534585+0000","last_active":"2023-10-09T05:57:54.569816+0000","last_peered":"2023-10-09T05:57:54.569816+0000","last_clean":"2023-10-09T05:57:54.569816+0000","last_became_active":"2023-10-09T05:57:53.534293+0000","last_became_peered":"2023-10-09T05:57:53.534293+0000","last_unstale":"2023-10-09T05:57:54.569816+0000","last_undegraded":"2023-10-09T05:57:54.569816+0000","last_fullsized":"2023-10-09T05:57:54.569816+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:08.034576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266308+0000","last_change":"2023-10-09T05:57:53.259929+0000","last_active":"2023-10-09T05:57:54.266308+0000","last_peered":"2023-10-09T05:57:54.266308+0000","last_clean":"2023-10-09T05:57:54.266308+0000","last_became_active":"2023-10-09T05:57:53.259649+0000","last_became_peered":"2023-10-09T05:57:53.259649+0000","last_unstale":"2023-10-09T05:57:54.266308+0000","last_undegraded":"2023-10-09T05:57:54.266308+0000","last_fullsized":"2023-10-09T05:57:54.266308+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.416276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.571142+0000","last_change":"2023-10-09T05:56:58.672203+0000","last_active":"2023-10-09T05:57:54.571142+0000","last_peered":"2023-10-09T05:57:54.571142+0000","last_clean":"2023-10-09T05:57:54.571142+0000","last_became_active":"2023-10-09T05:56:58.671871+0000","last_became_peered":"2023-10-09T05:56:58.671871+0000","last_unstale":"2023-10-09T05:57:54.571142+0000","last_undegraded":"2023-10-09T05:57:54.571142+0000","last_fullsized":"2023-10-09T05:57:54.571142+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:35.636544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569984+0000","last_change":"2023-10-09T05:57:53.277084+0000","last_active":"2023-10-09T05:57:54.569984+0000","last_peered":"2023-10-09T05:57:54.569984+0000","last_clean":"2023-10-09T05:57:54.569984+0000","last_became_active":"2023-10-09T05:57:53.276809+0000","last_became_peered":"2023-10-09T05:57:53.276809+0000","last_unstale":"2023-10-09T05:57:54.569984+0000","last_undegraded":"2023-10-09T05:57:54.569984+0000","last_fullsized":"2023-10-09T05:57:54.569984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:46.204902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569187+0000","last_change":"2023-10-09T05:57:53.254576+0000","last_active":"2023-10-09T05:57:54.569187+0000","last_peered":"2023-10-09T05:57:54.569187+0000","last_clean":"2023-10-09T05:57:54.569187+0000","last_became_active":"2023-10-09T05:57:53.253982+0000","last_became_peered":"2023-10-09T05:57:53.253982+0000","last_unstale":"2023-10-09T05:57:54.569187+0000","last_undegraded":"2023-10-09T05:57:54.569187+0000","last_fullsized":"2023-10-09T05:57:54.569187+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.283921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.568741+0000","last_change":"2023-10-09T05:56:58.401311+0000","last_active":"2023-10-09T05:57:54.568741+0000","last_peered":"2023-10-09T05:57:54.568741+0000","last_clean":"2023-10-09T05:57:54.568741+0000","last_became_active":"2023-10-09T05:56:58.400912+0000","last_became_peered":"2023-10-09T05:56:58.400912+0000","last_unstale":"2023-10-09T05:57:54.568741+0000","last_undegraded":"2023-10-09T05:57:54.568741+0000","last_fullsized":"2023-10-09T05:57:54.568741+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:19.221754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570984+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T05:57:54.570984+0000","last_peered":"2023-10-09T05:57:54.570984+0000","last_clean":"2023-10-09T05:57:54.570984+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T05:57:54.570984+0000","last_undegraded":"2023-10-09T05:57:54.570984+0000","last_fullsized":"2023-10-09T05:57:54.570984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569125+0000","last_change":"2023-10-09T05:57:53.255413+0000","last_active":"2023-10-09T05:57:54.569125+0000","last_peered":"2023-10-09T05:57:54.569125+0000","last_clean":"2023-10-09T05:57:54.569125+0000","last_became_active":"2023-10-09T05:57:53.255082+0000","last_became_peered":"2023-10-09T05:57:53.255082+0000","last_unstale":"2023-10-09T05:57:54.569125+0000","last_undegraded":"2023-10-09T05:57:54.569125+0000","last_fullsized":"2023-10-09T05:57:54.569125+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:13.596333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569885+0000","last_change":"2023-10-09T05:56:15.968136+0000","last_active":"2023-10-09T05:57:54.569885+0000","last_peered":"2023-10-09T05:57:54.569885+0000","last_clean":"2023-10-09T05:57:54.569885+0000","last_became_active":"2023-10-09T05:56:15.967691+0000","last_became_peered":"2023-10-09T05:56:15.967691+0000","last_unstale":"2023-10-09T05:57:54.569885+0000","last_undegraded":"2023-10-09T05:57:54.569885+0000","last_fullsized":"2023-10-09T05:57:54.569885+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:30.631087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265609+0000","last_change":"2023-10-09T05:56:58.400169+0000","last_active":"2023-10-09T05:57:54.265609+0000","last_peered":"2023-10-09T05:57:54.265609+0000","last_clean":"2023-10-09T05:57:54.265609+0000","last_became_active":"2023-10-09T05:56:58.399784+0000","last_became_peered":"2023-10-09T05:56:58.399784+0000","last_unstale":"2023-10-09T05:57:54.265609+0000","last_undegraded":"2023-10-09T05:57:54.265609+0000","last_fullsized":"2023-10-09T05:57:54.265609+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:07.486985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265717+0000","last_change":"2023-10-09T05:57:53.275958+0000","last_active":"2023-10-09T05:57:54.265717+0000","last_peered":"2023-10-09T05:57:54.265717+0000","last_clean":"2023-10-09T05:57:54.265717+0000","last_became_active":"2023-10-09T05:57:53.275687+0000","last_became_peered":"2023-10-09T05:57:53.275687+0000","last_unstale":"2023-10-09T05:57:54.265717+0000","last_undegraded":"2023-10-09T05:57:54.265717+0000","last_fullsized":"2023-10-09T05:57:54.265717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:18.686392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266971+0000","last_change":"2023-10-09T05:57:53.253529+0000","last_active":"2023-10-09T05:57:54.266971+0000","last_peered":"2023-10-09T05:57:54.266971+0000","last_clean":"2023-10-09T05:57:54.266971+0000","last_became_active":"2023-10-09T05:57:53.253221+0000","last_became_peered":"2023-10-09T05:57:53.253221+0000","last_unstale":"2023-10-09T05:57:54.266971+0000","last_undegraded":"2023-10-09T05:57:54.266971+0000","last_fullsized":"2023-10-09T05:57:54.266971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:42.625379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.267208+0000","last_change":"2023-10-09T05:56:58.751442+0000","last_active":"2023-10-09T05:57:54.267208+0000","last_peered":"2023-10-09T05:57:54.267208+0000","last_clean":"2023-10-09T05:57:54.267208+0000","last_became_active":"2023-10-09T05:56:58.751121+0000","last_became_peered":"2023-10-09T05:56:58.751121+0000","last_unstale":"2023-10-09T05:57:54.267208+0000","last_undegraded":"2023-10-09T05:57:54.267208+0000","last_fullsized":"2023-10-09T05:57:54.267208+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:25.340194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266222+0000","last_change":"2023-10-09T05:57:53.275157+0000","last_active":"2023-10-09T05:57:54.266222+0000","last_peered":"2023-10-09T05:57:54.266222+0000","last_clean":"2023-10-09T05:57:54.266222+0000","last_became_active":"2023-10-09T05:57:53.274929+0000","last_became_peered":"2023-10-09T05:57:53.274929+0000","last_unstale":"2023-10-09T05:57:54.266222+0000","last_undegraded":"2023-10-09T05:57:54.266222+0000","last_fullsized":"2023-10-09T05:57:54.266222+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:53.750756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569482+0000","last_change":"2023-10-09T05:57:53.259201+0000","last_active":"2023-10-09T05:57:54.569482+0000","last_peered":"2023-10-09T05:57:54.569482+0000","last_clean":"2023-10-09T05:57:54.569482+0000","last_became_active":"2023-10-09T05:57:53.258914+0000","last_became_peered":"2023-10-09T05:57:53.258914+0000","last_unstale":"2023-10-09T05:57:54.569482+0000","last_undegraded":"2023-10-09T05:57:54.569482+0000","last_fullsized":"2023-10-09T05:57:54.569482+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:37.785954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265752+0000","last_change":"2023-10-09T05:56:58.401280+0000","last_active":"2023-10-09T05:57:54.265752+0000","last_peered":"2023-10-09T05:57:54.265752+0000","last_clean":"2023-10-09T05:57:54.265752+0000","last_became_active":"2023-10-09T05:56:58.400923+0000","last_became_peered":"2023-10-09T05:56:58.400923+0000","last_unstale":"2023-10-09T05:57:54.265752+0000","last_undegraded":"2023-10-09T05:57:54.265752+0000","last_fullsized":"2023-10-09T05:57:54.265752+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:13.215907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265748+0000","last_change":"2023-10-09T05:57:53.275552+0000","last_active":"2023-10-09T05:57:54.265748+0000","last_peered":"2023-10-09T05:57:54.265748+0000","last_clean":"2023-10-09T05:57:54.265748+0000","last_became_active":"2023-10-09T05:57:53.275268+0000","last_became_peered":"2023-10-09T05:57:53.275268+0000","last_unstale":"2023-10-09T05:57:54.265748+0000","last_undegraded":"2023-10-09T05:57:54.265748+0000","last_fullsized":"2023-10-09T05:57:54.265748+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:45.897809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265807+0000","last_change":"2023-10-09T05:57:53.251153+0000","last_active":"2023-10-09T05:57:54.265807+0000","last_peered":"2023-10-09T05:57:54.265807+0000","last_clean":"2023-10-09T05:57:54.265807+0000","last_became_active":"2023-10-09T05:57:53.250509+0000","last_became_peered":"2023-10-09T05:57:53.250509+0000","last_unstale":"2023-10-09T05:57:54.265807+0000","last_undegraded":"2023-10-09T05:57:54.265807+0000","last_fullsized":"2023-10-09T05:57:54.265807+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:42.874815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266157+0000","last_change":"2023-10-09T05:56:58.400200+0000","last_active":"2023-10-09T05:57:54.266157+0000","last_peered":"2023-10-09T05:57:54.266157+0000","last_clean":"2023-10-09T05:57:54.266157+0000","last_became_active":"2023-10-09T05:56:58.399933+0000","last_became_peered":"2023-10-09T05:56:58.399933+0000","last_unstale":"2023-10-09T05:57:54.266157+0000","last_undegraded":"2023-10-09T05:57:54.266157+0000","last_fullsized":"2023-10-09T05:57:54.266157+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:57.074449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265794+0000","last_change":"2023-10-09T05:57:53.275913+0000","last_active":"2023-10-09T05:57:54.265794+0000","last_peered":"2023-10-09T05:57:54.265794+0000","last_clean":"2023-10-09T05:57:54.265794+0000","last_became_active":"2023-10-09T05:57:53.275647+0000","last_became_peered":"2023-10-09T05:57:53.275647+0000","last_unstale":"2023-10-09T05:57:54.265794+0000","last_undegraded":"2023-10-09T05:57:54.265794+0000","last_fullsized":"2023-10-09T05:57:54.265794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:07.751811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570433+0000","last_change":"2023-10-09T05:57:53.251835+0000","last_active":"2023-10-09T05:57:54.570433+0000","last_peered":"2023-10-09T05:57:54.570433+0000","last_clean":"2023-10-09T05:57:54.570433+0000","last_became_active":"2023-10-09T05:57:53.251545+0000","last_became_peered":"2023-10-09T05:57:53.251545+0000","last_unstale":"2023-10-09T05:57:54.570433+0000","last_undegraded":"2023-10-09T05:57:54.570433+0000","last_fullsized":"2023-10-09T05:57:54.570433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:23.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569513+0000","last_change":"2023-10-09T05:57:53.535195+0000","last_active":"2023-10-09T05:57:54.569513+0000","last_peered":"2023-10-09T05:57:54.569513+0000","last_clean":"2023-10-09T05:57:54.569513+0000","last_became_active":"2023-10-09T05:57:53.534885+0000","last_became_peered":"2023-10-09T05:57:53.534885+0000","last_unstale":"2023-10-09T05:57:54.569513+0000","last_undegraded":"2023-10-09T05:57:54.569513+0000","last_fullsized":"2023-10-09T05:57:54.569513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:32.977953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569327+0000","last_change":"2023-10-09T05:57:53.258112+0000","last_active":"2023-10-09T05:57:54.569327+0000","last_peered":"2023-10-09T05:57:54.569327+0000","last_clean":"2023-10-09T05:57:54.569327+0000","last_became_active":"2023-10-09T05:57:53.257843+0000","last_became_peered":"2023-10-09T05:57:53.257843+0000","last_unstale":"2023-10-09T05:57:54.569327+0000","last_undegraded":"2023-10-09T05:57:54.569327+0000","last_fullsized":"2023-10-09T05:57:54.569327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:39.749597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569684+0000","last_change":"2023-10-09T05:57:53.275944+0000","last_active":"2023-10-09T05:57:54.569684+0000","last_peered":"2023-10-09T05:57:54.569684+0000","last_clean":"2023-10-09T05:57:54.569684+0000","last_became_active":"2023-10-09T05:57:53.275647+0000","last_became_peered":"2023-10-09T05:57:53.275647+0000","last_unstale":"2023-10-09T05:57:54.569684+0000","last_undegraded":"2023-10-09T05:57:54.569684+0000","last_fullsized":"2023-10-09T05:57:54.569684+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:28.504169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.268303+0000","last_change":"2023-10-09T05:57:53.259446+0000","last_active":"2023-10-09T05:57:54.268303+0000","last_peered":"2023-10-09T05:57:54.268303+0000","last_clean":"2023-10-09T05:57:54.268303+0000","last_became_active":"2023-10-09T05:57:53.258735+0000","last_became_peered":"2023-10-09T05:57:53.258735+0000","last_unstale":"2023-10-09T05:57:54.268303+0000","last_undegraded":"2023-10-09T05:57:54.268303+0000","last_fullsized":"2023-10-09T05:57:54.268303+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:00.585554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569389+0000","last_change":"2023-10-09T05:57:53.536953+0000","last_active":"2023-10-09T05:57:54.569389+0000","last_peered":"2023-10-09T05:57:54.569389+0000","last_clean":"2023-10-09T05:57:54.569389+0000","last_became_active":"2023-10-09T05:57:53.536677+0000","last_became_peered":"2023-10-09T05:57:53.536677+0000","last_unstale":"2023-10-09T05:57:54.569389+0000","last_undegraded":"2023-10-09T05:57:54.569389+0000","last_fullsized":"2023-10-09T05:57:54.569389+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:10.172027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401810+0000","last_change":"2023-10-09T05:57:53.256569+0000","last_active":"2023-10-09T05:57:59.401810+0000","last_peered":"2023-10-09T05:57:59.401810+0000","last_clean":"2023-10-09T05:57:59.401810+0000","last_became_active":"2023-10-09T05:57:53.256289+0000","last_became_peered":"2023-10-09T05:57:53.256289+0000","last_unstale":"2023-10-09T05:57:59.401810+0000","last_undegraded":"2023-10-09T05:57:59.401810+0000","last_fullsized":"2023-10-09T05:57:59.401810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:50.199721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569698+0000","last_change":"2023-10-09T05:57:53.276885+0000","last_active":"2023-10-09T05:57:54.569698+0000","last_peered":"2023-10-09T05:57:54.569698+0000","last_clean":"2023-10-09T05:57:54.569698+0000","last_became_active":"2023-10-09T05:57:53.276642+0000","last_became_peered":"2023-10-09T05:57:53.276642+0000","last_unstale":"2023-10-09T05:57:54.569698+0000","last_undegraded":"2023-10-09T05:57:54.569698+0000","last_fullsized":"2023-10-09T05:57:54.569698+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:07.393354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265662+0000","last_change":"2023-10-09T05:57:53.253389+0000","last_active":"2023-10-09T05:57:54.265662+0000","last_peered":"2023-10-09T05:57:54.265662+0000","last_clean":"2023-10-09T05:57:54.265662+0000","last_became_active":"2023-10-09T05:57:53.253097+0000","last_became_peered":"2023-10-09T05:57:53.253097+0000","last_unstale":"2023-10-09T05:57:54.265662+0000","last_undegraded":"2023-10-09T05:57:54.265662+0000","last_fullsized":"2023-10-09T05:57:54.265662+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:09.809575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.571590+0000","last_change":"2023-10-09T05:57:54.571375+0000","last_active":"2023-10-09T05:57:54.571590+0000","last_peered":"2023-10-09T05:57:54.571590+0000","last_clean":"2023-10-09T05:57:54.571590+0000","last_became_active":"2023-10-09T05:57:54.571139+0000","last_became_peered":"2023-10-09T05:57:54.571139+0000","last_unstale":"2023-10-09T05:57:54.571590+0000","last_undegraded":"2023-10-09T05:57:54.571590+0000","last_fullsized":"2023-10-09T05:57:54.571590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:43.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.834431+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T05:57:55.834431+0000","last_peered":"2023-10-09T05:57:55.834431+0000","last_clean":"2023-10-09T05:57:55.834431+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T05:57:55.834431+0000","last_undegraded":"2023-10-09T05:57:55.834431+0000","last_fullsized":"2023-10-09T05:57:55.834431+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569165+0000","last_change":"2023-10-09T05:57:53.276035+0000","last_active":"2023-10-09T05:57:54.569165+0000","last_peered":"2023-10-09T05:57:54.569165+0000","last_clean":"2023-10-09T05:57:54.569165+0000","last_became_active":"2023-10-09T05:57:53.275567+0000","last_became_peered":"2023-10-09T05:57:53.275567+0000","last_unstale":"2023-10-09T05:57:54.569165+0000","last_undegraded":"2023-10-09T05:57:54.569165+0000","last_fullsized":"2023-10-09T05:57:54.569165+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:39.924359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.398986+0000","last_change":"2023-10-09T05:57:53.258813+0000","last_active":"2023-10-09T05:57:59.398986+0000","last_peered":"2023-10-09T05:57:59.398986+0000","last_clean":"2023-10-09T05:57:59.398986+0000","last_became_active":"2023-10-09T05:57:53.258532+0000","last_became_peered":"2023-10-09T05:57:53.258532+0000","last_unstale":"2023-10-09T05:57:59.398986+0000","last_undegraded":"2023-10-09T05:57:59.398986+0000","last_fullsized":"2023-10-09T05:57:59.398986+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:21.678747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265670+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T05:57:54.265670+0000","last_peered":"2023-10-09T05:57:54.265670+0000","last_clean":"2023-10-09T05:57:54.265670+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T05:57:54.265670+0000","last_undegraded":"2023-10-09T05:57:54.265670+0000","last_fullsized":"2023-10-09T05:57:54.265670+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.834199+0000","last_change":"2023-10-09T05:57:53.260387+0000","last_active":"2023-10-09T05:57:55.834199+0000","last_peered":"2023-10-09T05:57:55.834199+0000","last_clean":"2023-10-09T05:57:55.834199+0000","last_became_active":"2023-10-09T05:57:53.260079+0000","last_became_peered":"2023-10-09T05:57:53.260079+0000","last_unstale":"2023-10-09T05:57:55.834199+0000","last_undegraded":"2023-10-09T05:57:55.834199+0000","last_fullsized":"2023-10-09T05:57:55.834199+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:34.755237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569046+0000","last_change":"2023-10-09T05:57:53.536600+0000","last_active":"2023-10-09T05:57:54.569046+0000","last_peered":"2023-10-09T05:57:54.569046+0000","last_clean":"2023-10-09T05:57:54.569046+0000","last_became_active":"2023-10-09T05:57:53.535965+0000","last_became_peered":"2023-10-09T05:57:53.535965+0000","last_unstale":"2023-10-09T05:57:54.569046+0000","last_undegraded":"2023-10-09T05:57:54.569046+0000","last_fullsized":"2023-10-09T05:57:54.569046+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:07.970058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401995+0000","last_change":"2023-10-09T05:57:53.255419+0000","last_active":"2023-10-09T05:57:59.401995+0000","last_peered":"2023-10-09T05:57:59.401995+0000","last_clean":"2023-10-09T05:57:59.401995+0000","last_became_active":"2023-10-09T05:57:53.255147+0000","last_became_peered":"2023-10-09T05:57:53.255147+0000","last_unstale":"2023-10-09T05:57:59.401995+0000","last_undegraded":"2023-10-09T05:57:59.401995+0000","last_fullsized":"2023-10-09T05:57:59.401995+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:38.319084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569865+0000","last_change":"2023-10-09T05:57:53.536184+0000","last_active":"2023-10-09T05:57:54.569865+0000","last_peered":"2023-10-09T05:57:54.569865+0000","last_clean":"2023-10-09T05:57:54.569865+0000","last_became_active":"2023-10-09T05:57:53.535856+0000","last_became_peered":"2023-10-09T05:57:53.535856+0000","last_unstale":"2023-10-09T05:57:54.569865+0000","last_undegraded":"2023-10-09T05:57:54.569865+0000","last_fullsized":"2023-10-09T05:57:54.569865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:12.499996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569762+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T05:57:54.569762+0000","last_peered":"2023-10-09T05:57:54.569762+0000","last_clean":"2023-10-09T05:57:54.569762+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T05:57:54.569762+0000","last_undegraded":"2023-10-09T05:57:54.569762+0000","last_fullsized":"2023-10-09T05:57:54.569762+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265701+0000","last_change":"2023-10-09T05:57:53.535736+0000","last_active":"2023-10-09T05:57:54.265701+0000","last_peered":"2023-10-09T05:57:54.265701+0000","last_clean":"2023-10-09T05:57:54.265701+0000","last_became_active":"2023-10-09T05:57:53.535464+0000","last_became_peered":"2023-10-09T05:57:53.535464+0000","last_unstale":"2023-10-09T05:57:54.265701+0000","last_undegraded":"2023-10-09T05:57:54.265701+0000","last_fullsized":"2023-10-09T05:57:54.265701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:03.254052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569345+0000","last_change":"2023-10-09T05:57:53.256437+0000","last_active":"2023-10-09T05:57:54.569345+0000","last_peered":"2023-10-09T05:57:54.569345+0000","last_clean":"2023-10-09T05:57:54.569345+0000","last_became_active":"2023-10-09T05:57:53.255843+0000","last_became_peered":"2023-10-09T05:57:53.255843+0000","last_unstale":"2023-10-09T05:57:54.569345+0000","last_undegraded":"2023-10-09T05:57:54.569345+0000","last_fullsized":"2023-10-09T05:57:54.569345+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:02.382076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569985+0000","last_change":"2023-10-09T05:57:53.535941+0000","last_active":"2023-10-09T05:57:54.569985+0000","last_peered":"2023-10-09T05:57:54.569985+0000","last_clean":"2023-10-09T05:57:54.569985+0000","last_became_active":"2023-10-09T05:57:53.535646+0000","last_became_peered":"2023-10-09T05:57:53.535646+0000","last_unstale":"2023-10-09T05:57:54.569985+0000","last_undegraded":"2023-10-09T05:57:54.569985+0000","last_fullsized":"2023-10-09T05:57:54.569985+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:34.793388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.402288+0000","last_change":"2023-10-09T05:57:53.258209+0000","last_active":"2023-10-09T05:57:59.402288+0000","last_peered":"2023-10-09T05:57:59.402288+0000","last_clean":"2023-10-09T05:57:59.402288+0000","last_became_active":"2023-10-09T05:57:53.257937+0000","last_became_peered":"2023-10-09T05:57:53.257937+0000","last_unstale":"2023-10-09T05:57:59.402288+0000","last_undegraded":"2023-10-09T05:57:59.402288+0000","last_fullsized":"2023-10-09T05:57:59.402288+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:51.684373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569622+0000","last_change":"2023-10-09T05:57:53.535238+0000","last_active":"2023-10-09T05:57:54.569622+0000","last_peered":"2023-10-09T05:57:54.569622+0000","last_clean":"2023-10-09T05:57:54.569622+0000","last_became_active":"2023-10-09T05:57:53.534961+0000","last_became_peered":"2023-10-09T05:57:53.534961+0000","last_unstale":"2023-10-09T05:57:54.569622+0000","last_undegraded":"2023-10-09T05:57:54.569622+0000","last_fullsized":"2023-10-09T05:57:54.569622+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:47.646357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.402476+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T05:57:59.402476+0000","last_peered":"2023-10-09T05:57:59.402476+0000","last_clean":"2023-10-09T05:57:59.402476+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T05:57:59.402476+0000","last_undegraded":"2023-10-09T05:57:59.402476+0000","last_fullsized":"2023-10-09T05:57:59.402476+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265675+0000","last_change":"2023-10-09T05:57:53.535202+0000","last_active":"2023-10-09T05:57:54.265675+0000","last_peered":"2023-10-09T05:57:54.265675+0000","last_clean":"2023-10-09T05:57:54.265675+0000","last_became_active":"2023-10-09T05:57:53.534915+0000","last_became_peered":"2023-10-09T05:57:53.534915+0000","last_unstale":"2023-10-09T05:57:54.265675+0000","last_undegraded":"2023-10-09T05:57:54.265675+0000","last_fullsized":"2023-10-09T05:57:54.265675+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:57.805264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.399929+0000","last_change":"2023-10-09T05:57:53.259949+0000","last_active":"2023-10-09T05:57:59.399929+0000","last_peered":"2023-10-09T05:57:59.399929+0000","last_clean":"2023-10-09T05:57:59.399929+0000","last_became_active":"2023-10-09T05:57:53.259661+0000","last_became_peered":"2023-10-09T05:57:53.259661+0000","last_unstale":"2023-10-09T05:57:59.399929+0000","last_undegraded":"2023-10-09T05:57:59.399929+0000","last_fullsized":"2023-10-09T05:57:59.399929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:30.516054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570205+0000","last_change":"2023-10-09T05:57:53.537605+0000","last_active":"2023-10-09T05:57:54.570205+0000","last_peered":"2023-10-09T05:57:54.570205+0000","last_clean":"2023-10-09T05:57:54.570205+0000","last_became_active":"2023-10-09T05:57:53.536939+0000","last_became_peered":"2023-10-09T05:57:53.536939+0000","last_unstale":"2023-10-09T05:57:54.570205+0000","last_undegraded":"2023-10-09T05:57:54.570205+0000","last_fullsized":"2023-10-09T05:57:54.570205+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:57.551028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265628+0000","last_change":"2023-10-09T05:57:53.261118+0000","last_active":"2023-10-09T05:57:54.265628+0000","last_peered":"2023-10-09T05:57:54.265628+0000","last_clean":"2023-10-09T05:57:54.265628+0000","last_became_active":"2023-10-09T05:57:53.260832+0000","last_became_peered":"2023-10-09T05:57:53.260832+0000","last_unstale":"2023-10-09T05:57:54.265628+0000","last_undegraded":"2023-10-09T05:57:54.265628+0000","last_fullsized":"2023-10-09T05:57:54.265628+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:07.042447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570597+0000","last_change":"2023-10-09T05:57:53.534945+0000","last_active":"2023-10-09T05:57:54.570597+0000","last_peered":"2023-10-09T05:57:54.570597+0000","last_clean":"2023-10-09T05:57:54.570597+0000","last_became_active":"2023-10-09T05:57:53.534670+0000","last_became_peered":"2023-10-09T05:57:53.534670+0000","last_unstale":"2023-10-09T05:57:54.570597+0000","last_undegraded":"2023-10-09T05:57:54.570597+0000","last_fullsized":"2023-10-09T05:57:54.570597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:39.109589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570202+0000","last_change":"2023-10-09T05:57:53.258990+0000","last_active":"2023-10-09T05:57:54.570202+0000","last_peered":"2023-10-09T05:57:54.570202+0000","last_clean":"2023-10-09T05:57:54.570202+0000","last_became_active":"2023-10-09T05:57:53.258630+0000","last_became_peered":"2023-10-09T05:57:53.258630+0000","last_unstale":"2023-10-09T05:57:54.570202+0000","last_undegraded":"2023-10-09T05:57:54.570202+0000","last_fullsized":"2023-10-09T05:57:54.570202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:37.943621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.269714+0000","last_change":"2023-10-09T05:57:54.269425+0000","last_active":"2023-10-09T05:57:54.269714+0000","last_peered":"2023-10-09T05:57:54.269714+0000","last_clean":"2023-10-09T05:57:54.269714+0000","last_became_active":"2023-10-09T05:57:54.269156+0000","last_became_peered":"2023-10-09T05:57:54.269156+0000","last_unstale":"2023-10-09T05:57:54.269714+0000","last_undegraded":"2023-10-09T05:57:54.269714+0000","last_fullsized":"2023-10-09T05:57:54.269714+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:35.243726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570540+0000","last_change":"2023-10-09T05:57:53.255654+0000","last_active":"2023-10-09T05:57:54.570540+0000","last_peered":"2023-10-09T05:57:54.570540+0000","last_clean":"2023-10-09T05:57:54.570540+0000","last_became_active":"2023-10-09T05:57:53.255371+0000","last_became_peered":"2023-10-09T05:57:53.255371+0000","last_unstale":"2023-10-09T05:57:54.570540+0000","last_undegraded":"2023-10-09T05:57:54.570540+0000","last_fullsized":"2023-10-09T05:57:54.570540+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:43.254489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569902+0000","last_change":"2023-10-09T05:57:53.534674+0000","last_active":"2023-10-09T05:57:54.569902+0000","last_peered":"2023-10-09T05:57:54.569902+0000","last_clean":"2023-10-09T05:57:54.569902+0000","last_became_active":"2023-10-09T05:57:53.534397+0000","last_became_peered":"2023-10-09T05:57:53.534397+0000","last_unstale":"2023-10-09T05:57:54.569902+0000","last_undegraded":"2023-10-09T05:57:54.569902+0000","last_fullsized":"2023-10-09T05:57:54.569902+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:52.239670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265568+0000","last_change":"2023-10-09T05:57:53.260151+0000","last_active":"2023-10-09T05:57:54.265568+0000","last_peered":"2023-10-09T05:57:54.265568+0000","last_clean":"2023-10-09T05:57:54.265568+0000","last_became_active":"2023-10-09T05:57:53.259173+0000","last_became_peered":"2023-10-09T05:57:53.259173+0000","last_unstale":"2023-10-09T05:57:54.265568+0000","last_undegraded":"2023-10-09T05:57:54.265568+0000","last_fullsized":"2023-10-09T05:57:54.265568+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:03.531305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569688+0000","last_change":"2023-10-09T05:57:53.278837+0000","last_active":"2023-10-09T05:57:54.569688+0000","last_peered":"2023-10-09T05:57:54.569688+0000","last_clean":"2023-10-09T05:57:54.569688+0000","last_became_active":"2023-10-09T05:57:53.278551+0000","last_became_peered":"2023-10-09T05:57:53.278551+0000","last_unstale":"2023-10-09T05:57:54.569688+0000","last_undegraded":"2023-10-09T05:57:54.569688+0000","last_fullsized":"2023-10-09T05:57:54.569688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:30.406415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833404+0000","last_change":"2023-10-09T05:57:53.255186+0000","last_active":"2023-10-09T05:57:55.833404+0000","last_peered":"2023-10-09T05:57:55.833404+0000","last_clean":"2023-10-09T05:57:55.833404+0000","last_became_active":"2023-10-09T05:57:53.251853+0000","last_became_peered":"2023-10-09T05:57:53.251853+0000","last_unstale":"2023-10-09T05:57:55.833404+0000","last_undegraded":"2023-10-09T05:57:55.833404+0000","last_fullsized":"2023-10-09T05:57:55.833404+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:48.852063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234016,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.50900000000000001,"15min":0.49099999999999999},"min":{"1min":0.373,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.6339999999999999,"5min":2.6339999999999999,"15min":2.6339999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.52000000000000002,"15min":0.52700000000000002},"min":{"1min":0.36799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.51500000000000001,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.439},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.59799999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.52000000000000002,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.5259999999999998,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.61599999999999999,"15min":0.61199999999999999},"min":{"1min":0.36499999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.5579999999999998,"5min":2.5579999999999998,"15min":2.5579999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.36899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":2.5830000000000002,"5min":2.5830000000000002,"15min":2.5830000000000002},"last":0.63600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.748,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.495},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.51300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.51900000000000002,"15min":0.50900000000000001},"min":{"1min":0.32100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.54400000000000004,"15min":0.53700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.83999999999999997,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80100000000000005,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85699999999999998,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.622}]}]},{"osd":6,"up_from":46,"seq":197568495653,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.70499999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.747,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.54300000000000004,"15min":0.54900000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.79800000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81999999999999995,"5min":0.88,"15min":0.88},"last":0.68300000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.53300000000000003,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69199999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55700000000000005,"15min":0.56299999999999994},"min":{"1min":0.34000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81299999999999994,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.73899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.436,"15min":0.439},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63300000000000001,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73399999999999999,"5min":0.753,"15min":0.753},"last":0.627}]},{"osd":5,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65800000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.496,"15min":0.496},"min":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.69899999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.499},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.73499999999999999,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80300000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.72799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574908,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.34100000000000003,"15min":0.30299999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.51200000000000001,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.36899999999999999,"15min":0.33700000000000002},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.501,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.42599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.35299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.59899999999999998,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.437},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.46600000000000003,"15min":0.47699999999999998},"min":{"1min":0.29699999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.58399999999999996,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.40600000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59399999999999997,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74399999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.627},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.628,"15min":0.64600000000000002},"min":{"1min":0.45600000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.79900000000000004,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.61099999999999999,"15min":0.624},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85499999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.629,"15min":0.64200000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.84499999999999997,"5min":1.04,"15min":1.04},"last":0.58299999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59199999999999997,"15min":0.59899999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78400000000000003,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.40400000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.67200000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.58299999999999996,"15min":0.58099999999999996},"min":{"1min":0.41399999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76900000000000002,"5min":0.876,"15min":0.876},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.77600000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.70299999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738433,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.34799999999999998,"15min":0.33500000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.438,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.33100000000000002,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.42199999999999999,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.50800000000000001,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.495,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.32000000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.433,"15min":0.45100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.54700000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.44600000000000001,"15min":0.46100000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57099999999999995,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.55500000000000005,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63300000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.623,"15min":0.60599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.2679999999999998,"5min":4.2679999999999998,"15min":4.2679999999999998},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.58899999999999997,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58399999999999996,"15min":0.60199999999999998},"min":{"1min":0.372,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76800000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.52600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.59199999999999997,"15min":0.59899999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.68700000000000006}]},{"osd":7,"last update":"Mon Oct 9 05:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79100000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.623}]}]},{"osd":2,"up_from":18,"seq":77309411382,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.88200000000000001,"5min":2.9020000000000001,"15min":2.9020000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.373,"5min":0.40600000000000003,"15min":0.39700000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54800000000000004,"5min":2.9390000000000001,"15min":2.9390000000000001},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.41799999999999998,"15min":0.39800000000000002},"min":{"1min":0.23899999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.1839999999999999,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.436,"15min":0.42299999999999999},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.91800000000000004,"5min":2.9319999999999999,"15min":2.9319999999999999},"last":0.312}]},{"osd":3,"last update":"Mon Oct 9 05:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.1309999999999998,"5min":3.1309999999999998,"15min":3.1309999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.077,"5min":3.077,"15min":3.077},"last":0.54900000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.61599999999999999,"15min":0.60399999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.7959999999999998,"5min":2.7959999999999998,"15min":2.7959999999999998},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.627,"15min":0.621},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":2.988,"5min":2.988,"15min":2.988},"last":0.67800000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60899999999999999,"15min":0.59599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.2330000000000001,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.64600000000000002,"15min":0.64200000000000002},"min":{"1min":0.44400000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.995,"5min":2.831,"15min":2.831},"last":0.59799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:57:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.60999999999999999,"15min":0.59399999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.61399999999999999,"15min":0.59999999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"last":0.64900000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.71899999999999997,"15min":0.73699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.96999999999999997,"5min":2.8719999999999999,"15min":2.8719999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.71799999999999997,"15min":0.73699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.88500000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.72399999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.307,"5min":0.39700000000000002,"15min":0.42099999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.55300000000000005,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.41699999999999998,"15min":0.437},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.442,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.439,"15min":0.46300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.46100000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.45200000000000001,"15min":0.48399999999999999},"min":{"1min":0.26100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.46999999999999997,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.46400000000000002,"15min":0.48899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.51300000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.374},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.46700000000000003,"15min":0.495},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.52200000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56999999999999995,"15min":0.59199999999999997},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65900000000000003,"5min":1.077,"15min":1.077},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.57199999999999995,"15min":0.58799999999999997},"min":{"1min":0.35699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.82499999999999996,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:57:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.61099999999999999,"15min":0.626},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68999999999999995,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83599999999999997,"5min":1,"15min":1},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.752,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.60199999999999998,"15min":0.61099999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.70999999999999996,"5min":1.02,"15min":1.02},"last":0.65800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42399999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.44700000000000001,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.67100000000000004,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.70299999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":712704,"data_stored":300974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.52400000000000002,"15min":0.54400000000000004},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.69699999999999995,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.52800000000000002,"15min":0.53700000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.51900000000000002,"15min":0.53100000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.61299999999999999,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55500000000000005,"15min":0.56399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.89400000000000002,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.56000000000000005},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.71899999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.59699999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81299999999999994,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.44800000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.47499999999999998,"15min":0.44600000000000001},"min":{"1min":0.42199999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.79400000000000004,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.436},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.48199999999999998,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.75900000000000001,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.48399999999999999,"15min":0.46800000000000003},"min":{"1min":0.36299999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.53600000000000003,"15min":0.52100000000000002},"min":{"1min":0.36399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.54800000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.56899999999999995,"15min":0.55500000000000005},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.85999999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.48499999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.874,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.57999999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757290,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.54100000000000004},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.745,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67500000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.70399999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.50600000000000001,"15min":0.51300000000000001},"min":{"1min":0.30599999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69899999999999995,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.63200000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67900000000000005,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56999999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78300000000000003,"5min":0.872,"15min":0.872},"last":0.65900000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65700000000000003,"5min":1.522,"15min":1.522},"last":0.748},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56599999999999995,"15min":0.56799999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83099999999999996,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.76800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.45800000000000002,"15min":0.436},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.69499999999999995,"5min":0.752,"15min":0.752},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.46700000000000003,"15min":0.438},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.502,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73799999999999999,"5min":0.754,"15min":0.754},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.50600000000000001,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.72699999999999998,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59699999999999998,"15min":0.59399999999999997},"min":{"1min":0.39300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.79000000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.68799999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:39.887 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:58:40.229 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:40.680 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:40 smithi107 ceph-mon[101102]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:40.680 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:40 smithi107 ceph-mon[101102]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:40.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:40 smithi107 ceph-mon[118900]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:40.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:40 smithi107 ceph-mon[118900]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:40 smithi150 ceph-mon[105942]: from='client.24692 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:40 smithi150 ceph-mon[105942]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:42.124 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T05:58:42.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:42 smithi107 ceph-mon[101102]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:42.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:42 smithi107 ceph-mon[118900]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:42 smithi150 ceph-mon[105942]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:42.923 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":70,"stamp":"2023-10-09T05:58:41.339502+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385532,"kb_used_data":7228,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575684,"statfs":{"total":767960285184,"available":765517500416,"internally_reserved":0,"allocated":7401472,"data_stored":4173302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002737"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569068+0000","last_change":"2023-10-09T05:57:53.535595+0000","last_active":"2023-10-09T05:57:54.569068+0000","last_peered":"2023-10-09T05:57:54.569068+0000","last_clean":"2023-10-09T05:57:54.569068+0000","last_became_active":"2023-10-09T05:57:53.535316+0000","last_became_peered":"2023-10-09T05:57:53.535316+0000","last_unstale":"2023-10-09T05:57:54.569068+0000","last_undegraded":"2023-10-09T05:57:54.569068+0000","last_fullsized":"2023-10-09T05:57:54.569068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:39.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401404+0000","last_change":"2023-10-09T05:57:53.256207+0000","last_active":"2023-10-09T05:57:59.401404+0000","last_peered":"2023-10-09T05:57:59.401404+0000","last_clean":"2023-10-09T05:57:59.401404+0000","last_became_active":"2023-10-09T05:57:53.255333+0000","last_became_peered":"2023-10-09T05:57:53.255333+0000","last_unstale":"2023-10-09T05:57:59.401404+0000","last_undegraded":"2023-10-09T05:57:59.401404+0000","last_fullsized":"2023-10-09T05:57:59.401404+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:31.463784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570133+0000","last_change":"2023-10-09T05:57:53.277868+0000","last_active":"2023-10-09T05:57:54.570133+0000","last_peered":"2023-10-09T05:57:54.570133+0000","last_clean":"2023-10-09T05:57:54.570133+0000","last_became_active":"2023-10-09T05:57:53.277550+0000","last_became_peered":"2023-10-09T05:57:53.277550+0000","last_unstale":"2023-10-09T05:57:54.570133+0000","last_undegraded":"2023-10-09T05:57:54.570133+0000","last_fullsized":"2023-10-09T05:57:54.570133+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:56:10.542115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833082+0000","last_change":"2023-10-09T05:57:53.253563+0000","last_active":"2023-10-09T05:57:55.833082+0000","last_peered":"2023-10-09T05:57:55.833082+0000","last_clean":"2023-10-09T05:57:55.833082+0000","last_became_active":"2023-10-09T05:57:53.253271+0000","last_became_peered":"2023-10-09T05:57:53.253271+0000","last_unstale":"2023-10-09T05:57:55.833082+0000","last_undegraded":"2023-10-09T05:57:55.833082+0000","last_fullsized":"2023-10-09T05:57:55.833082+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:59:52.240488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569101+0000","last_change":"2023-10-09T05:57:53.273184+0000","last_active":"2023-10-09T05:57:54.569101+0000","last_peered":"2023-10-09T05:57:54.569101+0000","last_clean":"2023-10-09T05:57:54.569101+0000","last_became_active":"2023-10-09T05:57:53.272901+0000","last_became_peered":"2023-10-09T05:57:53.272901+0000","last_unstale":"2023-10-09T05:57:54.569101+0000","last_undegraded":"2023-10-09T05:57:54.569101+0000","last_fullsized":"2023-10-09T05:57:54.569101+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:04.982281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401902+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T05:57:59.401902+0000","last_peered":"2023-10-09T05:57:59.401902+0000","last_clean":"2023-10-09T05:57:59.401902+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T05:57:59.401902+0000","last_undegraded":"2023-10-09T05:57:59.401902+0000","last_fullsized":"2023-10-09T05:57:59.401902+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569378+0000","last_change":"2023-10-09T05:57:53.275894+0000","last_active":"2023-10-09T05:57:54.569378+0000","last_peered":"2023-10-09T05:57:54.569378+0000","last_clean":"2023-10-09T05:57:54.569378+0000","last_became_active":"2023-10-09T05:57:53.275595+0000","last_became_peered":"2023-10-09T05:57:53.275595+0000","last_unstale":"2023-10-09T05:57:54.569378+0000","last_undegraded":"2023-10-09T05:57:54.569378+0000","last_fullsized":"2023-10-09T05:57:54.569378+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:49.277619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401898+0000","last_change":"2023-10-09T05:57:53.261290+0000","last_active":"2023-10-09T05:57:59.401898+0000","last_peered":"2023-10-09T05:57:59.401898+0000","last_clean":"2023-10-09T05:57:59.401898+0000","last_became_active":"2023-10-09T05:57:53.260845+0000","last_became_peered":"2023-10-09T05:57:53.260845+0000","last_unstale":"2023-10-09T05:57:59.401898+0000","last_undegraded":"2023-10-09T05:57:59.401898+0000","last_fullsized":"2023-10-09T05:57:59.401898+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:25.123667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.267472+0000","last_change":"2023-10-09T05:57:53.276553+0000","last_active":"2023-10-09T05:57:54.267472+0000","last_peered":"2023-10-09T05:57:54.267472+0000","last_clean":"2023-10-09T05:57:54.267472+0000","last_became_active":"2023-10-09T05:57:53.276257+0000","last_became_peered":"2023-10-09T05:57:53.276257+0000","last_unstale":"2023-10-09T05:57:54.267472+0000","last_undegraded":"2023-10-09T05:57:54.267472+0000","last_fullsized":"2023-10-09T05:57:54.267472+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:18.708942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.832780+0000","last_change":"2023-10-09T05:57:53.258489+0000","last_active":"2023-10-09T05:57:55.832780+0000","last_peered":"2023-10-09T05:57:55.832780+0000","last_clean":"2023-10-09T05:57:55.832780+0000","last_became_active":"2023-10-09T05:57:53.255996+0000","last_became_peered":"2023-10-09T05:57:53.255996+0000","last_unstale":"2023-10-09T05:57:55.832780+0000","last_undegraded":"2023-10-09T05:57:55.832780+0000","last_fullsized":"2023-10-09T05:57:55.832780+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:41.352315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569211+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T05:57:54.569211+0000","last_peered":"2023-10-09T05:57:54.569211+0000","last_clean":"2023-10-09T05:57:54.569211+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T05:57:54.569211+0000","last_undegraded":"2023-10-09T05:57:54.569211+0000","last_fullsized":"2023-10-09T05:57:54.569211+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833012+0000","last_change":"2023-10-09T05:57:53.259443+0000","last_active":"2023-10-09T05:57:55.833012+0000","last_peered":"2023-10-09T05:57:55.833012+0000","last_clean":"2023-10-09T05:57:55.833012+0000","last_became_active":"2023-10-09T05:57:53.255956+0000","last_became_peered":"2023-10-09T05:57:53.255956+0000","last_unstale":"2023-10-09T05:57:55.833012+0000","last_undegraded":"2023-10-09T05:57:55.833012+0000","last_fullsized":"2023-10-09T05:57:55.833012+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:08.461863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.268101+0000","last_change":"2023-10-09T05:56:58.401504+0000","last_active":"2023-10-09T05:57:54.268101+0000","last_peered":"2023-10-09T05:57:54.268101+0000","last_clean":"2023-10-09T05:57:54.268101+0000","last_became_active":"2023-10-09T05:56:58.401182+0000","last_became_peered":"2023-10-09T05:56:58.401182+0000","last_unstale":"2023-10-09T05:57:54.268101+0000","last_undegraded":"2023-10-09T05:57:54.268101+0000","last_fullsized":"2023-10-09T05:57:54.268101+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:08.219090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570416+0000","last_change":"2023-10-09T05:57:53.535944+0000","last_active":"2023-10-09T05:57:54.570416+0000","last_peered":"2023-10-09T05:57:54.570416+0000","last_clean":"2023-10-09T05:57:54.570416+0000","last_became_active":"2023-10-09T05:57:53.535679+0000","last_became_peered":"2023-10-09T05:57:53.535679+0000","last_unstale":"2023-10-09T05:57:54.570416+0000","last_undegraded":"2023-10-09T05:57:54.570416+0000","last_fullsized":"2023-10-09T05:57:54.570416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:15.090698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265937+0000","last_change":"2023-10-09T05:56:58.672381+0000","last_active":"2023-10-09T05:57:54.265937+0000","last_peered":"2023-10-09T05:57:54.265937+0000","last_clean":"2023-10-09T05:57:54.265937+0000","last_became_active":"2023-10-09T05:56:58.672020+0000","last_became_peered":"2023-10-09T05:56:58.672020+0000","last_unstale":"2023-10-09T05:57:54.265937+0000","last_undegraded":"2023-10-09T05:57:54.265937+0000","last_fullsized":"2023-10-09T05:57:54.265937+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:25.352710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.399740+0000","last_change":"2023-10-09T05:57:53.258919+0000","last_active":"2023-10-09T05:57:59.399740+0000","last_peered":"2023-10-09T05:57:59.399740+0000","last_clean":"2023-10-09T05:57:59.399740+0000","last_became_active":"2023-10-09T05:57:53.258628+0000","last_became_peered":"2023-10-09T05:57:53.258628+0000","last_unstale":"2023-10-09T05:57:59.399740+0000","last_undegraded":"2023-10-09T05:57:59.399740+0000","last_fullsized":"2023-10-09T05:57:59.399740+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:10.747268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569816+0000","last_change":"2023-10-09T05:57:53.534585+0000","last_active":"2023-10-09T05:57:54.569816+0000","last_peered":"2023-10-09T05:57:54.569816+0000","last_clean":"2023-10-09T05:57:54.569816+0000","last_became_active":"2023-10-09T05:57:53.534293+0000","last_became_peered":"2023-10-09T05:57:53.534293+0000","last_unstale":"2023-10-09T05:57:54.569816+0000","last_undegraded":"2023-10-09T05:57:54.569816+0000","last_fullsized":"2023-10-09T05:57:54.569816+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:54:08.034576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266308+0000","last_change":"2023-10-09T05:57:53.259929+0000","last_active":"2023-10-09T05:57:54.266308+0000","last_peered":"2023-10-09T05:57:54.266308+0000","last_clean":"2023-10-09T05:57:54.266308+0000","last_became_active":"2023-10-09T05:57:53.259649+0000","last_became_peered":"2023-10-09T05:57:53.259649+0000","last_unstale":"2023-10-09T05:57:54.266308+0000","last_undegraded":"2023-10-09T05:57:54.266308+0000","last_fullsized":"2023-10-09T05:57:54.266308+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:04.416276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.571142+0000","last_change":"2023-10-09T05:56:58.672203+0000","last_active":"2023-10-09T05:57:54.571142+0000","last_peered":"2023-10-09T05:57:54.571142+0000","last_clean":"2023-10-09T05:57:54.571142+0000","last_became_active":"2023-10-09T05:56:58.671871+0000","last_became_peered":"2023-10-09T05:56:58.671871+0000","last_unstale":"2023-10-09T05:57:54.571142+0000","last_undegraded":"2023-10-09T05:57:54.571142+0000","last_fullsized":"2023-10-09T05:57:54.571142+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:35.636544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569984+0000","last_change":"2023-10-09T05:57:53.277084+0000","last_active":"2023-10-09T05:57:54.569984+0000","last_peered":"2023-10-09T05:57:54.569984+0000","last_clean":"2023-10-09T05:57:54.569984+0000","last_became_active":"2023-10-09T05:57:53.276809+0000","last_became_peered":"2023-10-09T05:57:53.276809+0000","last_unstale":"2023-10-09T05:57:54.569984+0000","last_undegraded":"2023-10-09T05:57:54.569984+0000","last_fullsized":"2023-10-09T05:57:54.569984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:46.204902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569187+0000","last_change":"2023-10-09T05:57:53.254576+0000","last_active":"2023-10-09T05:57:54.569187+0000","last_peered":"2023-10-09T05:57:54.569187+0000","last_clean":"2023-10-09T05:57:54.569187+0000","last_became_active":"2023-10-09T05:57:53.253982+0000","last_became_peered":"2023-10-09T05:57:53.253982+0000","last_unstale":"2023-10-09T05:57:54.569187+0000","last_undegraded":"2023-10-09T05:57:54.569187+0000","last_fullsized":"2023-10-09T05:57:54.569187+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.283921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.568741+0000","last_change":"2023-10-09T05:56:58.401311+0000","last_active":"2023-10-09T05:57:54.568741+0000","last_peered":"2023-10-09T05:57:54.568741+0000","last_clean":"2023-10-09T05:57:54.568741+0000","last_became_active":"2023-10-09T05:56:58.400912+0000","last_became_peered":"2023-10-09T05:56:58.400912+0000","last_unstale":"2023-10-09T05:57:54.568741+0000","last_undegraded":"2023-10-09T05:57:54.568741+0000","last_fullsized":"2023-10-09T05:57:54.568741+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:19.221754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570984+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T05:57:54.570984+0000","last_peered":"2023-10-09T05:57:54.570984+0000","last_clean":"2023-10-09T05:57:54.570984+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T05:57:54.570984+0000","last_undegraded":"2023-10-09T05:57:54.570984+0000","last_fullsized":"2023-10-09T05:57:54.570984+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569125+0000","last_change":"2023-10-09T05:57:53.255413+0000","last_active":"2023-10-09T05:57:54.569125+0000","last_peered":"2023-10-09T05:57:54.569125+0000","last_clean":"2023-10-09T05:57:54.569125+0000","last_became_active":"2023-10-09T05:57:53.255082+0000","last_became_peered":"2023-10-09T05:57:53.255082+0000","last_unstale":"2023-10-09T05:57:54.569125+0000","last_undegraded":"2023-10-09T05:57:54.569125+0000","last_fullsized":"2023-10-09T05:57:54.569125+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:13.596333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569885+0000","last_change":"2023-10-09T05:56:15.968136+0000","last_active":"2023-10-09T05:57:54.569885+0000","last_peered":"2023-10-09T05:57:54.569885+0000","last_clean":"2023-10-09T05:57:54.569885+0000","last_became_active":"2023-10-09T05:56:15.967691+0000","last_became_peered":"2023-10-09T05:56:15.967691+0000","last_unstale":"2023-10-09T05:57:54.569885+0000","last_undegraded":"2023-10-09T05:57:54.569885+0000","last_fullsized":"2023-10-09T05:57:54.569885+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:30.631087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265609+0000","last_change":"2023-10-09T05:56:58.400169+0000","last_active":"2023-10-09T05:57:54.265609+0000","last_peered":"2023-10-09T05:57:54.265609+0000","last_clean":"2023-10-09T05:57:54.265609+0000","last_became_active":"2023-10-09T05:56:58.399784+0000","last_became_peered":"2023-10-09T05:56:58.399784+0000","last_unstale":"2023-10-09T05:57:54.265609+0000","last_undegraded":"2023-10-09T05:57:54.265609+0000","last_fullsized":"2023-10-09T05:57:54.265609+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:07.486985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265717+0000","last_change":"2023-10-09T05:57:53.275958+0000","last_active":"2023-10-09T05:57:54.265717+0000","last_peered":"2023-10-09T05:57:54.265717+0000","last_clean":"2023-10-09T05:57:54.265717+0000","last_became_active":"2023-10-09T05:57:53.275687+0000","last_became_peered":"2023-10-09T05:57:53.275687+0000","last_unstale":"2023-10-09T05:57:54.265717+0000","last_undegraded":"2023-10-09T05:57:54.265717+0000","last_fullsized":"2023-10-09T05:57:54.265717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:52:18.686392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266971+0000","last_change":"2023-10-09T05:57:53.253529+0000","last_active":"2023-10-09T05:57:54.266971+0000","last_peered":"2023-10-09T05:57:54.266971+0000","last_clean":"2023-10-09T05:57:54.266971+0000","last_became_active":"2023-10-09T05:57:53.253221+0000","last_became_peered":"2023-10-09T05:57:53.253221+0000","last_unstale":"2023-10-09T05:57:54.266971+0000","last_undegraded":"2023-10-09T05:57:54.266971+0000","last_fullsized":"2023-10-09T05:57:54.266971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:42.625379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.267208+0000","last_change":"2023-10-09T05:56:58.751442+0000","last_active":"2023-10-09T05:57:54.267208+0000","last_peered":"2023-10-09T05:57:54.267208+0000","last_clean":"2023-10-09T05:57:54.267208+0000","last_became_active":"2023-10-09T05:56:58.751121+0000","last_became_peered":"2023-10-09T05:56:58.751121+0000","last_unstale":"2023-10-09T05:57:54.267208+0000","last_undegraded":"2023-10-09T05:57:54.267208+0000","last_fullsized":"2023-10-09T05:57:54.267208+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:25.340194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266222+0000","last_change":"2023-10-09T05:57:53.275157+0000","last_active":"2023-10-09T05:57:54.266222+0000","last_peered":"2023-10-09T05:57:54.266222+0000","last_clean":"2023-10-09T05:57:54.266222+0000","last_became_active":"2023-10-09T05:57:53.274929+0000","last_became_peered":"2023-10-09T05:57:53.274929+0000","last_unstale":"2023-10-09T05:57:54.266222+0000","last_undegraded":"2023-10-09T05:57:54.266222+0000","last_fullsized":"2023-10-09T05:57:54.266222+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:53.750756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569482+0000","last_change":"2023-10-09T05:57:53.259201+0000","last_active":"2023-10-09T05:57:54.569482+0000","last_peered":"2023-10-09T05:57:54.569482+0000","last_clean":"2023-10-09T05:57:54.569482+0000","last_became_active":"2023-10-09T05:57:53.258914+0000","last_became_peered":"2023-10-09T05:57:53.258914+0000","last_unstale":"2023-10-09T05:57:54.569482+0000","last_undegraded":"2023-10-09T05:57:54.569482+0000","last_fullsized":"2023-10-09T05:57:54.569482+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:59:37.785954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265752+0000","last_change":"2023-10-09T05:56:58.401280+0000","last_active":"2023-10-09T05:57:54.265752+0000","last_peered":"2023-10-09T05:57:54.265752+0000","last_clean":"2023-10-09T05:57:54.265752+0000","last_became_active":"2023-10-09T05:56:58.400923+0000","last_became_peered":"2023-10-09T05:56:58.400923+0000","last_unstale":"2023-10-09T05:57:54.265752+0000","last_undegraded":"2023-10-09T05:57:54.265752+0000","last_fullsized":"2023-10-09T05:57:54.265752+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:13.215907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265748+0000","last_change":"2023-10-09T05:57:53.275552+0000","last_active":"2023-10-09T05:57:54.265748+0000","last_peered":"2023-10-09T05:57:54.265748+0000","last_clean":"2023-10-09T05:57:54.265748+0000","last_became_active":"2023-10-09T05:57:53.275268+0000","last_became_peered":"2023-10-09T05:57:53.275268+0000","last_unstale":"2023-10-09T05:57:54.265748+0000","last_undegraded":"2023-10-09T05:57:54.265748+0000","last_fullsized":"2023-10-09T05:57:54.265748+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:45.897809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265807+0000","last_change":"2023-10-09T05:57:53.251153+0000","last_active":"2023-10-09T05:57:54.265807+0000","last_peered":"2023-10-09T05:57:54.265807+0000","last_clean":"2023-10-09T05:57:54.265807+0000","last_became_active":"2023-10-09T05:57:53.250509+0000","last_became_peered":"2023-10-09T05:57:53.250509+0000","last_unstale":"2023-10-09T05:57:54.265807+0000","last_undegraded":"2023-10-09T05:57:54.265807+0000","last_fullsized":"2023-10-09T05:57:54.265807+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:42.874815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.266157+0000","last_change":"2023-10-09T05:56:58.400200+0000","last_active":"2023-10-09T05:57:54.266157+0000","last_peered":"2023-10-09T05:57:54.266157+0000","last_clean":"2023-10-09T05:57:54.266157+0000","last_became_active":"2023-10-09T05:56:58.399933+0000","last_became_peered":"2023-10-09T05:56:58.399933+0000","last_unstale":"2023-10-09T05:57:54.266157+0000","last_undegraded":"2023-10-09T05:57:54.266157+0000","last_fullsized":"2023-10-09T05:57:54.266157+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:57.074449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265794+0000","last_change":"2023-10-09T05:57:53.275913+0000","last_active":"2023-10-09T05:57:54.265794+0000","last_peered":"2023-10-09T05:57:54.265794+0000","last_clean":"2023-10-09T05:57:54.265794+0000","last_became_active":"2023-10-09T05:57:53.275647+0000","last_became_peered":"2023-10-09T05:57:53.275647+0000","last_unstale":"2023-10-09T05:57:54.265794+0000","last_undegraded":"2023-10-09T05:57:54.265794+0000","last_fullsized":"2023-10-09T05:57:54.265794+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:07.751811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570433+0000","last_change":"2023-10-09T05:57:53.251835+0000","last_active":"2023-10-09T05:57:54.570433+0000","last_peered":"2023-10-09T05:57:54.570433+0000","last_clean":"2023-10-09T05:57:54.570433+0000","last_became_active":"2023-10-09T05:57:53.251545+0000","last_became_peered":"2023-10-09T05:57:53.251545+0000","last_unstale":"2023-10-09T05:57:54.570433+0000","last_undegraded":"2023-10-09T05:57:54.570433+0000","last_fullsized":"2023-10-09T05:57:54.570433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:23.375252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569513+0000","last_change":"2023-10-09T05:57:53.535195+0000","last_active":"2023-10-09T05:57:54.569513+0000","last_peered":"2023-10-09T05:57:54.569513+0000","last_clean":"2023-10-09T05:57:54.569513+0000","last_became_active":"2023-10-09T05:57:53.534885+0000","last_became_peered":"2023-10-09T05:57:53.534885+0000","last_unstale":"2023-10-09T05:57:54.569513+0000","last_undegraded":"2023-10-09T05:57:54.569513+0000","last_fullsized":"2023-10-09T05:57:54.569513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:32.977953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569327+0000","last_change":"2023-10-09T05:57:53.258112+0000","last_active":"2023-10-09T05:57:54.569327+0000","last_peered":"2023-10-09T05:57:54.569327+0000","last_clean":"2023-10-09T05:57:54.569327+0000","last_became_active":"2023-10-09T05:57:53.257843+0000","last_became_peered":"2023-10-09T05:57:53.257843+0000","last_unstale":"2023-10-09T05:57:54.569327+0000","last_undegraded":"2023-10-09T05:57:54.569327+0000","last_fullsized":"2023-10-09T05:57:54.569327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:39.749597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569684+0000","last_change":"2023-10-09T05:57:53.275944+0000","last_active":"2023-10-09T05:57:54.569684+0000","last_peered":"2023-10-09T05:57:54.569684+0000","last_clean":"2023-10-09T05:57:54.569684+0000","last_became_active":"2023-10-09T05:57:53.275647+0000","last_became_peered":"2023-10-09T05:57:53.275647+0000","last_unstale":"2023-10-09T05:57:54.569684+0000","last_undegraded":"2023-10-09T05:57:54.569684+0000","last_fullsized":"2023-10-09T05:57:54.569684+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:28.504169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.268303+0000","last_change":"2023-10-09T05:57:53.259446+0000","last_active":"2023-10-09T05:57:54.268303+0000","last_peered":"2023-10-09T05:57:54.268303+0000","last_clean":"2023-10-09T05:57:54.268303+0000","last_became_active":"2023-10-09T05:57:53.258735+0000","last_became_peered":"2023-10-09T05:57:53.258735+0000","last_unstale":"2023-10-09T05:57:54.268303+0000","last_undegraded":"2023-10-09T05:57:54.268303+0000","last_fullsized":"2023-10-09T05:57:54.268303+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:00.585554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569389+0000","last_change":"2023-10-09T05:57:53.536953+0000","last_active":"2023-10-09T05:57:54.569389+0000","last_peered":"2023-10-09T05:57:54.569389+0000","last_clean":"2023-10-09T05:57:54.569389+0000","last_became_active":"2023-10-09T05:57:53.536677+0000","last_became_peered":"2023-10-09T05:57:53.536677+0000","last_unstale":"2023-10-09T05:57:54.569389+0000","last_undegraded":"2023-10-09T05:57:54.569389+0000","last_fullsized":"2023-10-09T05:57:54.569389+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:10.172027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401810+0000","last_change":"2023-10-09T05:57:53.256569+0000","last_active":"2023-10-09T05:57:59.401810+0000","last_peered":"2023-10-09T05:57:59.401810+0000","last_clean":"2023-10-09T05:57:59.401810+0000","last_became_active":"2023-10-09T05:57:53.256289+0000","last_became_peered":"2023-10-09T05:57:53.256289+0000","last_unstale":"2023-10-09T05:57:59.401810+0000","last_undegraded":"2023-10-09T05:57:59.401810+0000","last_fullsized":"2023-10-09T05:57:59.401810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:49:50.199721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569698+0000","last_change":"2023-10-09T05:57:53.276885+0000","last_active":"2023-10-09T05:57:54.569698+0000","last_peered":"2023-10-09T05:57:54.569698+0000","last_clean":"2023-10-09T05:57:54.569698+0000","last_became_active":"2023-10-09T05:57:53.276642+0000","last_became_peered":"2023-10-09T05:57:53.276642+0000","last_unstale":"2023-10-09T05:57:54.569698+0000","last_undegraded":"2023-10-09T05:57:54.569698+0000","last_fullsized":"2023-10-09T05:57:54.569698+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:07.393354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265662+0000","last_change":"2023-10-09T05:57:53.253389+0000","last_active":"2023-10-09T05:57:54.265662+0000","last_peered":"2023-10-09T05:57:54.265662+0000","last_clean":"2023-10-09T05:57:54.265662+0000","last_became_active":"2023-10-09T05:57:53.253097+0000","last_became_peered":"2023-10-09T05:57:53.253097+0000","last_unstale":"2023-10-09T05:57:54.265662+0000","last_undegraded":"2023-10-09T05:57:54.265662+0000","last_fullsized":"2023-10-09T05:57:54.265662+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:09.809575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.571590+0000","last_change":"2023-10-09T05:57:54.571375+0000","last_active":"2023-10-09T05:57:54.571590+0000","last_peered":"2023-10-09T05:57:54.571590+0000","last_clean":"2023-10-09T05:57:54.571590+0000","last_became_active":"2023-10-09T05:57:54.571139+0000","last_became_peered":"2023-10-09T05:57:54.571139+0000","last_unstale":"2023-10-09T05:57:54.571590+0000","last_undegraded":"2023-10-09T05:57:54.571590+0000","last_fullsized":"2023-10-09T05:57:54.571590+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:43.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.834431+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T05:57:55.834431+0000","last_peered":"2023-10-09T05:57:55.834431+0000","last_clean":"2023-10-09T05:57:55.834431+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T05:57:55.834431+0000","last_undegraded":"2023-10-09T05:57:55.834431+0000","last_fullsized":"2023-10-09T05:57:55.834431+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569165+0000","last_change":"2023-10-09T05:57:53.276035+0000","last_active":"2023-10-09T05:57:54.569165+0000","last_peered":"2023-10-09T05:57:54.569165+0000","last_clean":"2023-10-09T05:57:54.569165+0000","last_became_active":"2023-10-09T05:57:53.275567+0000","last_became_peered":"2023-10-09T05:57:53.275567+0000","last_unstale":"2023-10-09T05:57:54.569165+0000","last_undegraded":"2023-10-09T05:57:54.569165+0000","last_fullsized":"2023-10-09T05:57:54.569165+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:39.924359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.398986+0000","last_change":"2023-10-09T05:57:53.258813+0000","last_active":"2023-10-09T05:57:59.398986+0000","last_peered":"2023-10-09T05:57:59.398986+0000","last_clean":"2023-10-09T05:57:59.398986+0000","last_became_active":"2023-10-09T05:57:53.258532+0000","last_became_peered":"2023-10-09T05:57:53.258532+0000","last_unstale":"2023-10-09T05:57:59.398986+0000","last_undegraded":"2023-10-09T05:57:59.398986+0000","last_fullsized":"2023-10-09T05:57:59.398986+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:21.678747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265670+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T05:57:54.265670+0000","last_peered":"2023-10-09T05:57:54.265670+0000","last_clean":"2023-10-09T05:57:54.265670+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T05:57:54.265670+0000","last_undegraded":"2023-10-09T05:57:54.265670+0000","last_fullsized":"2023-10-09T05:57:54.265670+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.834199+0000","last_change":"2023-10-09T05:57:53.260387+0000","last_active":"2023-10-09T05:57:55.834199+0000","last_peered":"2023-10-09T05:57:55.834199+0000","last_clean":"2023-10-09T05:57:55.834199+0000","last_became_active":"2023-10-09T05:57:53.260079+0000","last_became_peered":"2023-10-09T05:57:53.260079+0000","last_unstale":"2023-10-09T05:57:55.834199+0000","last_undegraded":"2023-10-09T05:57:55.834199+0000","last_fullsized":"2023-10-09T05:57:55.834199+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:34.755237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569046+0000","last_change":"2023-10-09T05:57:53.536600+0000","last_active":"2023-10-09T05:57:54.569046+0000","last_peered":"2023-10-09T05:57:54.569046+0000","last_clean":"2023-10-09T05:57:54.569046+0000","last_became_active":"2023-10-09T05:57:53.535965+0000","last_became_peered":"2023-10-09T05:57:53.535965+0000","last_unstale":"2023-10-09T05:57:54.569046+0000","last_undegraded":"2023-10-09T05:57:54.569046+0000","last_fullsized":"2023-10-09T05:57:54.569046+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:49:07.970058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.401995+0000","last_change":"2023-10-09T05:57:53.255419+0000","last_active":"2023-10-09T05:57:59.401995+0000","last_peered":"2023-10-09T05:57:59.401995+0000","last_clean":"2023-10-09T05:57:59.401995+0000","last_became_active":"2023-10-09T05:57:53.255147+0000","last_became_peered":"2023-10-09T05:57:53.255147+0000","last_unstale":"2023-10-09T05:57:59.401995+0000","last_undegraded":"2023-10-09T05:57:59.401995+0000","last_fullsized":"2023-10-09T05:57:59.401995+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:38.319084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569865+0000","last_change":"2023-10-09T05:57:53.536184+0000","last_active":"2023-10-09T05:57:54.569865+0000","last_peered":"2023-10-09T05:57:54.569865+0000","last_clean":"2023-10-09T05:57:54.569865+0000","last_became_active":"2023-10-09T05:57:53.535856+0000","last_became_peered":"2023-10-09T05:57:53.535856+0000","last_unstale":"2023-10-09T05:57:54.569865+0000","last_undegraded":"2023-10-09T05:57:54.569865+0000","last_fullsized":"2023-10-09T05:57:54.569865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:12.499996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569762+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T05:57:54.569762+0000","last_peered":"2023-10-09T05:57:54.569762+0000","last_clean":"2023-10-09T05:57:54.569762+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T05:57:54.569762+0000","last_undegraded":"2023-10-09T05:57:54.569762+0000","last_fullsized":"2023-10-09T05:57:54.569762+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265701+0000","last_change":"2023-10-09T05:57:53.535736+0000","last_active":"2023-10-09T05:57:54.265701+0000","last_peered":"2023-10-09T05:57:54.265701+0000","last_clean":"2023-10-09T05:57:54.265701+0000","last_became_active":"2023-10-09T05:57:53.535464+0000","last_became_peered":"2023-10-09T05:57:53.535464+0000","last_unstale":"2023-10-09T05:57:54.265701+0000","last_undegraded":"2023-10-09T05:57:54.265701+0000","last_fullsized":"2023-10-09T05:57:54.265701+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:03.254052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569345+0000","last_change":"2023-10-09T05:57:53.256437+0000","last_active":"2023-10-09T05:57:54.569345+0000","last_peered":"2023-10-09T05:57:54.569345+0000","last_clean":"2023-10-09T05:57:54.569345+0000","last_became_active":"2023-10-09T05:57:53.255843+0000","last_became_peered":"2023-10-09T05:57:53.255843+0000","last_unstale":"2023-10-09T05:57:54.569345+0000","last_undegraded":"2023-10-09T05:57:54.569345+0000","last_fullsized":"2023-10-09T05:57:54.569345+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:02.382076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569985+0000","last_change":"2023-10-09T05:57:53.535941+0000","last_active":"2023-10-09T05:57:54.569985+0000","last_peered":"2023-10-09T05:57:54.569985+0000","last_clean":"2023-10-09T05:57:54.569985+0000","last_became_active":"2023-10-09T05:57:53.535646+0000","last_became_peered":"2023-10-09T05:57:53.535646+0000","last_unstale":"2023-10-09T05:57:54.569985+0000","last_undegraded":"2023-10-09T05:57:54.569985+0000","last_fullsized":"2023-10-09T05:57:54.569985+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:34.793388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.402288+0000","last_change":"2023-10-09T05:57:53.258209+0000","last_active":"2023-10-09T05:57:59.402288+0000","last_peered":"2023-10-09T05:57:59.402288+0000","last_clean":"2023-10-09T05:57:59.402288+0000","last_became_active":"2023-10-09T05:57:53.257937+0000","last_became_peered":"2023-10-09T05:57:53.257937+0000","last_unstale":"2023-10-09T05:57:59.402288+0000","last_undegraded":"2023-10-09T05:57:59.402288+0000","last_fullsized":"2023-10-09T05:57:59.402288+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:51.684373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569622+0000","last_change":"2023-10-09T05:57:53.535238+0000","last_active":"2023-10-09T05:57:54.569622+0000","last_peered":"2023-10-09T05:57:54.569622+0000","last_clean":"2023-10-09T05:57:54.569622+0000","last_became_active":"2023-10-09T05:57:53.534961+0000","last_became_peered":"2023-10-09T05:57:53.534961+0000","last_unstale":"2023-10-09T05:57:54.569622+0000","last_undegraded":"2023-10-09T05:57:54.569622+0000","last_fullsized":"2023-10-09T05:57:54.569622+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:47.646357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.402476+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T05:57:59.402476+0000","last_peered":"2023-10-09T05:57:59.402476+0000","last_clean":"2023-10-09T05:57:59.402476+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T05:57:59.402476+0000","last_undegraded":"2023-10-09T05:57:59.402476+0000","last_fullsized":"2023-10-09T05:57:59.402476+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265675+0000","last_change":"2023-10-09T05:57:53.535202+0000","last_active":"2023-10-09T05:57:54.265675+0000","last_peered":"2023-10-09T05:57:54.265675+0000","last_clean":"2023-10-09T05:57:54.265675+0000","last_became_active":"2023-10-09T05:57:53.534915+0000","last_became_peered":"2023-10-09T05:57:53.534915+0000","last_unstale":"2023-10-09T05:57:54.265675+0000","last_undegraded":"2023-10-09T05:57:54.265675+0000","last_fullsized":"2023-10-09T05:57:54.265675+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:57.805264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:59.399929+0000","last_change":"2023-10-09T05:57:53.259949+0000","last_active":"2023-10-09T05:57:59.399929+0000","last_peered":"2023-10-09T05:57:59.399929+0000","last_clean":"2023-10-09T05:57:59.399929+0000","last_became_active":"2023-10-09T05:57:53.259661+0000","last_became_peered":"2023-10-09T05:57:53.259661+0000","last_unstale":"2023-10-09T05:57:59.399929+0000","last_undegraded":"2023-10-09T05:57:59.399929+0000","last_fullsized":"2023-10-09T05:57:59.399929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:30.516054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570205+0000","last_change":"2023-10-09T05:57:53.537605+0000","last_active":"2023-10-09T05:57:54.570205+0000","last_peered":"2023-10-09T05:57:54.570205+0000","last_clean":"2023-10-09T05:57:54.570205+0000","last_became_active":"2023-10-09T05:57:53.536939+0000","last_became_peered":"2023-10-09T05:57:53.536939+0000","last_unstale":"2023-10-09T05:57:54.570205+0000","last_undegraded":"2023-10-09T05:57:54.570205+0000","last_fullsized":"2023-10-09T05:57:54.570205+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:57.551028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265628+0000","last_change":"2023-10-09T05:57:53.261118+0000","last_active":"2023-10-09T05:57:54.265628+0000","last_peered":"2023-10-09T05:57:54.265628+0000","last_clean":"2023-10-09T05:57:54.265628+0000","last_became_active":"2023-10-09T05:57:53.260832+0000","last_became_peered":"2023-10-09T05:57:53.260832+0000","last_unstale":"2023-10-09T05:57:54.265628+0000","last_undegraded":"2023-10-09T05:57:54.265628+0000","last_fullsized":"2023-10-09T05:57:54.265628+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:07.042447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570597+0000","last_change":"2023-10-09T05:57:53.534945+0000","last_active":"2023-10-09T05:57:54.570597+0000","last_peered":"2023-10-09T05:57:54.570597+0000","last_clean":"2023-10-09T05:57:54.570597+0000","last_became_active":"2023-10-09T05:57:53.534670+0000","last_became_peered":"2023-10-09T05:57:53.534670+0000","last_unstale":"2023-10-09T05:57:54.570597+0000","last_undegraded":"2023-10-09T05:57:54.570597+0000","last_fullsized":"2023-10-09T05:57:54.570597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:49:39.109589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570202+0000","last_change":"2023-10-09T05:57:53.258990+0000","last_active":"2023-10-09T05:57:54.570202+0000","last_peered":"2023-10-09T05:57:54.570202+0000","last_clean":"2023-10-09T05:57:54.570202+0000","last_became_active":"2023-10-09T05:57:53.258630+0000","last_became_peered":"2023-10-09T05:57:53.258630+0000","last_unstale":"2023-10-09T05:57:54.570202+0000","last_undegraded":"2023-10-09T05:57:54.570202+0000","last_fullsized":"2023-10-09T05:57:54.570202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:37.943621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.269714+0000","last_change":"2023-10-09T05:57:54.269425+0000","last_active":"2023-10-09T05:57:54.269714+0000","last_peered":"2023-10-09T05:57:54.269714+0000","last_clean":"2023-10-09T05:57:54.269714+0000","last_became_active":"2023-10-09T05:57:54.269156+0000","last_became_peered":"2023-10-09T05:57:54.269156+0000","last_unstale":"2023-10-09T05:57:54.269714+0000","last_undegraded":"2023-10-09T05:57:54.269714+0000","last_fullsized":"2023-10-09T05:57:54.269714+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:35.243726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.570540+0000","last_change":"2023-10-09T05:57:53.255654+0000","last_active":"2023-10-09T05:57:54.570540+0000","last_peered":"2023-10-09T05:57:54.570540+0000","last_clean":"2023-10-09T05:57:54.570540+0000","last_became_active":"2023-10-09T05:57:53.255371+0000","last_became_peered":"2023-10-09T05:57:53.255371+0000","last_unstale":"2023-10-09T05:57:54.570540+0000","last_undegraded":"2023-10-09T05:57:54.570540+0000","last_fullsized":"2023-10-09T05:57:54.570540+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:43.254489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569902+0000","last_change":"2023-10-09T05:57:53.534674+0000","last_active":"2023-10-09T05:57:54.569902+0000","last_peered":"2023-10-09T05:57:54.569902+0000","last_clean":"2023-10-09T05:57:54.569902+0000","last_became_active":"2023-10-09T05:57:53.534397+0000","last_became_peered":"2023-10-09T05:57:53.534397+0000","last_unstale":"2023-10-09T05:57:54.569902+0000","last_undegraded":"2023-10-09T05:57:54.569902+0000","last_fullsized":"2023-10-09T05:57:54.569902+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:52.239670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.265568+0000","last_change":"2023-10-09T05:57:53.260151+0000","last_active":"2023-10-09T05:57:54.265568+0000","last_peered":"2023-10-09T05:57:54.265568+0000","last_clean":"2023-10-09T05:57:54.265568+0000","last_became_active":"2023-10-09T05:57:53.259173+0000","last_became_peered":"2023-10-09T05:57:53.259173+0000","last_unstale":"2023-10-09T05:57:54.265568+0000","last_undegraded":"2023-10-09T05:57:54.265568+0000","last_fullsized":"2023-10-09T05:57:54.265568+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:03.531305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T05:57:54.569688+0000","last_change":"2023-10-09T05:57:53.278837+0000","last_active":"2023-10-09T05:57:54.569688+0000","last_peered":"2023-10-09T05:57:54.569688+0000","last_clean":"2023-10-09T05:57:54.569688+0000","last_became_active":"2023-10-09T05:57:53.278551+0000","last_became_peered":"2023-10-09T05:57:53.278551+0000","last_unstale":"2023-10-09T05:57:54.569688+0000","last_undegraded":"2023-10-09T05:57:54.569688+0000","last_fullsized":"2023-10-09T05:57:54.569688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:01:30.406415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T05:57:55.833404+0000","last_change":"2023-10-09T05:57:53.255186+0000","last_active":"2023-10-09T05:57:55.833404+0000","last_peered":"2023-10-09T05:57:55.833404+0000","last_clean":"2023-10-09T05:57:55.833404+0000","last_became_active":"2023-10-09T05:57:53.251853+0000","last_became_peered":"2023-10-09T05:57:53.251853+0000","last_unstale":"2023-10-09T05:57:55.833404+0000","last_undegraded":"2023-10-09T05:57:55.833404+0000","last_fullsized":"2023-10-09T05:57:55.833404+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:48.852063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234017,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.50900000000000001,"15min":0.49099999999999999},"min":{"1min":0.373,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.6339999999999999,"5min":2.6339999999999999,"15min":2.6339999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.52000000000000002,"15min":0.52700000000000002},"min":{"1min":0.36799999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.443}]},{"osd":1,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.51500000000000001,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.51600000000000001,"15min":0.52000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56899999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.52000000000000002,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.5259999999999998,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.61599999999999999,"15min":0.61199999999999999},"min":{"1min":0.36499999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.5579999999999998,"5min":2.5579999999999998,"15min":2.5579999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.36899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":2.5830000000000002,"5min":2.5830000000000002,"15min":2.5830000000000002},"last":0.51700000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.33000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.748,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.499},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.66100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.51900000000000002,"15min":0.50900000000000001},"min":{"1min":0.32100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.54400000000000004,"15min":0.53700000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.83999999999999997,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.64700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80100000000000005,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.621},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85699999999999998,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.752}]}]},{"osd":6,"up_from":46,"seq":197568495654,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.51800000000000002,"15min":0.52400000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.70499999999999996,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.747,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.54300000000000004,"15min":0.54900000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.79800000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81999999999999995,"5min":0.88,"15min":0.88},"last":0.68300000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.53300000000000003,"15min":0.54300000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69199999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55700000000000005,"15min":0.56299999999999994},"min":{"1min":0.34000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81299999999999994,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.73899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.436,"15min":0.439},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63300000000000001,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73399999999999999,"5min":0.753,"15min":0.753},"last":0.627}]},{"osd":5,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65800000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.5,"5min":0.496,"15min":0.496},"min":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76500000000000001,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.69899999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.499},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.73499999999999999,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80300000000000005,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.72799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.34100000000000003,"15min":0.30299999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.51200000000000001,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.36899999999999999,"15min":0.33700000000000002},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.501,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.255}]},{"osd":2,"last update":"Mon Oct 9 05:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.25,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.40200000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:57:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.27200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.59899999999999998,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.38},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.46600000000000003,"15min":0.47699999999999998},"min":{"1min":0.29699999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.58399999999999996,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.34899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59399999999999997,"15min":0.60399999999999998},"min":{"1min":0.40999999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.74399999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.628,"15min":0.64600000000000002},"min":{"1min":0.45600000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.79900000000000004,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.59899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.60399999999999998,"15min":0.622},"min":{"1min":0.38100000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.90300000000000002,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.628,"15min":0.64200000000000002},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":0.61399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59199999999999997,"15min":0.59899999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78400000000000003,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.40400000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.626}]},{"osd":7,"last update":"Mon Oct 9 05:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.58299999999999996,"15min":0.58099999999999996},"min":{"1min":0.41399999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76900000000000002,"5min":0.876,"15min":0.876},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.77600000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58299999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738434,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.31900000000000001,"5min":0.34799999999999998,"15min":0.33500000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.438,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.376},{"interface":"front","average":{"1min":0.33100000000000002,"5min":0.35499999999999998,"15min":0.33700000000000002},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.42199999999999999,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.50800000000000001,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.495,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.48099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.433,"15min":0.45100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.54700000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.44600000000000001,"15min":0.46100000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57099999999999995,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.44400000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:58:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.55500000000000005,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.63300000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.623,"15min":0.60599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.2679999999999998,"5min":4.2679999999999998,"15min":4.2679999999999998},"last":0.79300000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.58899999999999997,"15min":0.60899999999999999},"min":{"1min":0.38100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58399999999999996,"15min":0.60199999999999998},"min":{"1min":0.372,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76800000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.78200000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.59199999999999997,"15min":0.59899999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71099999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79100000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.71999999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.88200000000000001,"5min":2.9020000000000001,"15min":2.9020000000000001},"last":0.254},{"interface":"front","average":{"1min":0.373,"5min":0.40600000000000003,"15min":0.39700000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54800000000000004,"5min":2.9390000000000001,"15min":2.9390000000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:58:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.41799999999999998,"15min":0.39800000000000002},"min":{"1min":0.23899999999999999,"5min":0.218,"15min":0.218},"max":{"1min":1.1839999999999999,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.372},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.436,"15min":0.42299999999999999},"min":{"1min":0.245,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.91800000000000004,"5min":2.9319999999999999,"15min":2.9319999999999999},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 05:57:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.1309999999999998,"5min":3.1309999999999998,"15min":3.1309999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.48399999999999999,"15min":0.48699999999999999},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.077,"5min":3.077,"15min":3.077},"last":0.29799999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:58:06 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.61599999999999999,"15min":0.60399999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.7959999999999998,"5min":2.7959999999999998,"15min":2.7959999999999998},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.627,"15min":0.621},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":2.988,"5min":2.988,"15min":2.988},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60899999999999999,"15min":0.59599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.2330000000000001,"5min":2.2989999999999999,"15min":2.2989999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.64600000000000002,"15min":0.64200000000000002},"min":{"1min":0.44400000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.995,"5min":2.831,"15min":2.831},"last":0.67600000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:57:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.60999999999999999,"15min":0.59399999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.61399999999999999,"15min":0.59999999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.71899999999999997,"15min":0.73699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.96999999999999997,"5min":2.8719999999999999,"15min":2.8719999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.71799999999999997,"15min":0.73699999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.88500000000000001,"5min":3.0190000000000001,"15min":3.0190000000000001},"last":0.57899999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182450,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.307,"5min":0.39700000000000002,"15min":0.42099999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.55300000000000005,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.41699999999999998,"15min":0.437},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.442,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.439,"15min":0.46300000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.46100000000000002,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.45200000000000001,"15min":0.48399999999999999},"min":{"1min":0.26100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.46999999999999997,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:57:51 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.46400000000000002,"15min":0.48899999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.51300000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.46700000000000003,"15min":0.495},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.52200000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.65700000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56999999999999995,"15min":0.59199999999999997},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65900000000000003,"5min":1.077,"15min":1.077},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.57199999999999995,"15min":0.58799999999999997},"min":{"1min":0.35699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.82499999999999996,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.82599999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.59699999999999998,"15min":0.621},"min":{"1min":0.434,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76000000000000001,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.59799999999999998,"15min":0.59899999999999998},"min":{"1min":0.39500000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.3879999999999999,"5min":3.3879999999999999,"15min":3.3879999999999999},"last":0.83599999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:57:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.57499999999999996,"15min":0.57799999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.752,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.60199999999999998,"15min":0.61099999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.70999999999999996,"5min":1.02,"15min":1.02},"last":0.84399999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42399999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.87},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.44700000000000001,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.67100000000000004,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.86499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018926,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":712704,"data_stored":300974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.52400000000000002,"15min":0.54400000000000004},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.315},"max":{"1min":0.69699999999999995,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.52800000000000002,"15min":0.53700000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.51900000000000002,"15min":0.53100000000000003},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.61299999999999999,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.373},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72199999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55500000000000005,"15min":0.56399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.89400000000000002,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.56000000000000005},"min":{"1min":0.35399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.71899999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.51100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.59699999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83399999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.81299999999999994,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:58:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.47499999999999998,"15min":0.44600000000000001},"min":{"1min":0.42199999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.79400000000000004,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.48199999999999998,"15min":0.45100000000000001},"min":{"1min":0.34999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.75900000000000001,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.58599999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.48399999999999999,"15min":0.46800000000000003},"min":{"1min":0.36299999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.53600000000000003,"15min":0.52100000000000002},"min":{"1min":0.36399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.56699999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.56899999999999995,"15min":0.55500000000000005},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.85999999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.48499999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.874,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.53700000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757291,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.54100000000000004},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.745,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.50900000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67500000000000004,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.70399999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.50600000000000001,"15min":0.51300000000000001},"min":{"1min":0.30599999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.69899999999999995,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71899999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.63200000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67900000000000005,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56999999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78300000000000003,"5min":0.872,"15min":0.872},"last":0.65900000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.65700000000000003,"5min":1.522,"15min":1.522},"last":0.748},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56599999999999995,"15min":0.56799999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.83099999999999996,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.76800000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.45800000000000002,"15min":0.436},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.69499999999999995,"5min":0.752,"15min":0.752},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.46700000000000003,"15min":0.438},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.502,"15min":0.496},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73799999999999999,"5min":0.754,"15min":0.754},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.50600000000000001,"15min":0.498},"min":{"1min":0.36799999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.72699999999999998,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:58:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59699999999999998,"15min":0.59399999999999997},"min":{"1min":0.39300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.79000000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.68799999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:58:42.929 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T05:58:42.929 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T05:58:42.930 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T05:58:42.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T05:58:43.278 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:43.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:43 smithi107 ceph-mon[101102]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:43.684 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:43 smithi107 ceph-mon[118900]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:43.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:43 smithi150 ceph-mon[105942]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:58:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:44 smithi107 ceph-mon[101102]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:44 smithi107 ceph-mon[118900]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:44.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:44 smithi150 ceph-mon[105942]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:45.144 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T05:58:45.144 INFO:teuthology.orchestra.run.smithi107.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T05:58:45.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3590187751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:58:45.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.435 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3590187751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.872 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.873 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.873 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:45.889 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T05:58:45.889 INFO:teuthology.run_tasks:Running task print... 2023-10-09T05:58:45.898 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T05:58:45.898 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T05:58:45.907 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T05:58:45.907 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:58:45.907 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T05:58:45.908 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T05:58:45.909 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T05:58:45.909 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T05:58:45.910 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T05:58:45.910 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T05:58:45.910 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T05:58:45.911 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi107.front.sepia.ceph.com 2023-10-09T05:58:45.911 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3590187751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-10-09T05:58:45.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:58:46.273 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:46.396 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:46 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:46] "GET /metrics HTTP/1.1" 200 238724 "" "Prometheus/2.43.0" 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[101102]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:58:46.482 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[101102]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:58:46.483 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[118900]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:46.483 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T05:58:46.483 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T05:58:46.483 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:58:46.483 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:46 smithi107 ceph-mon[118900]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:58:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:46 smithi150 ceph-mon[105942]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-10-09T05:58:46.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-09T05:58:46.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:46 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-10-09T05:58:46.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:46 smithi150 ceph-mon[105942]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T05:58:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:47 smithi107 ceph-mon[101102]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:58:47.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:47 smithi107 ceph-mon[118900]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:58:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:47 smithi150 ceph-mon[105942]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T05:58:48.524 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:48 smithi107 ceph-mon[101102]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:48.524 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:48 smithi107 ceph-mon[118900]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:48.757 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:48 smithi150 ceph-mon[105942]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:48.794 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE Bus STOPPING 2023-10-09T05:58:48.794 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:58:48.794 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE Bus STOPPED 2023-10-09T05:58:48.795 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE Bus STARTING 2023-10-09T05:58:49.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE Serving on http://:::9283 2023-10-09T05:58:49.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:48 smithi107 conmon[101524]: [09/Oct/2023:05:58:48] ENGINE Bus STARTED 2023-10-09T05:58:49.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:48 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:49.335 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T05:58:49.674 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:49.746 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_0' created 2023-10-09T05:58:49.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' 2023-10-09T05:58:49.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:49.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3142133266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:49.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:49.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:49.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3142133266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:49.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:49 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3142133266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:49 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T05:58:50.594 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T05:58:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[101102]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:58:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[101102]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:58:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[118900]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:58:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:50 smithi107 ceph-mon[118900]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:58:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:50 smithi150 ceph-mon[105942]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:50 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T05:58:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:50 smithi150 ceph-mon[105942]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T05:58:51.344 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:52.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:51 smithi107 ceph-mon[101102]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:58:52.080 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:51 smithi107 ceph-mon[118900]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:58:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:51 smithi150 ceph-mon[105942]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T05:58:52.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[101102]: pgmap v79: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[101102]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:58:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[101102]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:58:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[118900]: pgmap v79: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[118900]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:58:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:52 smithi107 ceph-mon[118900]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:58:53.036 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T05:58:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:52 smithi150 ceph-mon[105942]: pgmap v79: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:52 smithi150 ceph-mon[105942]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T05:58:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:52 smithi150 ceph-mon[105942]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T05:58:53.491 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:54.002 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T05:58:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[101102]: pgmap v81: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3469392994' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[118900]: pgmap v81: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3469392994' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:53 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:53 smithi150 ceph-mon[105942]: pgmap v81: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:53 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3469392994' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:53 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:58:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:54 smithi107 ceph-mon[101102]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:54 smithi107 ceph-mon[101102]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:58:55.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:54 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:54 smithi107 ceph-mon[118900]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:58:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:54 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T05:58:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:54 smithi150 ceph-mon[105942]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T05:58:55.530 INFO:teuthology.orchestra.run.smithi107.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:55.912 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T05:58:55.915 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T05:58:55.915 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T05:58:55.961 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T05:58:55.961 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T05:58:55.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_191452 2023-10-09T05:58:55.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:58:55.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T05:58:56.242 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T05:58:56.546 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:58:56 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:56] "GET /metrics HTTP/1.1" 200 238724 "" "Prometheus/2.43.0" 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: pgmap v83: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: Cluster is now healthy 2023-10-09T05:58:56.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:56.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:56.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: pgmap v83: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:56.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:56.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:56.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:58:56.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: Cluster is now healthy 2023-10-09T05:58:56.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:56.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:56.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:56 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: pgmap v83: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T05:58:56.934 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: Cluster is now healthy 2023-10-09T05:58:56.935 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:58:56.935 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:58:56.935 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:56 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:58:56.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 3578 3562 222.611 222.625 0.0072098 0.00447674 2023-10-09T05:58:57.042 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:58:57.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:57 smithi107 ceph-mon[118900]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:57.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:57 smithi107 ceph-mon[101102]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:57.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:57 smithi150 ceph-mon[105942]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T05:58:57.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 7051 7035 219.827 217.062 0.00470586 0.00453913 2023-10-09T05:58:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:58:58 smithi107 ceph-mon[101102]: pgmap v84: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2023-10-09T05:58:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:58:58 smithi107 ceph-mon[118900]: pgmap v84: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2023-10-09T05:58:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:58:58 smithi150 ceph-mon[105942]: pgmap v84: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 204 op/s 2023-10-09T05:58:58.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:58:58 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:58:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:58:58.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 10274 10258 213.69 201.438 0.00734014 0.00467425 2023-10-09T05:58:59.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 13136 13120 204.977 178.875 0.0041492 0.00487287 2023-10-09T05:59:00.566 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:00 smithi107 ceph-mon[101102]: pgmap v85: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2023-10-09T05:59:00.567 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:00 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:00 smithi150 ceph-mon[105942]: pgmap v85: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2023-10-09T05:59:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:00 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:00 smithi107 ceph-mon[118900]: pgmap v85: 89 pgs: 89 active+clean; 85 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2023-10-09T05:59:00.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:00 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:00.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 15780 15764 197.029 165.25 0.0046333 0.00506883 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (74s) 51s ago 2m 16.9M - 0.25.0 c8568f914cd2 2b73cb66110c 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (94s) 52s ago 94s 86.1M - 9.4.7 2c41d148cca3 eaf4772a342e 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (63s) 51s ago 62s 18.4M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (60s) 52s ago 60s 13.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443 running (5m) 52s ago 5m 439M - 17.2.6-1303-gb2903cb0 5517396c50d4 4c9c42dc3a0f 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:9283 running (7m) 51s ago 7m 494M - 17.2.6-1303-gb2903cb0 5517396c50d4 64caccec47d2 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (7m) 51s ago 7m 83.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 b3f3a3055d81 2023-10-09T05:59:01.388 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (6m) 52s ago 6m 61.7M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 0a878f231494 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (6m) 51s ago 6m 55.5M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c53cec84dedf 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (118s) 51s ago 117s 17.3M - 1.5.0 0da6a335fe13 da46b9f4dd3d 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (114s) 52s ago 114s 13.5M - 1.5.0 0da6a335fe13 3ead6ee07e6f 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (5m) 51s ago 5m 57.0M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 8de9f1d99699 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (5m) 51s ago 5m 58.6M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 674ef542e3e9 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (4m) 51s ago 4m 51.9M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 4f4ef8822b1f 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (4m) 51s ago 4m 52.7M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 e1f7d3e8c22d 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (4m) 52s ago 4m 51.1M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (3m) 52s ago 3m 48.6M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (3m) 52s ago 3m 49.8M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (2m) 52s ago 2m 51.6M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T05:59:01.389 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (70s) 52s ago 2m 36.6M - 2.43.0 a07b618ecd1d d2a78d53b241 2023-10-09T05:59:01.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 18317 18301 190.616 158.562 0.00780389 0.0052393 2023-10-09T05:59:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:02 smithi107 ceph-mon[101102]: pgmap v86: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-10-09T05:59:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:02 smithi107 ceph-mon[101102]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:02.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:02 smithi107 ceph-mon[118900]: pgmap v86: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-10-09T05:59:02.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:02 smithi107 ceph-mon[118900]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:02 smithi150 ceph-mon[105942]: pgmap v86: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2023-10-09T05:59:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:02 smithi150 ceph-mon[105942]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:59:02.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 20902 20886 186.463 161.562 0.00420112 0.00535762 2023-10-09T05:59:03.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 23361 23345 182.365 153.688 0.0070066 0.00547807 2023-10-09T05:59:04.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T05:59:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:04 smithi107 ceph-mon[101102]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:59:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:04 smithi107 ceph-mon[118900]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:59:04.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:04 smithi150 ceph-mon[105942]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T05:59:04.795 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:04.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 25460 25444 176.678 131.188 0.0108576 0.00565699 2023-10-09T05:59:05.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 27574 27558 172.222 132.125 0.00628523 0.00580167 2023-10-09T05:59:06.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:06 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:06] "GET /metrics HTTP/1.1" 200 241499 "" "Prometheus/2.43.0" 2023-10-09T05:59:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:06 smithi107 ceph-mon[101102]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:59:06.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:06 smithi107 ceph-mon[118900]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:59:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:06 smithi150 ceph-mon[105942]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T05:59:06.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 29767 29751 169.012 137.062 0.00747171 0.00591169 2023-10-09T05:59:07.964 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 31752 31736 165.254 124.062 0.00444414 0.00604556 2023-10-09T05:59:08.454 INFO:teuthology.orchestra.run.smithi107.stdout:wait for minority of mons to upgrade 2023-10-09T05:59:08.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:08 smithi107 ceph-mon[101102]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-10-09T05:59:08.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:08 smithi107 ceph-mon[118900]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-10-09T05:59:08.759 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:08 smithi150 ceph-mon[105942]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.36k op/s 2023-10-09T05:59:08.964 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 33690 33674 161.859 121.125 0.006313 0.00617365 2023-10-09T05:59:09.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:08 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:09.964 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 35070 35054 156.459 86.25 0.00212908 0.00630332 2023-10-09T05:59:10.592 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T05:59:10.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:10 smithi107 ceph-mon[101102]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:59:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:10 smithi107 ceph-mon[118900]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:59:10.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:10 smithi150 ceph-mon[105942]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T05:59:10.964 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 36829 36813 153.358 109.938 0.00864741 0.00651374 2023-10-09T05:59:11.283 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:59:11.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 38392 38376 149.87 97.6875 0.00893225 0.00666812 2023-10-09T05:59:12.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:12 smithi107 ceph-mon[101102]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2023-10-09T05:59:12.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:12 smithi107 ceph-mon[118900]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2023-10-09T05:59:12.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:12 smithi150 ceph-mon[105942]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s 2023-10-09T05:59:12.965 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 40220 40204 147.775 114.25 0.0028755 0.00676036 2023-10-09T05:59:13.965 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 41490 41474 143.975 79.375 0.012926 0.00693885 2023-10-09T05:59:14.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:14 smithi107 ceph-mon[101102]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:14 smithi107 ceph-mon[118900]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:14 smithi150 ceph-mon[105942]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:14.967 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 43146 43130 141.839 103.5 0.0121669 0.00704525 2023-10-09T05:59:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:15 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1390192362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:15 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1390192362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:15.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:15 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1390192362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:15.966 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T05:59:15.967339+0000 min lat: 0.000944566 max lat: 0.372919 avg lat: 0.00714513 2023-10-09T05:59:15.966 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:15.967 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 44781 44765 139.856 102.188 0.0189101 0.00714513 2023-10-09T05:59:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:16 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:16] "GET /metrics HTTP/1.1" 200 242136 "" "Prometheus/2.43.0" 2023-10-09T05:59:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:16 smithi107 ceph-mon[118900]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:16 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:16.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:16 smithi107 ceph-mon[101102]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:16.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:16 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:16.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:16 smithi150 ceph-mon[105942]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:16.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:16 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:16.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 46413 46397 138.046 102 0.013729 0.00723902 2023-10-09T05:59:17.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 47851 47835 135.856 89.875 0.00662257 0.00735605 2023-10-09T05:59:18.758 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:18 smithi150 ceph-mon[105942]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-10-09T05:59:18.758 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:18 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2586217223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:18.759 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:18 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:18.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:18 smithi107 ceph-mon[118900]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-10-09T05:59:18.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:18 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2586217223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:18.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:18 smithi107 ceph-mon[101102]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.39k op/s 2023-10-09T05:59:18.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:18 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2586217223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:18.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 49351 49335 134.025 93.75 0.00643 0.00745635 2023-10-09T05:59:19.969 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 50809 50793 132.232 91.125 0.0071629 0.0075547 2023-10-09T05:59:20.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:20 smithi107 ceph-mon[118900]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T05:59:20.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:20 smithi150 ceph-mon[105942]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T05:59:20.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:20 smithi107 ceph-mon[101102]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-10-09T05:59:20.969 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 52158 52142 130.315 84.3125 0.00519872 0.00767014 2023-10-09T05:59:21.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:21 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3134179529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:21 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3134179529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:21.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:21 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3134179529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:21.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 53601 53585 128.769 90.1875 0.00690727 0.00776039 2023-10-09T05:59:22.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:22 smithi107 ceph-mon[101102]: pgmap v96: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:59:22.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:22 smithi107 ceph-mon[118900]: pgmap v96: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:59:22.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:22 smithi150 ceph-mon[105942]: pgmap v96: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T05:59:22.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 54987 54971 127.208 86.625 0.0138944 0.00785562 2023-10-09T05:59:23.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 56218 56202 125.413 76.9375 0.0109069 0.00796962 2023-10-09T05:59:24.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:24 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2828367918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:24 smithi107 ceph-mon[118900]: pgmap v97: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:24 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2828367918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:24 smithi107 ceph-mon[101102]: pgmap v97: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:24 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2828367918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:24 smithi150 ceph-mon[105942]: pgmap v97: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:24.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 57589 57573 124.043 85.6875 0.00574614 0.0080576 2023-10-09T05:59:25.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 58881 58865 122.6 80.75 0.018944 0.00815077 2023-10-09T05:59:26.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:26 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:26] "GET /metrics HTTP/1.1" 200 242136 "" "Prometheus/2.43.0" 2023-10-09T05:59:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:26 smithi107 ceph-mon[118900]: pgmap v98: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:26 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3356826070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:26.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:26 smithi107 ceph-mon[101102]: pgmap v98: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:26.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:26 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3356826070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:26 smithi150 ceph-mon[105942]: pgmap v98: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T05:59:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:26 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3356826070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:26.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 60189 60173 121.283 81.75 0.0106955 0.00824077 2023-10-09T05:59:27.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 61700 61684 120.444 94.4375 0.00778039 0.00829857 2023-10-09T05:59:28.759 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:28 smithi150 ceph-mon[105942]: pgmap v99: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T05:59:28.759 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:28 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:28 smithi107 ceph-mon[101102]: pgmap v99: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T05:59:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:28 smithi107 ceph-mon[118900]: pgmap v99: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T05:59:28.972 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 63078 63062 119.396 86.125 0.00981797 0.00837022 2023-10-09T05:59:29.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:29 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2782624512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:29.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:29 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2782624512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:29.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:29 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2782624512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:29.972 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 64577 64561 118.64 93.6875 0.00715141 0.0084249 2023-10-09T05:59:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:30 smithi107 ceph-mon[118900]: pgmap v100: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T05:59:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:30 smithi107 ceph-mon[101102]: pgmap v100: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T05:59:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:30.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:30 smithi150 ceph-mon[105942]: pgmap v100: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T05:59:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:30.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 66015 65999 117.818 89.875 0.00698095 0.0084833 2023-10-09T05:59:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2924464517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:31.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2924464517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2924464517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:31.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 67411 67395 116.968 87.25 0.0126797 0.0085448 2023-10-09T05:59:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:32 smithi107 ceph-mon[118900]: pgmap v101: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:59:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:32 smithi107 ceph-mon[101102]: pgmap v101: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:59:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:32 smithi150 ceph-mon[105942]: pgmap v101: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T05:59:32.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 68832 68816 116.207 88.8125 0.00671379 0.00860035 2023-10-09T05:59:33.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 70187 70171 115.377 84.6875 0.0135921 0.00866267 2023-10-09T05:59:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:34 smithi107 ceph-mon[101102]: pgmap v102: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:34.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:34 smithi107 ceph-mon[118900]: pgmap v102: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:34 smithi150 ceph-mon[105942]: pgmap v102: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:34.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 71564 71548 114.625 86.0625 0.0114255 0.00871933 2023-10-09T05:59:35.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2940798449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:35 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2940798449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:35 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2940798449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:35.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T05:59:35.974432+0000 min lat: 0.000944566 max lat: 0.372919 avg lat: 0.00877398 2023-10-09T05:59:35.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:35.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 72940 72924 113.91 86 0.0135234 0.00877398 2023-10-09T05:59:36.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:36 smithi107 ceph-mon[101102]: pgmap v103: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:36.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:36 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:36] "GET /metrics HTTP/1.1" 200 242248 "" "Prometheus/2.43.0" 2023-10-09T05:59:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:36 smithi107 ceph-mon[118900]: pgmap v103: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:36.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:36 smithi150 ceph-mon[105942]: pgmap v103: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T05:59:36.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 74244 74228 113.119 81.5 0.0130671 0.0088355 2023-10-09T05:59:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:37 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2212935551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:37 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2212935551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:37 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2212935551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:37.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 75596 75580 112.438 84.5 0.0105567 0.00888958 2023-10-09T05:59:38.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:38 smithi107 ceph-mon[101102]: pgmap v104: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:38 smithi107 ceph-mon[118900]: pgmap v104: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:38.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:38 smithi150 ceph-mon[105942]: pgmap v104: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T05:59:38.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 76878 76862 111.686 80.125 0.00511484 0.00894854 2023-10-09T05:59:39.155 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:38 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:39.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:39 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3471640482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:39.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:39 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3471640482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:39.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:39 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3471640482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:39.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 78150 78134 110.955 79.5 0.00843532 0.00900828 2023-10-09T05:59:40.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:40 smithi107 ceph-mon[101102]: pgmap v105: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:59:40.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:40 smithi107 ceph-mon[118900]: pgmap v105: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:59:40.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:40 smithi150 ceph-mon[105942]: pgmap v105: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T05:59:40.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 79456 79440 110.303 81.625 0.00807261 0.00906154 2023-10-09T05:59:41.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 80667 80651 109.55 75.6875 0.0066032 0.00912452 2023-10-09T05:59:42.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:42 smithi107 ceph-mon[101102]: pgmap v106: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T05:59:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:42 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/685911355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:42.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:42 smithi107 ceph-mon[118900]: pgmap v106: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T05:59:42.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:42 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/685911355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:42 smithi150 ceph-mon[105942]: pgmap v106: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-09T05:59:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:42 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/685911355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:42.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 15 81766 81751 108.682 68.75 0.0181211 0.00919606 2023-10-09T05:59:43.974 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 82937 82921 107.941 73.125 0.0109058 0.00925922 2023-10-09T05:59:44.975 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 84046 84030 107.151 69.3125 0.012438 0.00932827 2023-10-09T05:59:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1129846951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:45.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:59:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:44 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1129846951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T05:59:45.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:44 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1129846951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:45.975 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 85157 85141 106.396 69.4375 0.017288 0.00939456 2023-10-09T05:59:46.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:45 smithi107 ceph-mon[118900]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi150 2023-10-09T05:59:46.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:46.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:45 smithi107 ceph-mon[101102]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi150 2023-10-09T05:59:46.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:46.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:45 smithi150 ceph-mon[105942]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi150 2023-10-09T05:59:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:59:46.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:46 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:46] "GET /metrics HTTP/1.1" 200 242330 "" "Prometheus/2.43.0" 2023-10-09T05:59:46.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 86201 86185 105.59 65.25 0.0170492 0.00946559 2023-10-09T05:59:47.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:46 smithi107 ceph-mon[101102]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:46 smithi107 ceph-mon[118900]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:47.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:46 smithi150 ceph-mon[105942]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T05:59:47.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 87263 87247 104.836 66.375 0.0188248 0.00953388 2023-10-09T05:59:48.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:47 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2089021685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:48.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:47 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2089021685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:47 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2089021685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:48.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 88285 88269 104.063 63.875 0.0133556 0.00960473 2023-10-09T05:59:49.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:48 smithi107 ceph-mon[101102]: pgmap v109: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:59:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:48 smithi107 ceph-mon[118900]: pgmap v109: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:59:49.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:48 smithi150 ceph-mon[105942]: pgmap v109: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T05:59:49.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:48 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:49.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 89292 89276 103.301 62.9375 0.0120305 0.00967596 2023-10-09T05:59:50.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2233433498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2233433498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2233433498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:50.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 90314 90298 102.584 63.875 0.0126379 0.00974371 2023-10-09T05:59:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:50 smithi107 ceph-mon[118900]: pgmap v110: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:59:51.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:50 smithi107 ceph-mon[101102]: pgmap v110: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:59:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:50 smithi150 ceph-mon[105942]: pgmap v110: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T05:59:51.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 91293 91277 101.845 61.1875 0.018581 0.00981444 2023-10-09T05:59:52.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 92299 92283 101.162 62.875 0.00758054 0.009881 2023-10-09T05:59:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:52 smithi107 ceph-mon[101102]: pgmap v111: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:59:53.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:52 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2040766430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:52 smithi107 ceph-mon[118900]: pgmap v111: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:59:53.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:52 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2040766430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:52 smithi150 ceph-mon[105942]: pgmap v111: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T05:59:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:52 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2040766430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:53.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 93303 93287 100.499 62.75 0.0137435 0.0099457 2023-10-09T05:59:54.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:54 smithi107 ceph-mon[118900]: pgmap v112: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:59:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:54 smithi107 ceph-mon[101102]: pgmap v112: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:59:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:54 smithi150 ceph-mon[105942]: pgmap v112: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T05:59:54.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 94268 94252 99.8182 60.3125 0.0164554 0.010013 2023-10-09T05:59:55.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:55 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1583510903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:55.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:55 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1583510903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:55.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:55 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1583510903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:55.978 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T05:59:55.979289+0000 min lat: 0.000944566 max lat: 0.372919 avg lat: 0.0100637 2023-10-09T05:59:55.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T05:59:55.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 95396 95380 99.3262 70.5 0.0117251 0.0100637 2023-10-09T05:59:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:56 smithi107 ceph-mon[101102]: pgmap v113: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T05:59:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:56 smithi107 ceph-mon[118900]: pgmap v113: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T05:59:56.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 05:59:56 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:56] "GET /metrics HTTP/1.1" 200 242330 "" "Prometheus/2.43.0" 2023-10-09T05:59:56.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:56 smithi150 ceph-mon[105942]: pgmap v113: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T05:59:56.978 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 96427 96411 98.7543 64.4375 0.0131306 0.0101216 2023-10-09T05:59:57.978 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 97433 97417 98.1756 62.875 0.0340014 0.0101818 2023-10-09T05:59:58.761 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:58 smithi150 ceph-mon[105942]: pgmap v114: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:59:58.761 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 05:59:58 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/777825620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:58 smithi107 ceph-mon[101102]: pgmap v114: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:59:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 05:59:58 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/777825620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:58 smithi107 ceph-mon[118900]: pgmap v114: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T05:59:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 05:59:58 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/777825620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T05:59:58.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 98377 98361 97.5536 59 0.0162495 0.0102462 2023-10-09T05:59:59.155 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 05:59:58 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:05:59:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T05:59:59.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 99311 99295 96.9414 58.375 0.00771004 0.01031 2023-10-09T06:00:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[101102]: pgmap v115: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T06:00:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[101102]: overall HEALTH_OK 2023-10-09T06:00:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2535696328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[118900]: pgmap v115: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T06:00:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[118900]: overall HEALTH_OK 2023-10-09T06:00:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2535696328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:00 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:00 smithi150 ceph-mon[105942]: pgmap v115: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-10-09T06:00:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:00 smithi150 ceph-mon[105942]: overall HEALTH_OK 2023-10-09T06:00:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:00 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2535696328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:00 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:00.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 100264 100248 96.3664 59.5625 0.0211322 0.0103721 2023-10-09T06:00:01.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 101188 101172 95.7813 57.75 0.0082421 0.010435 2023-10-09T06:00:02.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:02 smithi107 ceph-mon[101102]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:00:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:02 smithi107 ceph-mon[118900]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:00:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:02 smithi150 ceph-mon[105942]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:00:02.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 102082 102066 95.1857 55.875 0.0201978 0.0105 2023-10-09T06:00:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:03 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1705897676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:03.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:03 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1705897676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:03.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:03 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1705897676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:03.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 103012 102996 94.6408 58.125 0.0217891 0.0105616 2023-10-09T06:00:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:04 smithi107 ceph-mon[101102]: pgmap v117: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T06:00:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:04 smithi107 ceph-mon[118900]: pgmap v117: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T06:00:04.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:04 smithi150 ceph-mon[105942]: pgmap v117: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-09T06:00:04.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 103876 103860 94.0518 54 0.0125025 0.0106271 2023-10-09T06:00:05.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:05 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/4236135849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:05 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/4236135849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:05 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/4236135849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:05.979 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 104739 104723 93.4787 53.9375 0.00998348 0.0106919 2023-10-09T06:00:06.499 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:06 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:06] "GET /metrics HTTP/1.1" 200 242342 "" "Prometheus/2.43.0" 2023-10-09T06:00:06.499 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:06 smithi107 ceph-mon[101102]: pgmap v118: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T06:00:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:06 smithi107 ceph-mon[118900]: pgmap v118: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T06:00:06.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:06 smithi150 ceph-mon[105942]: pgmap v118: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-10-09T06:00:06.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 105602 105586 92.9218 53.9375 0.0288928 0.0107575 2023-10-09T06:00:07.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 106444 106428 92.3622 52.625 0.00580148 0.0108219 2023-10-09T06:00:08.770 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:08 smithi150 ceph-mon[105942]: pgmap v119: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:00:08.770 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:08 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1382566603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:08.771 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:08 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:00:08.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:08 smithi107 ceph-mon[101102]: pgmap v119: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:00:08.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:08 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1382566603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:08.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:08 smithi107 ceph-mon[118900]: pgmap v119: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:00:08.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:08 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1382566603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:08.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 107302 107286 91.8316 53.625 0.0120146 0.0108841 2023-10-09T06:00:09.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 108146 108130 91.3035 52.75 0.0153032 0.0109471 2023-10-09T06:00:10.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:10 smithi107 ceph-mon[101102]: pgmap v120: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2023-10-09T06:00:10.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:10 smithi107 ceph-mon[118900]: pgmap v120: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2023-10-09T06:00:10.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:10 smithi150 ceph-mon[105942]: pgmap v120: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s 2023-10-09T06:00:10.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 108958 108942 90.7628 50.75 0.0110231 0.011012 2023-10-09T06:00:11.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:11 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3593040844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:11 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3593040844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:11 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3593040844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:11.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 109829 109813 90.2848 54.4375 0.0124217 0.0110711 2023-10-09T06:00:12.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:12 smithi107 ceph-mon[118900]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:00:12.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:12 smithi107 ceph-mon[101102]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:00:12.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:12 smithi150 ceph-mon[105942]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-09T06:00:12.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 110639 110623 89.7698 50.625 0.0153611 0.0111347 2023-10-09T06:00:13.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 111440 111424 89.2607 50.0625 0.00363124 0.0111977 2023-10-09T06:00:14.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:14 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/226601534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:14.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:14 smithi107 ceph-mon[101102]: pgmap v122: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:00:14.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:14 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/226601534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:14.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:14 smithi107 ceph-mon[118900]: pgmap v122: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:00:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:14 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/226601534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:14 smithi150 ceph-mon[105942]: pgmap v122: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-09T06:00:14.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 112130 112114 88.6768 43.125 0.0298609 0.011271 2023-10-09T06:00:15.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:15 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:15.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:15 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:15 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:15.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:00:15.981195+0000 min lat: 0.000944566 max lat: 0.372919 avg lat: 0.0113325 2023-10-09T06:00:15.980 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:00:15.981 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 112943 112927 88.2035 50.8125 0.0183468 0.0113325 2023-10-09T06:00:16.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:16 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:16] "GET /metrics HTTP/1.1" 200 242374 "" "Prometheus/2.43.0" 2023-10-09T06:00:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:16 smithi107 ceph-mon[101102]: pgmap v123: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-10-09T06:00:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:16 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3602791537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:16 smithi107 ceph-mon[118900]: pgmap v123: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-10-09T06:00:16.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:16 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3602791537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:16 smithi150 ceph-mon[105942]: pgmap v123: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-10-09T06:00:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:16 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3602791537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:16.986 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 113742 113726 87.7246 49.9375 0.0243792 0.0113952 2023-10-09T06:00:17.986 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 114587 114571 87.2989 52.8125 0.0062808 0.01145 2023-10-09T06:00:18.736 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:18 smithi150 ceph-mon[105942]: pgmap v124: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:00:18.736 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:18 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2261484621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:18.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:18 smithi107 ceph-mon[101102]: pgmap v124: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:00:18.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:18 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2261484621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:18.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:18 smithi107 ceph-mon[118900]: pgmap v124: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T06:00:18.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:18 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2261484621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:18.986 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 115484 115468 86.9226 56.0625 0.026205 0.0114994 2023-10-09T06:00:19.054 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:18 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:00:19.992 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 116440 116424 86.5932 59.75 0.018651 0.0115436 2023-10-09T06:00:20.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:20 smithi107 ceph-mon[101102]: pgmap v125: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T06:00:20.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:20 smithi150 ceph-mon[105942]: pgmap v125: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T06:00:20.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:20 smithi107 ceph-mon[118900]: pgmap v125: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T06:00:20.992 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 117372 117356 86.2598 58.25 0.0290965 0.0115886 2023-10-09T06:00:21.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:21 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3530387793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:21.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:21 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3530387793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:21.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:21 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3530387793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:21.993 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 118323 118307 85.9479 59.4375 0.0194096 0.0116305 2023-10-09T06:00:22.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:22 smithi107 ceph-mon[101102]: pgmap v126: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:00:22.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:22 smithi107 ceph-mon[118900]: pgmap v126: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:00:22.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:22 smithi150 ceph-mon[105942]: pgmap v126: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s 2023-10-09T06:00:22.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 119256 119240 85.6296 58.3125 0.0320693 0.0116729 2023-10-09T06:00:23.993 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 120150 120134 85.2915 55.875 0.0204839 0.0117195 2023-10-09T06:00:24.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:24 smithi107 ceph-mon[101102]: pgmap v127: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:00:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:24 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/352386205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:24 smithi107 ceph-mon[118900]: pgmap v127: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:00:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:24 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/352386205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:24.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:24 smithi150 ceph-mon[105942]: pgmap v127: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:00:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:24 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/352386205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:24.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 121074 121058 84.982 57.75 0.0203568 0.0117629 2023-10-09T06:00:25.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0179 2023-10-09T06:00:25.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 121940 2023-10-09T06:00:25.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:00:25.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:00:25.994 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 84.6637 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 37.8033 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 222.625 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 43.125 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1354 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 604.853 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3562 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 690 2023-10-09T06:00:25.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.011807 2023-10-09T06:00:25.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.00832419 2023-10-09T06:00:25.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 0.372919 2023-10-09T06:00:25.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.000944566 2023-10-09T06:00:25.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:00:26.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:26 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:26] "GET /metrics HTTP/1.1" 200 242374 "" "Prometheus/2.43.0" 2023-10-09T06:00:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:26 smithi107 ceph-mon[101102]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-10-09T06:00:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:26 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3776808826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:26 smithi107 ceph-mon[118900]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-10-09T06:00:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:26 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3776808826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:26.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:26 smithi150 ceph-mon[105942]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s 2023-10-09T06:00:26.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:26 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3776808826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:28.758 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:28 smithi150 ceph-mon[105942]: pgmap v129: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.08k op/s 2023-10-09T06:00:28.759 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:28 smithi150 conmon[109180]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:00:28.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:28 smithi107 ceph-mon[101102]: pgmap v129: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.08k op/s 2023-10-09T06:00:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:28 smithi107 ceph-mon[118900]: pgmap v129: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.08k op/s 2023-10-09T06:00:29.889 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:29 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2443894568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:29.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:29 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2443894568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:29.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:29 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2443894568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: pgmap v130: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 753 op/s 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: Upgrade: Updating mgr.x 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:00:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: Deploying daemon mgr.x on smithi150 2023-10-09T06:00:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:30 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:31.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: pgmap v130: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 753 op/s 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: Upgrade: Updating mgr.x 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: Deploying daemon mgr.x on smithi150 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: pgmap v130: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 753 op/s 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: Upgrade: Updating mgr.x 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: Deploying daemon mgr.x on smithi150 2023-10-09T06:00:31.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:30 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:32.333 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3717025114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:32.334 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:32 smithi150 systemd[1]: Stopping Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:00:32.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3717025114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3717025114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:33.206 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:32 smithi150 ceph-mon[105942]: pgmap v131: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 873 op/s 2023-10-09T06:00:33.206 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:33 smithi150 bash[159472]: 4c9c42dc3a0f2bbc7065586f1934d43546e28877716d748120d09a798327bf5c 2023-10-09T06:00:33.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:32 smithi107 ceph-mon[118900]: pgmap v131: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 873 op/s 2023-10-09T06:00:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:32 smithi107 ceph-mon[101102]: pgmap v131: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 873 op/s 2023-10-09T06:00:34.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:34 smithi107 ceph-mon[118900]: pgmap v132: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.01k op/s 2023-10-09T06:00:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:34 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3588262786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:34 smithi107 ceph-mon[101102]: pgmap v132: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.01k op/s 2023-10-09T06:00:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:34 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3588262786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:34 smithi150 ceph-mon[105942]: pgmap v132: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 1.01k op/s 2023-10-09T06:00:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:34 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3588262786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:34.409 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:33 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:00:34.409 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:34 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T06:00:34.409 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:34 smithi150 systemd[1]: Stopped Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:34.409 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:34 smithi150 systemd[1]: Starting Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:00:35.651 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 bash[159802]: 157473f4f8c95f65504171d186c1765d696c18e45b058058ce6579ebc5a61b17 2023-10-09T06:00:35.651 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 systemd[1]: Started Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:00:35.651 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 conmon[159842]: 2023-10-09T06:00:35.516+0000 7f628e822200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:00:35.651 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 conmon[159842]: 2023-10-09T06:00:35.646+0000 7f628e822200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:00:35.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 conmon[159842]: 2023-10-09T06:00:35.805+0000 7f628e822200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:00:36.275 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:35 smithi150 conmon[159842]: 2023-10-09T06:00:35.970+0000 7f628e822200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:00:36.275 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:36 smithi150 conmon[159842]: 2023-10-09T06:00:36.268+0000 7f628e822200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:00:36.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:36 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:36] "GET /metrics HTTP/1.1" 200 242453 "" "Prometheus/2.43.0" 2023-10-09T06:00:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[118900]: pgmap v133: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 753 op/s 2023-10-09T06:00:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:36.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[101102]: pgmap v133: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 753 op/s 2023-10-09T06:00:36.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:36 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:36 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:36 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:36 smithi150 ceph-mon[105942]: pgmap v133: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 753 op/s 2023-10-09T06:00:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:36 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:37.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:36 smithi150 conmon[159842]: 2023-10-09T06:00:36.895+0000 7f628e822200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:00:37.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:37 smithi150 conmon[159842]: 2023-10-09T06:00:37.093+0000 7f628e822200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:00:37.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:37 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/783337870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:37 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/783337870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:37.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:37 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/783337870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:37.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:37 smithi150 conmon[159842]: 2023-10-09T06:00:37.240+0000 7f628e822200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:00:37.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:37 smithi150 conmon[159842]: 2023-10-09T06:00:37.397+0000 7f628e822200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:00:38.411 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:38 smithi150 conmon[159842]: 2023-10-09T06:00:38.099+0000 7f628e822200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:00:38.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:38 smithi107 ceph-mon[101102]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.09k op/s 2023-10-09T06:00:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:38 smithi107 ceph-mon[118900]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.09k op/s 2023-10-09T06:00:38.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:38 smithi150 ceph-mon[105942]: pgmap v134: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.09k op/s 2023-10-09T06:00:39.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:39 smithi150 conmon[159842]: 2023-10-09T06:00:39.176+0000 7f628e822200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:00:39.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:39 smithi150 conmon[159842]: 2023-10-09T06:00:39.313+0000 7f628e822200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:00:40.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/824838291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:40.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:39 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/824838291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:40.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:39 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:39 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:40.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:39 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/824838291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:40 smithi150 ceph-mon[105942]: pgmap v135: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:00:40.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:40 smithi150 conmon[159842]: 2023-10-09T06:00:40.594+0000 7f628e822200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:00:40.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:40 smithi150 conmon[159842]: 2023-10-09T06:00:40.738+0000 7f628e822200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:00:41.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:40 smithi107 ceph-mon[101102]: pgmap v135: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:00:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:40 smithi107 ceph-mon[118900]: pgmap v135: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:00:41.405 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:41 smithi150 conmon[159842]: 2023-10-09T06:00:41.038+0000 7f628e822200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:00:41.812 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:41 smithi150 conmon[159842]: 2023-10-09T06:00:41.521+0000 7f628e822200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:00:42.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:41 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2653297467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:42.118 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:41 smithi150 conmon[159842]: 2023-10-09T06:00:41.863+0000 7f628e822200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:00:42.118 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:42 smithi150 conmon[159842]: 2023-10-09T06:00:42.117+0000 7f628e822200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:00:42.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:41 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2653297467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:42.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:41 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2653297467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:42 smithi150 ceph-mon[105942]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:00:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:42 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:42 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:42.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:42 smithi150 conmon[159842]: 2023-10-09T06:00:42.586+0000 7f628e822200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:00:43.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[118900]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:00:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:43.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[101102]: pgmap v136: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-09T06:00:43.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:43.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:42 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:44.159 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:43 smithi150 conmon[159842]: 2023-10-09T06:00:43.750+0000 7f628e822200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:00:44.159 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:43 smithi150 conmon[159842]: 2023-10-09T06:00:43.957+0000 7f628e822200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:00:44.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:44 smithi150 conmon[159842]: 2023-10-09T06:00:44.275+0000 7f628e822200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:00:44.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:44 smithi150 conmon[159842]: 2023-10-09T06:00:44.463+0000 7f628e822200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:00:44.971 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:44 smithi150 ceph-mon[105942]: pgmap v137: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T06:00:44.971 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:44 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:44.971 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:44 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:44.971 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:44 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/851808581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:44.971 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:44 smithi150 conmon[159842]: 2023-10-09T06:00:44.827+0000 7f628e822200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:00:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[118900]: pgmap v137: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T06:00:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/851808581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[101102]: pgmap v137: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:45.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:44 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/851808581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:46.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:45 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:46.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:45 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:45 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:46.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:46 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:46] "GET /metrics HTTP/1.1" 200 242455 "" "Prometheus/2.43.0" 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: 2023-10-09T06:00:46.207+0000 7f628e822200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: [09/Oct/2023:06:00:46] ENGINE Bus STARTING 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: CherryPy Checker: 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: The Application mounted at '' has an empty config. 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: [09/Oct/2023:06:00:46] ENGINE Serving on http://:::9283 2023-10-09T06:00:46.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:46 smithi150 conmon[159842]: [09/Oct/2023:06:00:46] ENGINE Bus STARTED 2023-10-09T06:00:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: pgmap v138: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:00:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: Standby manager daemon x restarted 2023-10-09T06:00:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: Standby manager daemon x started 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: pgmap v138: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: Standby manager daemon x restarted 2023-10-09T06:00:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T06:00:47.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:00:47.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:00:47.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:00:47.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:46 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: pgmap v138: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: Standby manager daemon x restarted 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:00:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:46 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:00:48.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:47 smithi107 ceph-mon[101102]: mgrmap e20: y(active, since 4m), standbys: x 2023-10-09T06:00:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:47 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/899089762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:47 smithi107 ceph-mon[118900]: mgrmap e20: y(active, since 4m), standbys: x 2023-10-09T06:00:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:47 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/899089762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:47 smithi150 ceph-mon[105942]: mgrmap e20: y(active, since 4m), standbys: x 2023-10-09T06:00:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:47 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/899089762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:48 smithi107 ceph-mon[118900]: pgmap v139: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:00:49.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:48 smithi107 ceph-mon[101102]: pgmap v139: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:00:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:48 smithi150 ceph-mon[105942]: pgmap v139: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:00:49.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:48 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:00:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2863836687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:50.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2863836687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:50.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2863836687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:50 smithi107 ceph-mon[101102]: pgmap v140: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:00:51.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:50 smithi107 ceph-mon[118900]: pgmap v140: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:00:51.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:50 smithi150 ceph-mon[105942]: pgmap v140: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T06:00:53.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:52 smithi107 ceph-mon[101102]: pgmap v141: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:00:53.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:52 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3382010831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:53.092 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:52 smithi107 ceph-mon[118900]: pgmap v141: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:00:53.092 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:52 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3382010831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:53.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:52 smithi150 ceph-mon[105942]: pgmap v141: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:00:53.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:52 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3382010831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:53.110 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE Bus STOPPING 2023-10-09T06:00:53.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.090+0000 7fa743d3e700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T06:00:53.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: ignoring --setuser ceph since I am not root 2023-10-09T06:00:53.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: ignoring --setgroup ceph since I am not root 2023-10-09T06:00:53.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.301+0000 7fd236bba1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:00:53.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:00:53.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE Bus STOPPED 2023-10-09T06:00:53.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE Bus STARTING 2023-10-09T06:00:53.674 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.409+0000 7fd236bba1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:00:53.674 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.541+0000 7fd236bba1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:00:53.674 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.670+0000 7fd236bba1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE Serving on http://:::9283 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: [09/Oct/2023:06:00:53] ENGINE Bus STARTED 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.527+0000 7f61e8476700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.527+0000 7f61e8476700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.527+0000 7f61e8476700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.527+0000 7f61e8476700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.527+0000 7f61e8476700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.807 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.530+0000 7f61eec83700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.808 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.530+0000 7f61eec83700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.808 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.530+0000 7f61eec83700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.808 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.530+0000 7f61eec83700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:53.808 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:53 smithi150 conmon[159842]: 2023-10-09T06:00:53.530+0000 7f61eec83700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: mgrmap e21: x(active, starting, since 0.0130737s) 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:00:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:00:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: Manager daemon x is now available 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.155 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:53 smithi107 conmon[101524]: 2023-10-09T06:00:53.923+0000 7fd236bba1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: mgrmap e21: x(active, starting, since 0.0130737s) 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:00:54.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: Manager daemon x is now available 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:54.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:53 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 172.21.15.107:0/3606816937' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.14454 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T06:00:54.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: mgrmap e21: x(active, starting, since 0.0130737s) 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:00:54.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: Manager daemon x is now available 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:00:54.163 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.163 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:53 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T06:00:54.745 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:54 smithi107 conmon[101524]: 2023-10-09T06:00:54.421+0000 7fd236bba1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:00:54.745 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:54 smithi107 conmon[101524]: 2023-10-09T06:00:54.554+0000 7fd236bba1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:00:55.099 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:54 smithi107 conmon[101524]: 2023-10-09T06:00:54.743+0000 7fd236bba1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:00:55.099 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:54 smithi107 conmon[101524]: 2023-10-09T06:00:54.879+0000 7fd236bba1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:00:55.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: Deploying cephadm binary to smithi107 2023-10-09T06:00:55.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: Deploying cephadm binary to smithi150 2023-10-09T06:00:55.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: mgrmap e22: x(active, since 1.02953s) 2023-10-09T06:00:55.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: pgmap v3: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:55.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3616748212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:55.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: Deploying cephadm binary to smithi107 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: Deploying cephadm binary to smithi150 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: mgrmap e22: x(active, since 1.02953s) 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: pgmap v3: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:55 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3616748212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:55.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: Deploying cephadm binary to smithi107 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: Deploying cephadm binary to smithi150 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: mgrmap e22: x(active, since 1.02953s) 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: pgmap v3: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:00:55.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:55 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3616748212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:55.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:55 smithi107 conmon[101524]: 2023-10-09T06:00:55.488+0000 7fd236bba1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:00:56.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:56 smithi107 conmon[101524]: 2023-10-09T06:00:56.284+0000 7fd236bba1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:00:56.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:56 smithi107 ceph-mon[118900]: pgmap v4: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:56.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:56 smithi107 ceph-mon[118900]: mgrmap e23: x(active, since 2s) 2023-10-09T06:00:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:56 smithi107 ceph-mon[101102]: pgmap v4: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:56.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:56 smithi107 ceph-mon[101102]: mgrmap e23: x(active, since 2s) 2023-10-09T06:00:56.417 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:56 smithi150 ceph-mon[105942]: pgmap v4: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:56.417 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:56 smithi150 ceph-mon[105942]: mgrmap e23: x(active, since 2s) 2023-10-09T06:00:56.903 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:56 smithi107 conmon[101524]: 2023-10-09T06:00:56.527+0000 7fd236bba1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:00:57.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[101102]: [09/Oct/2023:06:00:55] ENGINE Bus STARTING 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[101102]: [09/Oct/2023:06:00:55] ENGINE Serving on https://172.21.15.150:7150 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[101102]: [09/Oct/2023:06:00:55] ENGINE Serving on http://172.21.15.150:8765 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[101102]: [09/Oct/2023:06:00:55] ENGINE Bus STARTED 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[118900]: [09/Oct/2023:06:00:55] ENGINE Bus STARTING 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[118900]: [09/Oct/2023:06:00:55] ENGINE Serving on https://172.21.15.150:7150 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[118900]: [09/Oct/2023:06:00:55] ENGINE Serving on http://172.21.15.150:8765 2023-10-09T06:00:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:57 smithi107 ceph-mon[118900]: [09/Oct/2023:06:00:55] ENGINE Bus STARTED 2023-10-09T06:00:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:57 smithi150 ceph-mon[105942]: [09/Oct/2023:06:00:55] ENGINE Bus STARTING 2023-10-09T06:00:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:57 smithi150 ceph-mon[105942]: [09/Oct/2023:06:00:55] ENGINE Serving on https://172.21.15.150:7150 2023-10-09T06:00:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:57 smithi150 ceph-mon[105942]: [09/Oct/2023:06:00:55] ENGINE Serving on http://172.21.15.150:8765 2023-10-09T06:00:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:57 smithi150 ceph-mon[105942]: [09/Oct/2023:06:00:55] ENGINE Bus STARTED 2023-10-09T06:00:58.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:57 smithi107 conmon[101524]: 2023-10-09T06:00:57.801+0000 7fd236bba1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:00:58.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:57 smithi107 conmon[101524]: 2023-10-09T06:00:57.947+0000 7fd236bba1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:00:58.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:58 smithi107 conmon[101524]: 2023-10-09T06:00:58.216+0000 7fd236bba1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:00:58.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:58 smithi107 ceph-mon[101102]: pgmap v5: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:58.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:00:58 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3634782716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:58.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:58 smithi107 ceph-mon[118900]: pgmap v5: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:58.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:00:58 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3634782716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:58.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:58 smithi150 ceph-mon[105942]: pgmap v5: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:00:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:00:58 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3634782716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:00:59.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:58 smithi107 conmon[101524]: 2023-10-09T06:00:58.659+0000 7fd236bba1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:00:59.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:58 smithi107 conmon[101524]: 2023-10-09T06:00:58.804+0000 7fd236bba1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:00:59.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:00:58 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:00:58] "GET /metrics HTTP/1.1" 200 33231 "" "Prometheus/2.43.0" 2023-10-09T06:00:59.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:59 smithi107 conmon[101524]: 2023-10-09T06:00:59.243+0000 7fd236bba1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:01:00.153 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:00:59 smithi107 conmon[101524]: 2023-10-09T06:00:59.664+0000 7fd236bba1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:01:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:00 smithi150 ceph-mon[105942]: pgmap v6: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:01:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:00 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/4172056007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:00 smithi107 ceph-mon[101102]: pgmap v6: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:01:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:00 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/4172056007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:00 smithi107 ceph-mon[118900]: pgmap v6: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail 2023-10-09T06:01:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:00 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/4172056007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:01.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:00 smithi107 conmon[101524]: 2023-10-09T06:01:00.812+0000 7fd236bba1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:01:01.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:01 smithi107 conmon[101524]: 2023-10-09T06:01:01.007+0000 7fd236bba1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:01:01.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:01 smithi107 conmon[101524]: 2023-10-09T06:01:01.309+0000 7fd236bba1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:01:01.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:01 smithi107 conmon[101524]: 2023-10-09T06:01:01.499+0000 7fd236bba1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:01:02.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:01 smithi107 conmon[101524]: 2023-10-09T06:01:01.858+0000 7fd236bba1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:01:02.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[101102]: pgmap v7: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-09T06:01:02.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.654 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:02 smithi107 conmon[175101]: ts=2023-10-09T06:01:02.324Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi107.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.107:8443: connect: connection refused" 2023-10-09T06:01:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:02 smithi150 ceph-mon[105942]: pgmap v7: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:02 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:02 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:02 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:02 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[118900]: pgmap v7: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:02 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.616 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: 2023-10-09T06:01:03.191+0000 7fd236bba1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:01:03.616 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: [09/Oct/2023:06:01:03] ENGINE Bus STARTING 2023-10-09T06:01:03.617 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: CherryPy Checker: 2023-10-09T06:01:03.617 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: The Application mounted at '' has an empty config. 2023-10-09T06:01:03.617 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: 2023-10-09T06:01:03.617 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: [09/Oct/2023:06:01:03] ENGINE Serving on http://:::9283 2023-10-09T06:01:03.617 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[101524]: [09/Oct/2023:06:01:03] ENGINE Bus STARTED 2023-10-09T06:01:03.617 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:03 smithi107 conmon[175101]: ts=2023-10-09T06:01:03.616Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=3 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi107 to 2500M 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3134082275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: pgmap v8: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: Standby manager daemon y started 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:01:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi107 to 2500M 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3134082275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: pgmap v8: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: Standby manager daemon y started 2023-10-09T06:01:03.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:01:03.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:01:03.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:01:03.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:01:03.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:03.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:03 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:04.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:04.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:04.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi107 to 2500M 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3134082275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:04.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: pgmap v8: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: Standby manager daemon y started 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/146284839' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:04.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:03 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:05.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:05 smithi150 ceph-mon[105942]: mgrmap e24: x(active, since 10s), standbys: y 2023-10-09T06:01:05.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:05 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:01:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:05 smithi107 ceph-mon[101102]: mgrmap e24: x(active, since 10s), standbys: y 2023-10-09T06:01:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:05 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:01:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:05 smithi107 ceph-mon[118900]: mgrmap e24: x(active, since 10s), standbys: y 2023-10-09T06:01:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:05 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:01:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:06 smithi107 ceph-mon[118900]: pgmap v9: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:01:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:06 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/130173149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:06 smithi107 ceph-mon[101102]: pgmap v9: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:01:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:06 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/130173149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:06.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:06 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:01:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:06 smithi150 ceph-mon[105942]: pgmap v9: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T06:01:06.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:06 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/130173149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:08.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:08 smithi107 ceph-mon[118900]: pgmap v10: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:08 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3949197823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:08.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:08 smithi107 ceph-mon[101102]: pgmap v10: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:08.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:08 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3949197823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:08 smithi150 ceph-mon[105942]: pgmap v10: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:08 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3949197823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:09.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:08 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:08] "GET /metrics HTTP/1.1" 200 36166 "" "Prometheus/2.43.0" 2023-10-09T06:01:09.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:09.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:09.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:10.468 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:10 smithi150 ceph-mon[105942]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:10.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:10 smithi107 ceph-mon[118900]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:10.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:10 smithi107 ceph-mon[101102]: pgmap v11: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T06:01:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:11 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2556919472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:11 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2556919472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:11 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2556919472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T06:01:12.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:12 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:12 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:01:13.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[118900]: Adjusting osd_memory_target on smithi150 to 2500M 2023-10-09T06:01:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:01:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:01:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[101102]: Adjusting osd_memory_target on smithi150 to 2500M 2023-10-09T06:01:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:01:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:13 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:01:13.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:13 smithi150 ceph-mon[105942]: Adjusting osd_memory_target on smithi150 to 2500M 2023-10-09T06:01:13.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:13 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:01:13.664 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:13 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:01:14.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/571143209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:14.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/571143209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:14 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/571143209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:14.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:14 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:15 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:15 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:16 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:01:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[101102]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:01:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[101102]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T06:01:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[101102]: pgmap v14: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-09T06:01:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2457201975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[118900]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:01:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[118900]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T06:01:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[118900]: pgmap v14: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-09T06:01:16.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:16 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2457201975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:16 smithi150 ceph-mon[105942]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T06:01:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:16 smithi150 ceph-mon[105942]: Reconfiguring daemon alertmanager.a on smithi107 2023-10-09T06:01:16.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:16 smithi150 ceph-mon[105942]: pgmap v14: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-09T06:01:16.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:16 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2457201975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:18.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:18 smithi107 ceph-mon[101102]: pgmap v15: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:18.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:18 smithi107 ceph-mon[118900]: pgmap v15: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:18.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:18 smithi150 ceph-mon[105942]: pgmap v15: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:19.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:18 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:18] "GET /metrics HTTP/1.1" 200 36166 "" "Prometheus/2.43.0" 2023-10-09T06:01:19.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:19 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/354069299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:19.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:19 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/354069299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:19.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:19 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/354069299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:20 smithi107 ceph-mon[101102]: pgmap v16: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:20 smithi107 ceph-mon[118900]: pgmap v16: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:20.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:20 smithi150 ceph-mon[105942]: pgmap v16: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-09T06:01:21.400 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:20 smithi107 systemd[1]: Stopping Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:21.692 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:21 smithi107 conmon[175101]: ts=2023-10-09T06:01:21.412Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:01:21.692 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:21 smithi107 bash[197822]: 2b73cb66110c15baf49df58fd17ccb85aaa3cc99dd69d97df51239f558576a79 2023-10-09T06:01:22.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:22 smithi107 ceph-mon[101102]: pgmap v17: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:01:22.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:22 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2459016212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:22.653 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:22 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:01:22.653 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:22 smithi107 systemd[1]: Stopped Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:22.653 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:22 smithi107 systemd[1]: Starting Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:22.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:22 smithi107 ceph-mon[118900]: pgmap v17: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:01:22.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:22 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2459016212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:22 smithi150 ceph-mon[105942]: pgmap v17: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T06:01:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:22 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2459016212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 bash[198136]: 0715bcb46fa19d9241db61283fbe5ef20473f0dcc55ca2305582acde0cd05c1c 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 systemd[1]: Started Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.434Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.434Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.435Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.107 port=9094 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.440Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.476Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.476Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T06:01:23.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.484Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T06:01:23.903 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:23 smithi107 conmon[198176]: ts=2023-10-09T06:01:23.484Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T06:01:24.401 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:24 smithi107 systemd[1]: Stopping Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[118900]: pgmap v18: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:01:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2674788786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:24 smithi150 ceph-mon[105942]: pgmap v18: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:01:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:24 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:24 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:24 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:24 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2674788786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:24.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[101102]: pgmap v18: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T06:01:24.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:24.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:24.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:24 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2674788786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:25.079 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:24 smithi107 bash[198426]: da46b9f4dd3d39a31e5eb467084f9f5c9a02f61ed1953a47b4fdc7b53e421b36 2023-10-09T06:01:25.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:25 smithi107 ceph-mon[118900]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:01:25.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:25 smithi107 ceph-mon[118900]: Reconfiguring daemon node-exporter.a on smithi107 2023-10-09T06:01:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:25 smithi150 ceph-mon[105942]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:01:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:25 smithi150 ceph-mon[105942]: Reconfiguring daemon node-exporter.a on smithi107 2023-10-09T06:01:25.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:25 smithi107 ceph-mon[101102]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T06:01:25.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:25 smithi107 ceph-mon[101102]: Reconfiguring daemon node-exporter.a on smithi107 2023-10-09T06:01:25.788 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:25 smithi107 conmon[198176]: ts=2023-10-09T06:01:25.441Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000701924s 2023-10-09T06:01:26.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:26 smithi107 ceph-mon[101102]: pgmap v19: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:01:26.110 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:25 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:01:26.110 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:25 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:01:26.110 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:25 smithi107 systemd[1]: Stopped Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:26.110 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:25 smithi107 systemd[1]: Starting Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:26.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[101524]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:01:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:26 smithi107 ceph-mon[118900]: pgmap v19: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:01:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:26 smithi150 ceph-mon[105942]: pgmap v19: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.745Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.745Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.747Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T06:01:27.103 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T06:01:27.104 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T06:01:27.105 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T06:01:27.106 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T06:01:27.107 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.748Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T06:01:27.108 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.749Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T06:01:27.109 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 conmon[198795]: ts=2023-10-09T06:01:26.749Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T06:01:27.109 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 bash[198739]: 32eddd377f02f6032d9a8b2ebb14f528ed9022108113b5ccc6f5f8c5c1936f5f 2023-10-09T06:01:27.109 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:01:26 smithi107 systemd[1]: Started Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:27.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/824843200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:27.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:27.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:27.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/824843200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:27.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:27.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:27 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:27.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:27 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/824843200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:27.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:27 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:27.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:27 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:28.250 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:28 smithi150 ceph-mon[105942]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:01:28.250 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:28 smithi150 ceph-mon[105942]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:01:28.251 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:28 smithi150 ceph-mon[105942]: pgmap v20: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:28.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[101102]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:01:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[101102]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:01:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[101102]: pgmap v20: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[118900]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:01:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[118900]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:01:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:28 smithi107 ceph-mon[118900]: pgmap v20: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:29.158 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:28 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:28] "GET /metrics HTTP/1.1" 200 36164 "" "Prometheus/2.43.0" 2023-10-09T06:01:30.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:30 smithi107 ceph-mon[118900]: pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:30.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:30 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/572305910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:30.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:30 smithi107 ceph-mon[101102]: pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:30.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:30 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/572305910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:30.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:30 smithi150 ceph-mon[105942]: pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:01:30.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:30 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/572305910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:32.488 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 systemd[1]: Stopping Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:32.488 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:32 smithi150 ceph-mon[105942]: pgmap v22: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-09T06:01:32.488 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3607883254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:32 smithi107 ceph-mon[118900]: pgmap v22: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-09T06:01:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3607883254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:32.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:32 smithi107 ceph-mon[101102]: pgmap v22: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-09T06:01:32.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:32 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3607883254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:33.332 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.986Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T06:01:33.333 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.987Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T06:01:33.334 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.987Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:01:33.334 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.987Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T06:01:33.334 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:32 smithi150 conmon[154574]: ts=2023-10-09T06:01:32.987Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T06:01:33.334 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:33 smithi150 bash[172624]: d2a78d53b2412858e9ddb4c1bad17c69167c69baeb73cc34bc4d9a2f4b6a3540 2023-10-09T06:01:33.905 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:33 smithi107 conmon[198176]: ts=2023-10-09T06:01:33.445Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004510096s 2023-10-09T06:01:34.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:34 smithi107 ceph-mon[101102]: pgmap v23: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:34 smithi107 ceph-mon[118900]: pgmap v23: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:34 smithi150 ceph-mon[105942]: pgmap v23: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:34.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:33 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:01:34.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:33 smithi150 systemd[1]: Stopped Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:34.404 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:33 smithi150 systemd[1]: Starting Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:35.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3436164623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:35.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3436164623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:35 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:35 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3436164623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:35 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:35 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 bash[172937]: 2313f18f1dcb5051a3abe20ead22f9cc9173f28e46617e5c93f882adbe752150 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.926Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.926Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.926Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi150 (none))" 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.926Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.926Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 systemd[1]: Started Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:35.408 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.928Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.929Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.944Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.944Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.944Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.965Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=20.795003ms 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.965Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.966Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.998Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.998Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:34 smithi150 conmon[172976]: ts=2023-10-09T06:01:34.998Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=76.882µs wal_replay_duration=33.291203ms wbl_replay_duration=122ns total_replay_duration=54.235713ms 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.007Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T06:01:35.409 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.007Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T06:01:35.410 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.007Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T06:01:35.410 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.035Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=28.111363ms db_storage=996ns remote_storage=1.472µs web_handler=456ns query_engine=940ns scrape=571.641µs scrape_sd=131.837µs notify=15.627µs notify_sd=13.033µs rules=27.018708ms tracing=7.544µs 2023-10-09T06:01:35.410 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.036Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T06:01:35.410 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:35 smithi150 conmon[172976]: ts=2023-10-09T06:01:35.036Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T06:01:36.120 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:35 smithi150 systemd[1]: Stopping Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:36.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:36 smithi150 ceph-mon[105942]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:01:36.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:36 smithi150 ceph-mon[105942]: Reconfiguring daemon node-exporter.b on smithi150 2023-10-09T06:01:36.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:36 smithi150 ceph-mon[105942]: pgmap v24: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[101102]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[101102]: Reconfiguring daemon node-exporter.b on smithi150 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[101102]: pgmap v24: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[118900]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[118900]: Reconfiguring daemon node-exporter.b on smithi150 2023-10-09T06:01:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:36 smithi107 ceph-mon[118900]: pgmap v24: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:36.407 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:36 smithi150 bash[173199]: 3ead6ee07e6f1295a28fd8c279e614e6c30e4c097027ae88e163730022f90f9f 2023-10-09T06:01:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:37 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3730061768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:37 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3730061768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:37 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3730061768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:37.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:37 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:01:37.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:37 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:01:37.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:37 smithi150 systemd[1]: Stopped Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:37.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:37 smithi150 systemd[1]: Starting Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:38.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:38 smithi107 ceph-mon[101102]: pgmap v25: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:38 smithi107 ceph-mon[118900]: pgmap v25: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:38.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:38 smithi150 ceph-mon[105942]: pgmap v25: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:38.907 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.577Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T06:01:38.907 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.578Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T06:01:38.908 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T06:01:38.909 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T06:01:38.910 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T06:01:38.911 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.583Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.584Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 conmon[173550]: ts=2023-10-09T06:01:38.584Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 bash[173512]: 6b4c6b51266f7802bc2eea23772890cbf80cd043873bf983fb67ec63394cc0a6 2023-10-09T06:01:38.912 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:01:38 smithi150 systemd[1]: Started Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:39 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:39 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:39 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:39.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:39.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:39.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:39 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:40.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:40 smithi150 ceph-mon[105942]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:01:40.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:40 smithi150 ceph-mon[105942]: Reconfiguring daemon grafana.a on smithi150 2023-10-09T06:01:40.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:40 smithi150 ceph-mon[105942]: pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:40.143 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:40 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/790447471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[101102]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[101102]: Reconfiguring daemon grafana.a on smithi150 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[101102]: pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/790447471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[118900]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[118900]: Reconfiguring daemon grafana.a on smithi150 2023-10-09T06:01:40.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[118900]: pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T06:01:40.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:40 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/790447471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:42 smithi107 ceph-mon[101102]: pgmap v27: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2023-10-09T06:01:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:42 smithi107 ceph-mon[118900]: pgmap v27: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2023-10-09T06:01:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:42 smithi150 ceph-mon[105942]: pgmap v27: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2023-10-09T06:01:42.910 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:42 smithi150 systemd[1]: Stopping Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:43.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:43 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/84313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:43.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:43 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/84313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:43 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/84313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:43.404 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:43 smithi150 bash[174004]: eaf4772a342e6dae5240371e1e42448adaf05fa001bfcde0fa9062a3b50607c0 2023-10-09T06:01:43.406 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:43 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:43] "GET /metrics HTTP/1.1" 200 36165 "" "Prometheus/2.43.0" 2023-10-09T06:01:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:44 smithi150 ceph-mon[105942]: pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T06:01:44.404 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:44 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:01:44.404 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:44 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T06:01:44.405 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:44 smithi150 systemd[1]: Stopped Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:44.405 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:44 smithi150 systemd[1]: Starting Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:44 smithi107 ceph-mon[101102]: pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T06:01:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:44 smithi107 ceph-mon[118900]: pgmap v28: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T06:01:45.369 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:45 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/88851667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:45.369 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STOPPING 2023-10-09T06:01:45.370 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:01:45.370 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STOPPED 2023-10-09T06:01:45.370 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STARTING 2023-10-09T06:01:45.370 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 bash[174319]: 96db62cb38d01fc96d6785a06c0a792a08ec563c8c0f3426a54991f086275416 2023-10-09T06:01:45.370 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 systemd[1]: Started Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:45 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/88851667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:45 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/88851667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Serving on http://:::9283 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STARTED 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STOPPING 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STOPPED 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STARTING 2023-10-09T06:01:45.655 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Serving on http://:::9283 2023-10-09T06:01:45.656 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[159842]: [09/Oct/2023:06:01:45] ENGINE Bus STARTED 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.420849758Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421158094Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421187459Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421199557Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421214553Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421229238Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.42123929Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421249396Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421264783Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421274667Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T06:01:45.658 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421284858Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421294568Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.42130733Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=settings t=2023-10-09T06:01:45.421317957Z level=info msg="App mode production" 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=sqlstore t=2023-10-09T06:01:45.421398469Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=sqlstore t=2023-10-09T06:01:45.42142458Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=migrator t=2023-10-09T06:01:45.446688861Z level=info msg="Starting DB migrations" 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=migrator t=2023-10-09T06:01:45.455505263Z level=info msg="migrations completed" performed=0 skipped=481 duration=843.221µs 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=sqlstore t=2023-10-09T06:01:45.456119283Z level=info msg="Created default organization" 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=plugin.loader t=2023-10-09T06:01:45.551501846Z level=info msg="Plugin registered" pluginID=input 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=plugin.loader t=2023-10-09T06:01:45.591501729Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T06:01:45.659 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=plugin.loader t=2023-10-09T06:01:45.591538473Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T06:01:45.660 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=secrets t=2023-10-09T06:01:45.591592348Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T06:01:45.660 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=query_data t=2023-10-09T06:01:45.593825479Z level=info msg="Query Service initialization" 2023-10-09T06:01:45.660 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:45 smithi150 conmon[174357]: logger=live.push_http t=2023-10-09T06:01:45.59766188Z level=info msg="Live Push Gateway initialization" 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: pgmap v29: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: pgmap v29: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:46 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: pgmap v29: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150:9095"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T06:01:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:46.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:46.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:46 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:46 smithi150 conmon[174357]: logger=infra.usagestats.collector t=2023-10-09T06:01:46.795855193Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:46 smithi150 conmon[174357]: logger=provisioning.datasources t=2023-10-09T06:01:46.938388995Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:46 smithi150 conmon[174357]: logger=provisioning.datasources t=2023-10-09T06:01:46.938431575Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:46 smithi150 conmon[174357]: logger=provisioning.datasources t=2023-10-09T06:01:46.949132345Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=provisioning.alerting t=2023-10-09T06:01:47.144073435Z level=info msg="starting to provision alerting" 2023-10-09T06:01:47.148 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=provisioning.alerting t=2023-10-09T06:01:47.144120828Z level=info msg="finished to provision alerting" 2023-10-09T06:01:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:47 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:47 smithi150 ceph-mon[105942]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:47.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:47 smithi150 ceph-mon[105942]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=http.server t=2023-10-09T06:01:47.146559285Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=ngalert.state.manager t=2023-10-09T06:01:47.146877759Z level=info msg="Warming state cache for startup" 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=grafanaStorageLogger t=2023-10-09T06:01:47.147910534Z level=info msg="storage starting" 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=ngalert.state.manager t=2023-10-09T06:01:47.298345118Z level=info msg="State cache has been initialized" states=0 duration=151.462228ms 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=ticker t=2023-10-09T06:01:47.298449706Z level=info msg=starting first_tick=2023-10-09T06:01:50Z 2023-10-09T06:01:47.409 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:01:47 smithi150 conmon[174357]: logger=ngalert.multiorg.alertmanager t=2023-10-09T06:01:47.298473343Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[101102]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[101102]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi150:3000"}]: dispatch 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[118900]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:47 smithi107 ceph-mon[118900]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T06:01:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:48 smithi107 ceph-mon[101102]: pgmap v30: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:48 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1546063557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:48 smithi107 ceph-mon[118900]: pgmap v30: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:48 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1546063557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:48 smithi150 ceph-mon[105942]: pgmap v30: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:48 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1546063557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:49.158 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:48 smithi150 conmon[172976]: ts=2023-10-09T06:01:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:50.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:50.544 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.544 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.544 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:01:50.544 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:50 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:01:50.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:50 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[101102]: Upgrade: Updating mgr.y 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[101102]: Deploying daemon mgr.y on smithi107 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2589040635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[118900]: Upgrade: Updating mgr.y 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[118900]: Deploying daemon mgr.y on smithi107 2023-10-09T06:01:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:51 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2589040635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:51.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:51 smithi150 ceph-mon[105942]: Upgrade: Updating mgr.y 2023-10-09T06:01:51.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:51 smithi150 ceph-mon[105942]: Deploying daemon mgr.y on smithi107 2023-10-09T06:01:51.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:51 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2589040635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:52.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:51 smithi107 systemd[1]: Stopping Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:52.587 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:52 smithi107 ceph-mon[101102]: pgmap v32: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T06:01:52.588 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:52 smithi107 bash[199638]: 64caccec47d25147b3079387044924a81e2b54f14ecf9b68f04d7bae65fa598b 2023-10-09T06:01:52.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:52 smithi107 ceph-mon[118900]: pgmap v32: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T06:01:52.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:52 smithi150 ceph-mon[105942]: pgmap v32: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-10-09T06:01:53.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:53 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1018680963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:53.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:53 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:53.408 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:01:53 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:01:53] "GET /metrics HTTP/1.1" 200 36165 "" "Prometheus/2.43.0" 2023-10-09T06:01:53.507 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:53 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1018680963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:53.508 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:53 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:53.510 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:53 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1018680963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:53.510 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:53 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:01:53.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:53 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:01:53.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:53 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T06:01:53.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:53 smithi107 systemd[1]: Stopped Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:53.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:53 smithi107 systemd[1]: Starting Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:01:53.902 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:01:53 smithi107 conmon[198176]: ts=2023-10-09T06:01:53.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi107.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.107:8443: connect: connection refused" 2023-10-09T06:01:54.517 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:54 smithi107 ceph-mon[101102]: pgmap v33: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:54.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:54 smithi107 ceph-mon[118900]: pgmap v33: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:54.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:54 smithi150 ceph-mon[105942]: pgmap v33: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:54.985 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:54 smithi107 bash[200001]: 19d12e0e339c004167a13acd9251b923174db62531030bd0b8ccad1330d3d5aa 2023-10-09T06:01:54.985 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:54 smithi107 systemd[1]: Started Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:01:54.985 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:54 smithi107 conmon[200038]: 2023-10-09T06:01:54.863+0000 7faeb22f9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:01:55.300 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:54 smithi107 conmon[200038]: 2023-10-09T06:01:54.984+0000 7faeb22f9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:01:55.301 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:55 smithi107 conmon[200038]: 2023-10-09T06:01:55.128+0000 7faeb22f9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:01:55.301 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:55 smithi107 conmon[200038]: 2023-10-09T06:01:55.299+0000 7faeb22f9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:01:55.580 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:55 smithi107 conmon[200038]: 2023-10-09T06:01:55.575+0000 7faeb22f9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[101102]: pgmap v34: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/605338233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:55.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:55.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:55.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[118900]: pgmap v34: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:01:55.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:55 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/605338233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:56.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:55 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:56.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:55 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:01:56.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:55 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:01:56.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:55 smithi150 ceph-mon[105942]: pgmap v34: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2023-10-09T06:01:56.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:55 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/605338233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:56.523 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:56 smithi107 conmon[200038]: 2023-10-09T06:01:56.189+0000 7faeb22f9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:01:56.524 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:56 smithi107 conmon[200038]: 2023-10-09T06:01:56.386+0000 7faeb22f9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:01:56.524 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:56 smithi107 conmon[200038]: 2023-10-09T06:01:56.522+0000 7faeb22f9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:01:56.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:56 smithi107 conmon[200038]: 2023-10-09T06:01:56.670+0000 7faeb22f9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:01:57.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:57 smithi107 conmon[200038]: 2023-10-09T06:01:57.359+0000 7faeb22f9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:01:58.527 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:58 smithi107 ceph-mon[101102]: pgmap v35: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:58.528 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:58 smithi107 conmon[200038]: 2023-10-09T06:01:58.382+0000 7faeb22f9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:01:58.528 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:58 smithi107 conmon[200038]: 2023-10-09T06:01:58.524+0000 7faeb22f9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:01:58.528 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:58 smithi107 ceph-mon[118900]: pgmap v35: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:58.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:58 smithi150 ceph-mon[105942]: pgmap v35: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:01:59.157 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:01:58 smithi150 conmon[172976]: ts=2023-10-09T06:01:58.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:01:59.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:01:59 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3789497362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:01:59 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3789497362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:01:59.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:01:59 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3789497362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:00.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:59 smithi107 conmon[200038]: 2023-10-09T06:01:59.745+0000 7faeb22f9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:00.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:01:59 smithi107 conmon[200038]: 2023-10-09T06:01:59.888+0000 7faeb22f9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:02:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[101102]: pgmap v36: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:02:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:00 smithi150 ceph-mon[105942]: pgmap v36: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:02:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:00 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:00 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:00 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:00 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.655 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:00 smithi107 conmon[200038]: 2023-10-09T06:02:00.186+0000 7faeb22f9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:00.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[118900]: pgmap v36: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:02:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:00.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:00 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.000 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:00 smithi107 conmon[200038]: 2023-10-09T06:02:00.649+0000 7faeb22f9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:02:01.001 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:00 smithi107 conmon[200038]: 2023-10-09T06:02:00.996+0000 7faeb22f9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:02:01.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3777785655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:01.250 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.250 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.250 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3777785655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:01.250 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.250 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:01 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:01 smithi107 conmon[200038]: 2023-10-09T06:02:01.248+0000 7faeb22f9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:02:01.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:01 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3777785655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:01.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:01 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:01.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:01 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:02.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:01 smithi107 conmon[200038]: 2023-10-09T06:02:01.694+0000 7faeb22f9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:02:02.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:02 smithi107 ceph-mon[101102]: pgmap v37: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:02:02.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:02 smithi107 ceph-mon[118900]: pgmap v37: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:02:02.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:02 smithi150 ceph-mon[105942]: pgmap v37: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:02:03.010 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:02 smithi107 conmon[200038]: 2023-10-09T06:02:02.797+0000 7faeb22f9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:02:03.333 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[200038]: 2023-10-09T06:02:03.009+0000 7faeb22f9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:02:03.334 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[200038]: 2023-10-09T06:02:03.331+0000 7faeb22f9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:02:03.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:03 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:02:03] "GET /metrics HTTP/1.1" 200 36165 "" "Prometheus/2.43.0" 2023-10-09T06:02:03.650 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[200038]: 2023-10-09T06:02:03.517+0000 7faeb22f9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:02:03.651 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[198176]: ts=2023-10-09T06:02:03.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi107.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.107:8443: connect: connection refused" 2023-10-09T06:02:03.651 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[198176]: ts=2023-10-09T06:02:03.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi107.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.107:8443: connect: connection refused" 2023-10-09T06:02:04.085 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:03 smithi107 conmon[200038]: 2023-10-09T06:02:03.809+0000 7faeb22f9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:02:04.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:04 smithi107 ceph-mon[118900]: pgmap v38: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:02:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:04 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/270535929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:04 smithi107 ceph-mon[101102]: pgmap v38: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:02:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:04 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/270535929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:04.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:04 smithi150 ceph-mon[105942]: pgmap v38: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T06:02:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:04 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/270535929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: Standby manager daemon y restarted 2023-10-09T06:02:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: Standby manager daemon y started 2023-10-09T06:02:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:02:05.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:05.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:02:05.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: Standby manager daemon y restarted 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: Standby manager daemon y started 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:02:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:05 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:05.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: Standby manager daemon y restarted 2023-10-09T06:02:05.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: Standby manager daemon y started 2023-10-09T06:02:05.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T06:02:05.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:05 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:04 smithi107 conmon[200038]: 2023-10-09T06:02:04.982+0000 7faeb22f9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:04 smithi107 conmon[200038]: [09/Oct/2023:06:02:04] ENGINE Bus STARTING 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[200038]: CherryPy Checker: 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[200038]: The Application mounted at '' has an empty config. 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[200038]: 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[200038]: [09/Oct/2023:06:02:05] ENGINE Serving on http://:::9283 2023-10-09T06:02:05.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[200038]: [09/Oct/2023:06:02:05] ENGINE Bus STARTED 2023-10-09T06:02:06.136 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:06 smithi107 ceph-mon[101102]: mgrmap e25: x(active, since 72s), standbys: y 2023-10-09T06:02:06.136 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:06 smithi107 ceph-mon[101102]: pgmap v39: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:02:06.137 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:02:05 smithi107 conmon[198176]: ts=2023-10-09T06:02:05.884Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2023-10-09T06:02:06.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:06 smithi107 ceph-mon[118900]: mgrmap e25: x(active, since 72s), standbys: y 2023-10-09T06:02:06.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:06 smithi107 ceph-mon[118900]: pgmap v39: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:02:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:06 smithi150 ceph-mon[105942]: mgrmap e25: x(active, since 72s), standbys: y 2023-10-09T06:02:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:06 smithi150 ceph-mon[105942]: pgmap v39: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2023-10-09T06:02:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:07 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3595941982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:07.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:07 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3595941982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:07.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:07 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3595941982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:08.595 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:08 smithi107 ceph-mon[101102]: pgmap v40: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:08.600 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:08 smithi107 ceph-mon[118900]: pgmap v40: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:08.661 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:08 smithi150 ceph-mon[105942]: pgmap v40: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-10-09T06:02:09.014 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:08 smithi150 conmon[172976]: ts=2023-10-09T06:02:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:02:09.014 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.013+0000 7f6281ecf700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.24893 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: mgrmap e26: y(active, starting, since 0.0164546s) 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:02:09.270 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:02:09.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3462957419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: Manager daemon y is now available 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: ignoring --setuser ceph since I am not root 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: ignoring --setgroup ceph since I am not root 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 ceph-mgr[159865]: -- 172.21.15.150:0/2769935681 <== mon.1 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55eedd5f91e0 con 0x55eedd5d0c00 2023-10-09T06:02:09.272 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.267+0000 7f4f8251c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:02:09.330 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE Bus STOPPING 2023-10-09T06:02:09.331 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:02:09.331 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE Bus STOPPED 2023-10-09T06:02:09.331 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE Bus STARTING 2023-10-09T06:02:09.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:09.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.24893 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: mgrmap e26: y(active, starting, since 0.0164546s) 2023-10-09T06:02:09.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:02:09.333 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3462957419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: Manager daemon y is now available 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.334 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 172.21.15.150:0/731076951' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.24893 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: mgrmap e26: y(active, starting, since 0.0164546s) 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:02:09.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3462957419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:09.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: Manager daemon y is now available 2023-10-09T06:02:09.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:09 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:09.536 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.399+0000 7f4f8251c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:02:09.536 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.534+0000 7f4f8251c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:02:09.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.329+0000 7fae15736700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.329+0000 7fae15736700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.329+0000 7fae15736700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.329+0000 7fae15736700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.329+0000 7fae15736700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.343+0000 7fae0ef29700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.343+0000 7fae0ef29700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.343+0000 7fae0ef29700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.343+0000 7fae0ef29700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: 2023-10-09T06:02:09.343+0000 7fae0ef29700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE Serving on http://:::9283 2023-10-09T06:02:09.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:09 smithi107 conmon[200038]: [09/Oct/2023:06:02:09] ENGINE Bus STARTED 2023-10-09T06:02:09.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.679+0000 7f4f8251c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: mgrmap e27: y(active, since 1.027s) 2023-10-09T06:02:10.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:10 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.415 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:09 smithi150 conmon[159842]: 2023-10-09T06:02:09.959+0000 7f4f8251c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: mgrmap e27: y(active, since 1.027s) 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: mgrmap e27: y(active, since 1.027s) 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:10 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:10.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:10 smithi150 conmon[159842]: 2023-10-09T06:02:10.604+0000 7f4f8251c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:02:10.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:10 smithi150 conmon[159842]: 2023-10-09T06:02:10.821+0000 7f4f8251c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:11 smithi150 ceph-mon[105942]: [09/Oct/2023:06:02:09] ENGINE Bus STARTING 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:11 smithi150 ceph-mon[105942]: [09/Oct/2023:06:02:09] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:11 smithi150 ceph-mon[105942]: [09/Oct/2023:06:02:09] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:11 smithi150 ceph-mon[105942]: [09/Oct/2023:06:02:09] ENGINE Bus STARTED 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:11 smithi150 ceph-mon[105942]: pgmap v3: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:10 smithi150 conmon[159842]: 2023-10-09T06:02:10.974+0000 7f4f8251c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:02:11.413 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:11 smithi150 conmon[159842]: 2023-10-09T06:02:11.126+0000 7f4f8251c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:02:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[118900]: [09/Oct/2023:06:02:09] ENGINE Bus STARTING 2023-10-09T06:02:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[118900]: [09/Oct/2023:06:02:09] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:02:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[118900]: [09/Oct/2023:06:02:09] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:02:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[118900]: [09/Oct/2023:06:02:09] ENGINE Bus STARTED 2023-10-09T06:02:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[118900]: pgmap v3: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:11.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[101102]: [09/Oct/2023:06:02:09] ENGINE Bus STARTING 2023-10-09T06:02:11.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[101102]: [09/Oct/2023:06:02:09] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:02:11.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[101102]: [09/Oct/2023:06:02:09] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:02:11.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[101102]: [09/Oct/2023:06:02:09] ENGINE Bus STARTED 2023-10-09T06:02:11.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:11 smithi107 ceph-mon[101102]: pgmap v3: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:12.105 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:11 smithi150 conmon[159842]: 2023-10-09T06:02:11.765+0000 7f4f8251c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:02:12.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:12 smithi150 ceph-mon[105942]: pgmap v4: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:12.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:12 smithi150 ceph-mon[105942]: mgrmap e28: y(active, since 2s) 2023-10-09T06:02:12.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:12 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/4029911990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:12.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[101102]: pgmap v4: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[101102]: mgrmap e28: y(active, since 2s) 2023-10-09T06:02:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/4029911990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[118900]: pgmap v4: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[118900]: mgrmap e28: y(active, since 2s) 2023-10-09T06:02:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:12 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/4029911990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:13.155 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:12 smithi150 conmon[159842]: 2023-10-09T06:02:12.805+0000 7f4f8251c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:02:13.155 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:12 smithi150 conmon[159842]: 2023-10-09T06:02:12.949+0000 7f4f8251c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:02:14.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[118900]: pgmap v5: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:14.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[101102]: pgmap v5: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:14.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:14 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.662 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:14 smithi150 ceph-mon[105942]: pgmap v5: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:02:14.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:14 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.663 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:14 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:14.663 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:14 smithi150 conmon[159842]: 2023-10-09T06:02:14.285+0000 7f4f8251c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:14.663 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:14 smithi150 conmon[159842]: 2023-10-09T06:02:14.445+0000 7f4f8251c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:02:15.156 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:14 smithi150 conmon[159842]: 2023-10-09T06:02:14.745+0000 7f4f8251c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:15.554 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:15 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2533282133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:15.554 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:15 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.554 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:15 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.555 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:15 smithi150 conmon[159842]: 2023-10-09T06:02:15.224+0000 7f4f8251c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:02:15.555 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:15 smithi150 conmon[159842]: 2023-10-09T06:02:15.553+0000 7f4f8251c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2533282133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2533282133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:15 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:15.810 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:15 smithi150 conmon[159842]: 2023-10-09T06:02:15.805+0000 7f4f8251c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:02:16.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:16 smithi150 conmon[159842]: 2023-10-09T06:02:16.272+0000 7f4f8251c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:02:16.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:16 smithi150 ceph-mon[105942]: pgmap v6: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2023-10-09T06:02:16.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:16 smithi107 ceph-mon[101102]: pgmap v6: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2023-10-09T06:02:16.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:16 smithi107 ceph-mon[118900]: pgmap v6: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 44 op/s 2023-10-09T06:02:17.862 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:17 smithi150 conmon[159842]: 2023-10-09T06:02:17.407+0000 7f4f8251c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:02:17.862 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:17 smithi150 conmon[159842]: 2023-10-09T06:02:17.577+0000 7f4f8251c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:02:17.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:17 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:17.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:17 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:17.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:17 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3160534397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:17.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:17 smithi150 ceph-mon[105942]: pgmap v7: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s 2023-10-09T06:02:18.125 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:17 smithi150 conmon[159842]: 2023-10-09T06:02:17.921+0000 7f4f8251c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:02:18.125 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:18 smithi150 conmon[159842]: 2023-10-09T06:02:18.123+0000 7f4f8251c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3160534397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[101102]: pgmap v7: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3160534397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:18.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:17 smithi107 ceph-mon[118900]: pgmap v7: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s 2023-10-09T06:02:18.907 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:18 smithi150 conmon[159842]: 2023-10-09T06:02:18.496+0000 7f4f8251c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:02:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:19 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:19.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:19 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:19.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:19 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:19.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:19 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:19 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:19.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:19 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:20.156 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: 2023-10-09T06:02:19.838+0000 7f4f8251c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: [09/Oct/2023:06:02:19] ENGINE Bus STARTING 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: CherryPy Checker: 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: The Application mounted at '' has an empty config. 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: [09/Oct/2023:06:02:19] ENGINE Serving on http://:::9283 2023-10-09T06:02:20.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:19 smithi150 conmon[159842]: [09/Oct/2023:06:02:19] ENGINE Bus STARTED 2023-10-09T06:02:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: pgmap v8: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-10-09T06:02:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3444862052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: Standby manager daemon x started 2023-10-09T06:02:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: pgmap v8: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3444862052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:20.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:20 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: pgmap v8: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3444862052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:20.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:20 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2936563522' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:21.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:21 smithi107 ceph-mon[101102]: mgrmap e29: y(active, since 11s), standbys: x 2023-10-09T06:02:21.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:21 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:02:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:21 smithi107 ceph-mon[118900]: mgrmap e29: y(active, since 11s), standbys: x 2023-10-09T06:02:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:21 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:02:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:21 smithi150 ceph-mon[105942]: mgrmap e29: y(active, since 11s), standbys: x 2023-10-09T06:02:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:21 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:02:22.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: pgmap v9: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T06:02:22.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:22.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:22.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1642392214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:22.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: pgmap v9: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T06:02:22.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:22.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:22.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:22 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1642392214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: pgmap v9: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:22.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:22 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1642392214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:23.387 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:23 smithi150 conmon[159842]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:02:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T06:02:23.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:02:23.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:02:23.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:23.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:23 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:23 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:23 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:23 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:23.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:23 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: pgmap v10: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[118900]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: pgmap v10: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:02:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:24 smithi107 ceph-mon[101102]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: pgmap v10: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:25.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:25.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T06:02:25.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:24 smithi150 ceph-mon[105942]: Reconfiguring daemon prometheus.a on smithi150 2023-10-09T06:02:26.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:25 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/54466327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:25 smithi107 ceph-mon[101102]: pgmap v11: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T06:02:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:25 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/54466327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:25 smithi107 ceph-mon[118900]: pgmap v11: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T06:02:26.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:25 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/54466327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:25 smithi150 ceph-mon[105942]: pgmap v11: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-10-09T06:02:28.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:28 smithi107 ceph-mon[101102]: pgmap v12: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T06:02:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:28 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2641122234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:28 smithi107 ceph-mon[118900]: pgmap v12: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T06:02:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:28 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2641122234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:28.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:28 smithi150 ceph-mon[105942]: pgmap v12: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T06:02:28.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:28 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2641122234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:30 smithi107 ceph-mon[118900]: pgmap v13: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T06:02:30.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 systemd[1]: Stopping Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:02:30.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:30 smithi150 ceph-mon[105942]: pgmap v13: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T06:02:30.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:30 smithi107 ceph-mon[101102]: pgmap v13: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.466Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:02:30.795 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.467Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.472Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.472Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 conmon[172976]: ts=2023-10-09T06:02:30.472Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T06:02:30.796 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:30 smithi150 bash[178154]: 2313f18f1dcb5051a3abe20ead22f9cc9173f28e46617e5c93f882adbe752150 2023-10-09T06:02:31.135 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2185651052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:31 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2185651052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:31.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2185651052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:31.907 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:31 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T06:02:31.907 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:31 smithi150 systemd[1]: Stopped Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:02:31.907 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:31 smithi150 systemd[1]: Starting Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:02:32.394 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:32 smithi150 ceph-mon[105942]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T06:02:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:32 smithi107 ceph-mon[118900]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T06:02:32.402 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 bash[178462]: 3a1d67d52b6478afe80dfee423b2dd6f7ee638c4c0887e41753851459e7a7fe8 2023-10-09T06:02:32.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:32 smithi107 ceph-mon[101102]: pgmap v14: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T06:02:32.655 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 systemd[1]: Started Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.414Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.414Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.414Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi150 (none))" 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.414Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.414Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T06:02:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.416Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.421Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.445Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.445Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.445Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.447Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.248339ms 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.447Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T06:02:32.657 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.447Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.476Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.479Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.479Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.479Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=61.46µs wal_replay_duration=31.862862ms wbl_replay_duration=128ns total_replay_duration=34.235425ms 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.483Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.483Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T06:02:32.658 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.483Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T06:02:32.659 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.515Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=32.558997ms db_storage=1.022µs remote_storage=1.41µs web_handler=457ns query_engine=1.26µs scrape=553.633µs scrape_sd=118.712µs notify=24.449µs notify_sd=19.929µs rules=31.535472ms tracing=7.875µs 2023-10-09T06:02:32.659 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.516Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T06:02:32.659 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:32 smithi150 conmon[178500]: ts=2023-10-09T06:02:32.516Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T06:02:32.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE Bus STOPPING 2023-10-09T06:02:32.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:02:32.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE Bus STOPPED 2023-10-09T06:02:32.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE Bus STARTING 2023-10-09T06:02:32.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE Serving on http://:::9283 2023-10-09T06:02:32.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:32 smithi107 conmon[200038]: [09/Oct/2023:06:02:32] ENGINE Bus STARTED 2023-10-09T06:02:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2325539244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:33.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:33 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2325539244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:33.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi150.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:33.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:33 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2325539244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:34 smithi107 ceph-mon[101102]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:02:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:34 smithi107 ceph-mon[118900]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:02:34.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:34 smithi150 ceph-mon[105942]: pgmap v15: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-09T06:02:35.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:35 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:35 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:35 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:02:35.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:35 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:35.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:35 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3886379945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:35.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:02:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3886379945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:35.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:02:35.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:02:35.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:35.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:35 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3886379945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:36.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[118900]: pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T06:02:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[118900]: Upgrade: Updating mgr.x 2023-10-09T06:02:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[118900]: Deploying daemon mgr.x on smithi150 2023-10-09T06:02:36.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[101102]: pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T06:02:36.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[101102]: Upgrade: Updating mgr.x 2023-10-09T06:02:36.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:36 smithi107 ceph-mon[101102]: Deploying daemon mgr.x on smithi150 2023-10-09T06:02:36.910 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:36 smithi150 ceph-mon[105942]: pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-09T06:02:36.910 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:36 smithi150 ceph-mon[105942]: Upgrade: Updating mgr.x 2023-10-09T06:02:36.910 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:36 smithi150 ceph-mon[105942]: Deploying daemon mgr.x on smithi150 2023-10-09T06:02:37.340 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:37 smithi150 systemd[1]: Stopping Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:02:38.405 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:38 smithi150 bash[179110]: 157473f4f8c95f65504171d186c1765d696c18e45b058058ce6579ebc5a61b17 2023-10-09T06:02:38.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:38 smithi107 ceph-mon[101102]: pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-09T06:02:38.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:38 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1558272448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:38 smithi150 ceph-mon[105942]: pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-09T06:02:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:38 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1558272448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:38.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:38 smithi107 ceph-mon[118900]: pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-09T06:02:38.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:38 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1558272448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:39.410 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:38 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:02:39.410 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:38 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T06:02:39.410 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:38 smithi150 systemd[1]: Stopped Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:02:39.410 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:39 smithi150 systemd[1]: Starting Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:02:39.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:39 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:39.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:39 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:39.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:39 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:40.620 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 bash[179445]: 7326a92d4fbc8dd0fcc80de4dd70bc009038dd5e485de432e175fff02109a587 2023-10-09T06:02:40.620 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 ceph-mgr[179510]: -- 172.21.15.150:0/3088609304 <== mon.0 v2:172.21.15.107:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556c68da71e0 con 0x556c68d7f000 2023-10-09T06:02:40.620 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 systemd[1]: Started Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:02:40.621 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 conmon[179483]: 2023-10-09T06:02:40.471+0000 7f54da401200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:02:40.621 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:40 smithi150 ceph-mon[105942]: pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:02:40.621 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:40 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.621 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:40 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.621 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:40 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[101102]: pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[118900]: pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:40.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:40 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:40.911 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 conmon[179483]: 2023-10-09T06:02:40.609+0000 7f54da401200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:02:40.911 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 conmon[179483]: 2023-10-09T06:02:40.763+0000 7f54da401200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:02:41.226 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:40 smithi150 conmon[179483]: 2023-10-09T06:02:40.923+0000 7f54da401200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:02:41.226 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:41 smithi150 conmon[179483]: 2023-10-09T06:02:41.219+0000 7f54da401200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:02:41.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:41 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2579259796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:41.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:41 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:41 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2579259796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:41.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2579259796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:41.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:41 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:41.912 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:41 smithi150 conmon[179483]: 2023-10-09T06:02:41.843+0000 7f54da401200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:02:42.382 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:42 smithi150 conmon[179483]: 2023-10-09T06:02:42.065+0000 7f54da401200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:02:42.382 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:42 smithi150 conmon[179483]: 2023-10-09T06:02:42.218+0000 7f54da401200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:02:42.382 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:42 smithi150 conmon[179483]: 2023-10-09T06:02:42.380+0000 7f54da401200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:02:42.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:02:42] "GET /metrics HTTP/1.1" 200 36160 "" "Prometheus/2.43.0" 2023-10-09T06:02:42.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:42 smithi107 ceph-mon[118900]: pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-09T06:02:42.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:42 smithi107 ceph-mon[101102]: pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-09T06:02:42.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:42 smithi150 ceph-mon[105942]: pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-09T06:02:43.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:43 smithi150 conmon[179483]: 2023-10-09T06:02:43.104+0000 7f54da401200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:02:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:44 smithi107 ceph-mon[101102]: pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:02:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:44 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/312133916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:44 smithi107 ceph-mon[118900]: pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:02:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:44 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/312133916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:44.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:44 smithi150 conmon[179483]: 2023-10-09T06:02:44.190+0000 7f54da401200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:02:44.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:44 smithi150 conmon[179483]: 2023-10-09T06:02:44.352+0000 7f54da401200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:02:44.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:44 smithi150 ceph-mon[105942]: pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T06:02:44.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:44 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/312133916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:46.121 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:45 smithi150 conmon[179483]: 2023-10-09T06:02:45.687+0000 7f54da401200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:46.121 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:45 smithi150 conmon[179483]: 2023-10-09T06:02:45.854+0000 7f54da401200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:02:46.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:46 smithi107 ceph-mon[118900]: pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:02:46.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:46 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3985022985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:46.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:46 smithi107 ceph-mon[101102]: pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:02:46.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:46 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3985022985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:46.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:46 smithi150 ceph-mon[105942]: pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:02:46.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:46 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3985022985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:46.408 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:46 smithi150 conmon[179483]: 2023-10-09T06:02:46.178+0000 7f54da401200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:02:47.014 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:46 smithi150 conmon[179483]: 2023-10-09T06:02:46.661+0000 7f54da401200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:02:47.014 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:47 smithi150 conmon[179483]: 2023-10-09T06:02:47.009+0000 7f54da401200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:02:47.265 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:47 smithi150 conmon[179483]: 2023-10-09T06:02:47.264+0000 7f54da401200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:02:48.129 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:47 smithi150 conmon[179483]: 2023-10-09T06:02:47.701+0000 7f54da401200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:02:48.132 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:48 smithi150 ceph-mon[105942]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-10-09T06:02:48.132 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:48 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:48.132 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:48 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:48.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[101102]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-10-09T06:02:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[118900]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-10-09T06:02:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:48 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:49.028 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:48 smithi150 conmon[178500]: ts=2023-10-09T06:02:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:02:49.029 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:48 smithi150 conmon[179483]: 2023-10-09T06:02:48.808+0000 7f54da401200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:02:49.029 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:49 smithi150 conmon[179483]: 2023-10-09T06:02:49.025+0000 7f54da401200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:02:49.383 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:49 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3849389369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:49 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3849389369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:49.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:49 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3849389369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:49.659 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:49 smithi150 conmon[179483]: 2023-10-09T06:02:49.382+0000 7f54da401200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:02:49.660 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:49 smithi150 conmon[179483]: 2023-10-09T06:02:49.584+0000 7f54da401200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:02:50.405 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:49 smithi150 conmon[179483]: 2023-10-09T06:02:49.974+0000 7f54da401200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:02:50.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:50 smithi150 ceph-mon[105942]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T06:02:50.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:50 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:50.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:50 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[118900]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T06:02:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:50.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[101102]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T06:02:50.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:50.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:50 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:51.475 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: 2023-10-09T06:02:51.362+0000 7f54da401200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:02:51.476 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: [09/Oct/2023:06:02:51] ENGINE Bus STARTING 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/98350180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: Standby manager daemon x restarted 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:51.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/98350180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: Standby manager daemon x restarted 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: Standby manager daemon x started 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:51 smithi107 ceph-mon[101102]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/98350180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: Standby manager daemon x restarted 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:02:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:51 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/2038858555' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:02:51.906 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: CherryPy Checker: 2023-10-09T06:02:51.907 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: The Application mounted at '' has an empty config. 2023-10-09T06:02:51.907 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: 2023-10-09T06:02:51.907 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: [09/Oct/2023:06:02:51] ENGINE Serving on http://:::9283 2023-10-09T06:02:51.907 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:02:51 smithi150 conmon[179483]: [09/Oct/2023:06:02:51] ENGINE Bus STARTED 2023-10-09T06:02:52.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:02:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:02:52] "GET /metrics HTTP/1.1" 200 36160 "" "Prometheus/2.43.0" 2023-10-09T06:02:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:52 smithi150 ceph-mon[105942]: pgmap v24: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T06:02:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:52 smithi150 ceph-mon[105942]: mgrmap e30: y(active, since 42s), standbys: x 2023-10-09T06:02:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:52 smithi107 ceph-mon[101102]: pgmap v24: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T06:02:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:52 smithi107 ceph-mon[101102]: mgrmap e30: y(active, since 42s), standbys: x 2023-10-09T06:02:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:52 smithi107 ceph-mon[118900]: pgmap v24: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-09T06:02:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:52 smithi107 ceph-mon[118900]: mgrmap e30: y(active, since 42s), standbys: x 2023-10-09T06:02:54.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[118900]: pgmap v25: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T06:02:54.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2116074423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:54.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[101102]: pgmap v25: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T06:02:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/2116074423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:54.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:54 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:54 smithi150 ceph-mon[105942]: pgmap v25: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-09T06:02:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:54 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2116074423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:54.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:54 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:02:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:56 smithi150 ceph-mon[105942]: pgmap v26: 89 pgs: 89 active+clean; 1016 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T06:02:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:56 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1648828253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:56 smithi107 ceph-mon[101102]: pgmap v26: 89 pgs: 89 active+clean; 1016 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T06:02:56.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:56 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/1648828253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:56 smithi107 ceph-mon[118900]: pgmap v26: 89 pgs: 89 active+clean; 1016 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T06:02:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:56 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1648828253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: pgmap v27: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:58.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:02:58.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: pgmap v27: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:02:58.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:58 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:02:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: pgmap v27: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:02:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:58 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T06:02:58.907 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:02:58 smithi150 conmon[178500]: ts=2023-10-09T06:02:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:02:59.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[101102]: Upgrade: Setting container_image for all mgr 2023-10-09T06:02:59.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[101102]: Upgrade: It appears safe to stop mon.a 2023-10-09T06:02:59.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/3305983096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:59 smithi150 ceph-mon[105942]: Upgrade: Setting container_image for all mgr 2023-10-09T06:02:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:59 smithi150 ceph-mon[105942]: Upgrade: It appears safe to stop mon.a 2023-10-09T06:02:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:02:59 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/3305983096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:02:59.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[118900]: Upgrade: Setting container_image for all mgr 2023-10-09T06:02:59.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[118900]: Upgrade: It appears safe to stop mon.a 2023-10-09T06:02:59.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:02:59 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/3305983096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:00.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:00 smithi107 ceph-mon[118900]: pgmap v28: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T06:03:00.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:00 smithi107 ceph-mon[101102]: pgmap v28: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T06:03:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:00 smithi150 ceph-mon[105942]: pgmap v28: 89 pgs: 89 active+clean; 800 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: pgmap v29: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: Upgrade: Updating mon.a 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: Deploying daemon mon.a on smithi107 2023-10-09T06:03:02.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[101102]: from='client.? 172.21.15.107:0/989054794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:03:02] "GET /metrics HTTP/1.1" 200 36157 "" "Prometheus/2.43.0" 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: pgmap v29: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: Upgrade: Updating mon.a 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:02.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: Deploying daemon mon.a on smithi107 2023-10-09T06:03:02.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:02 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/989054794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: pgmap v29: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: Upgrade: Updating mon.a 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='mgr.y' 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: from='mgr.25031 172.21.15.107:0/1502283939' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: Deploying daemon mon.a on smithi107 2023-10-09T06:03:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:02 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/989054794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:03.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:03 smithi107 systemd[1]: Stopping Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:03:04.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:04 smithi107 conmon[101077]: 2023-10-09T06:03:04.124+0000 7f5482f4c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:03:04.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:04 smithi107 conmon[101077]: 2023-10-09T06:03:04.124+0000 7f5482f4c700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T06:03:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:04 smithi107 bash[212331]: b3f3a3055d815d719b789734cd608f9bc277544a5c87d93a8707e62f41111da5 2023-10-09T06:03:05.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:05 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T06:03:05.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:05 smithi107 systemd[1]: Stopped Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:03:05.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:05 smithi107 systemd[1]: Starting Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:03:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:03:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 8 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: pidfile_write: ignore empty --pid-file 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: load: jerasure load: lrc 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: RocksDB version: 7.9.2 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Git sha 0 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: DB SUMMARY 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: DB Session ID: TG688W83R5YDIAMEYBQJ 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: CURRENT file: CURRENT 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1162 Bytes 2023-10-09T06:03:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2720434 ; 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.error_if_exists: 0 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.create_if_missing: 0 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.paranoid_checks: 1 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.env: 0x556ada5dd720 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.info_log: 0x556adc6a1420 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:03:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.statistics: (nil) 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.use_fsync: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_log_file_size: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_fallocate: 1 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.use_direct_reads: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:03:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.db_log_dir: 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.wal_dir: 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.write_buffer_manager: 0x556adb930c80 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:03:06.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.unordered_write: 0 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.row_cache: None 2023-10-09T06:03:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.wal_filter: None 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.two_write_queues: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.wal_compression: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.atomic_flush: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.log_readahead_size: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:03:06.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_background_jobs: 2 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_background_compactions: -1 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_subcompactions: 1 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:03:06.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_open_files: -1 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_background_flushes: -1 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Compression algorithms supported: 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kZSTD supported: 0 2023-10-09T06:03:06.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kXpressCompression supported: 0 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kZlibCompression supported: 1 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kSnappyCompression supported: 1 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kLZ4Compression supported: 1 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: kBZip2Compression supported: 0 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.merge_operator: 2023-10-09T06:03:06.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_filter: None 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556adc6a1520) 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: cache_index_and_filter_blocks: 1 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: pin_top_level_index_and_filter: 1 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_type: 0 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: data_block_index_type: 0 2023-10-09T06:03:06.911 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_shortening: 1 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: checksum: 4 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: no_block_cache: 0 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache: 0x556adb9b3350 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_name: BinnedLRUCache 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_options: 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: capacity : 536870912 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: num_shard_bits : 4 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: strict_capacity_limit : 0 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: high_pri_pool_ratio: 0.000 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_cache_compressed: (nil) 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: persistent_cache: (nil) 2023-10-09T06:03:06.912 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_size: 4096 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_size_deviation: 10 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_restart_interval: 16 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: index_block_restart_interval: 1 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: metadata_block_size: 4096 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: partition_filters: 0 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: use_delta_encoding: 1 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: filter_policy: bloomfilter 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: whole_key_filtering: 1 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: verify_compression: 0 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: read_amp_bytes_per_bit: 0 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: format_version: 5 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: enable_index_compression: 1 2023-10-09T06:03:06.913 INFO:journalctl@ceph.mon.a.smithi107.stdout: block_align: 0 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout: max_auto_readahead_size: 262144 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout: prepopulate_block_cache: 0 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout: initial_auto_readahead_size: 8192 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression: NoCompression 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.num_levels: 7 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:03:06.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:03:06.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:03:06.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:03:06.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:03:06.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:03:06.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:03:06.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:03:06.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:03:06.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:03:06.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.table_properties_collectors: 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.inplace_update_support: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.bloom_locality: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.max_successive_merges: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:03:06.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.ttl: 2592000 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enable_blob_files: false 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.min_blob_size: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:03:06.921 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12254, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9da4782a-2b14-4df3-9d90-f353f10b30c2 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831386486670, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831386497506, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2500064, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12255, "largest_seqno": 13719, "table_properties": {"data_size": 2493335, "index_size": 4110, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14678, "raw_average_key_size": 23, "raw_value_size": 2479854, "raw_average_value_size": 3961, "num_data_blocks": 188, "num_entries": 626, "num_filter_entries": 626, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831386, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9da4782a-2b14-4df3-9d90-f353f10b30c2", "db_session_id": "TG688W83R5YDIAMEYBQJ", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831386497614, "job": 1, "event": "recovery_finished"} 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-10-09T06:03:06.922 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556adba4e000 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: DB pointer 0x556adba3c000 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** DB Stats ** 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:03:06.923 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: L0 1/0 2.38 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 478.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: L6 1/0 5.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: Sum 2/0 7.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 478.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 478.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:03:06.924 INFO:journalctl@ceph.mon.a.smithi107.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 478.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Cumulative compaction: 0.00 GB write, 134.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Interval compaction: 0.00 GB write, 134.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:03:06.925 INFO:journalctl@ceph.mon.a.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: Block cache BinnedLRUCache@0x556adb9b3350#8 capacity: 512.00 MB usage: 51.70 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: Block cache entry stats(count,size,portion): DataBlock(3,24.23 KB,0.00462234%) FilterBlock(2,9.81 KB,0.00187159%) IndexBlock(2,17.66 KB,0.00336766%) Misc(1,0.00 KB,0%) 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: starting mon.a rank 0 at public addrs [v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] at bind addrs [v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???) e3 preinit fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).mds e9 new map 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).mds e9 print_map 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: e9 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T06:03:06.926 INFO:journalctl@ceph.mon.a.smithi107.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: legacy client fscid: 1 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: Filesystem 'foo' (1) 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: fs_name foo 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: epoch 8 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: created 2023-10-09T05:57:53.253294+0000 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: modified 2023-10-09T05:58:03.974541+0000 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: tableserver 0 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: root 0 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: session_timeout 60 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: session_autoclose 300 2023-10-09T06:03:06.927 INFO:journalctl@ceph.mon.a.smithi107.stdout: max_file_size 1099511627776 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: required_client_features {} 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: last_failure 0 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: last_failure_osd_epoch 0 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: max_mds 1 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: in 0 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: up {0=24560} 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: failed 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: damaged 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: stopped 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: data_pools [4] 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: metadata_pool 3 2023-10-09T06:03:06.928 INFO:journalctl@ceph.mon.a.smithi107.stdout: inline_data disabled 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: balancer 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: bal_rank_mask -1 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: standby_count_wanted 1 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: [mds.foo.smithi107.dqtsqc{0:24560} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: Standby daemons: 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout: [mds.foo.smithi150.znbnco{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T06:03:06.929 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:03:06.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:03:06.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:03:06.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 ceph-mon[212750]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-09T06:03:06.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 bash[212665]: 7f66f5126e5d2be5eec38dce6b0768742764d739ed0be71d0bed1918af4cbad3 2023-10-09T06:03:06.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:06 smithi107 systemd[1]: Started Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:03:08.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/1198702974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: pgmap v31: 89 pgs: 89 active+clean; 551 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:03:08.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/172318987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: pgmap v32: 89 pgs: 89 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:03:08.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: mon.a calling monitor election 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: mgrmap e30: y(active, since 58s), standbys: x 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: mgrmap e31: y(active, since 58s), standbys: x 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: overall HEALTH_OK 2023-10-09T06:03:08.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[118900]: from='mgr.25031 ' entity='' 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: ignoring --setuser ceph since I am not root 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: ignoring --setgroup ceph since I am not root 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: 2023-10-09T06:03:08.238+0000 7f54c740a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: 2023-10-09T06:03:08.348+0000 7f54c740a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1198702974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: pgmap v31: 89 pgs: 89 active+clean; 551 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/172318987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: pgmap v32: 89 pgs: 89 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T06:03:08.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: mgrmap e30: y(active, since 58s), standbys: x 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: mgrmap e31: y(active, since 58s), standbys: x 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:08 smithi107 ceph-mon[212750]: from='mgr.25031 ' entity='' 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1198702974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.353 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: pgmap v31: 89 pgs: 89 active+clean; 551 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/172318987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: pgmap v32: 89 pgs: 89 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: mon.a calling monitor election 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: mgrmap e30: y(active, since 58s), standbys: x 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: mgrmap e31: y(active, since 58s), standbys: x 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: overall HEALTH_OK 2023-10-09T06:03:08.354 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:08 smithi150 ceph-mon[105942]: from='mgr.25031 ' entity='' 2023-10-09T06:03:08.357 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: ignoring --setuser ceph since I am not root 2023-10-09T06:03:08.358 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: ignoring --setgroup ceph since I am not root 2023-10-09T06:03:08.358 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: 2023-10-09T06:03:08.218+0000 7f3a20c72200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T06:03:08.358 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: 2023-10-09T06:03:08.348+0000 7f3a20c72200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T06:03:08.646 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: 2023-10-09T06:03:08.489+0000 7f54c740a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:03:08.647 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: 2023-10-09T06:03:08.634+0000 7f54c740a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:03:08.648 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: 2023-10-09T06:03:08.492+0000 7f3a20c72200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T06:03:08.648 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: 2023-10-09T06:03:08.643+0000 7f3a20c72200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T06:03:08.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:08 smithi107 conmon[200038]: 2023-10-09T06:03:08.893+0000 7f54c740a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:03:08.904 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[178500]: ts=2023-10-09T06:03:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:03:09.409 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:08 smithi150 conmon[179483]: 2023-10-09T06:03:08.913+0000 7f3a20c72200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T06:03:09.588 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T06:03:09.781 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:09 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2938364032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:09.783 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:09 smithi150 conmon[179483]: 2023-10-09T06:03:09.461+0000 7f3a20c72200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:03:09.783 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:09 smithi150 conmon[179483]: 2023-10-09T06:03:09.647+0000 7f3a20c72200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:03:09.783 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:09 smithi150 conmon[179483]: 2023-10-09T06:03:09.781+0000 7f3a20c72200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:03:09.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2938364032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:09.795 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:09 smithi107 conmon[200038]: 2023-10-09T06:03:09.455+0000 7f54c740a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T06:03:09.795 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:09 smithi107 conmon[200038]: 2023-10-09T06:03:09.659+0000 7f54c740a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T06:03:09.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:09 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2938364032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:03:10.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:09 smithi107 conmon[200038]: 2023-10-09T06:03:09.793+0000 7f54c740a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T06:03:10.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:09 smithi107 conmon[200038]: 2023-10-09T06:03:09.934+0000 7f54c740a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:03:10.157 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:09 smithi150 conmon[179483]: 2023-10-09T06:03:09.937+0000 7f3a20c72200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T06:03:10.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:10 smithi107 conmon[200038]: 2023-10-09T06:03:10.560+0000 7f54c740a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:03:10.910 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:10 smithi150 conmon[179483]: 2023-10-09T06:03:10.612+0000 7f3a20c72200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T06:03:11.090 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:03:11.786 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:11.903 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:11 smithi107 conmon[200038]: 2023-10-09T06:03:11.507+0000 7f54c740a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:03:11.904 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:11 smithi107 conmon[200038]: 2023-10-09T06:03:11.649+0000 7f54c740a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:03:11.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:11 smithi150 conmon[179483]: 2023-10-09T06:03:11.600+0000 7f3a20c72200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T06:03:11.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:11 smithi150 conmon[179483]: 2023-10-09T06:03:11.733+0000 7f3a20c72200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T06:03:12.880 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:12 smithi107 conmon[200038]: 2023-10-09T06:03:12.873+0000 7f54c740a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:03:12.949 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 121940 objects 2023-10-09T06:03:12.950 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :166.951 2023-10-09T06:03:13.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:12 smithi107 conmon[200038]: 2023-10-09T06:03:12.985+0000 7f54c740a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:03:13.212 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:12 smithi150 conmon[179483]: 2023-10-09T06:03:12.940+0000 7f3a20c72200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:03:13.212 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:13 smithi150 conmon[179483]: 2023-10-09T06:03:13.036+0000 7f3a20c72200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T06:03:13.460 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:13 smithi107 conmon[200038]: 2023-10-09T06:03:13.160+0000 7f54c740a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:03:13.499 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:13 smithi150 conmon[179483]: 2023-10-09T06:03:13.211+0000 7f3a20c72200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T06:03:13.712 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:13 smithi107 conmon[200038]: 2023-10-09T06:03:13.459+0000 7f54c740a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:03:13.712 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:13 smithi107 conmon[200038]: 2023-10-09T06:03:13.706+0000 7f54c740a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:03:13.882 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:13 smithi150 conmon[179483]: 2023-10-09T06:03:13.498+0000 7f3a20c72200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T06:03:13.882 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:13 smithi150 conmon[179483]: 2023-10-09T06:03:13.730+0000 7f3a20c72200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T06:03:14.141 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:13 smithi107 conmon[200038]: 2023-10-09T06:03:13.866+0000 7f54c740a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:03:14.151 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:13 smithi150 conmon[179483]: 2023-10-09T06:03:13.881+0000 7f3a20c72200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T06:03:14.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:14 smithi107 conmon[200038]: 2023-10-09T06:03:14.140+0000 7f54c740a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:03:14.404 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:14 smithi150 conmon[179483]: 2023-10-09T06:03:14.151+0000 7f3a20c72200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T06:03:15.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:14 smithi107 conmon[200038]: 2023-10-09T06:03:14.852+0000 7f54c740a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:03:15.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:14 smithi107 conmon[200038]: 2023-10-09T06:03:14.985+0000 7f54c740a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:03:15.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:14 smithi150 conmon[179483]: 2023-10-09T06:03:14.867+0000 7f3a20c72200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T06:03:15.154 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:15 smithi150 conmon[179483]: 2023-10-09T06:03:15.001+0000 7f3a20c72200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T06:03:15.521 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:15 smithi107 conmon[200038]: 2023-10-09T06:03:15.183+0000 7f54c740a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:03:15.522 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:15 smithi107 conmon[200038]: 2023-10-09T06:03:15.298+0000 7f54c740a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:03:15.542 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:15 smithi150 conmon[179483]: 2023-10-09T06:03:15.200+0000 7f3a20c72200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T06:03:15.542 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:15 smithi150 conmon[179483]: 2023-10-09T06:03:15.315+0000 7f3a20c72200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T06:03:15.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:15 smithi107 conmon[200038]: 2023-10-09T06:03:15.520+0000 7f54c740a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:03:15.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:15 smithi150 conmon[179483]: 2023-10-09T06:03:15.542+0000 7f3a20c72200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T06:03:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.399+0000 7f54c740a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:03:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: [09/Oct/2023:06:03:16] ENGINE Bus STARTING 2023-10-09T06:03:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5423739700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5423739700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5423739700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5423739700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5423739700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5421735700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5421735700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5421735700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5421735700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.572+0000 7f5421735700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: CherryPy Checker: 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: The Application mounted at '' has an empty config. 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: [09/Oct/2023:06:03:16] ENGINE Serving on http://:::9283 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[200038]: [09/Oct/2023:06:03:16] ENGINE Bus STARTED 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 conmon[118876]: 2023-10-09T06:03:16.416+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:16.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: Active manager daemon y restarted 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: Activating manager daemon y 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: mgrmap e32: y(active, starting, since 0.00542261s), standbys: x 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:03:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: Standby manager daemon x restarted 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: Standby manager daemon x started 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:03:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:03:16.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:03:16.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:03:16.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:03:16.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:03:16.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:03:16.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:03:16.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: Active manager daemon y restarted 2023-10-09T06:03:16.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: Activating manager daemon y 2023-10-09T06:03:16.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:03:16.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: mgrmap e32: y(active, starting, since 0.00542261s), standbys: x 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: Standby manager daemon x restarted 2023-10-09T06:03:16.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: Standby manager daemon x started 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:03:16.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:03:16.661 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[212750]: Manager daemon y is now available 2023-10-09T06:03:16.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:03:16.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:16 smithi107 ceph-mon[118900]: Manager daemon y is now available 2023-10-09T06:03:16.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: 2023-10-09T06:03:16.415+0000 7f3a20c72200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T06:03:16.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: [09/Oct/2023:06:03:16] ENGINE Bus STARTING 2023-10-09T06:03:16.904 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: CherryPy Checker: 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: The Application mounted at '' has an empty config. 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: [09/Oct/2023:06:03:16] ENGINE Serving on http://:::9283 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[179483]: [09/Oct/2023:06:03:16] ENGINE Bus STARTED 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 conmon[105918]: 2023-10-09T06:03:16.416+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: Active manager daemon y restarted 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: Activating manager daemon y 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: mgrmap e32: y(active, starting, since 0.00542261s), standbys: x 2023-10-09T06:03:16.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: Standby manager daemon x restarted 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: Standby manager daemon x started 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.znbnco"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi107.dqtsqc"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T06:03:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.? 172.21.15.150:0/52136269' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T06:03:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T06:03:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:16 smithi150 ceph-mon[105942]: Manager daemon y is now available 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (114s) - 6m 18.0M - 0.25.0 c8568f914cd2 0715bcb46fa1 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (92s) - 5m 84.2M - 9.4.7 2c41d148cca3 96db62cb38d0 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (5m) - 5m 20.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (5m) - 5m 14.9M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443,9283,8765 running (37s) - 10m 82.7M - 18.2.0-598-gb4fa663f 3060897fdf97 7326a92d4fbc 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:8443,9283,8765 running (83s) - 11m 484M - 18.2.0-598-gb4fa663f 3060897fdf97 19d12e0e339c 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (11m) - 12m 105M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 b3f3a3055d81 2023-10-09T06:03:17.438 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (10m) - 10m 80.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 0a878f231494 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (10m) - 10m 67.2M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c53cec84dedf 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (111s) - 6m 9453k - 1.5.0 0da6a335fe13 32eddd377f02 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (99s) - 6m 8921k - 1.5.0 0da6a335fe13 6b4c6b51266f 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (9m) - 9m 1846M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 8de9f1d99699 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (9m) - 9m 1106M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 674ef542e3e9 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (9m) - 9m 2388M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4f4ef8822b1f 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (8m) - 8m 786M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e1f7d3e8c22d 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (8m) - 8m 2006M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (7m) - 7m 2143M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (7m) - 7m 1032M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T06:03:17.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (7m) - 7m 1511M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T06:03:17.440 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (45s) - 6m 42.1M - 2.43.0 a07b618ecd1d 3a1d67d52b64 2023-10-09T06:03:17.763 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:17 smithi107 conmon[200038]: 2023-10-09T06:03:17.430+0000 7f5488b69700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T06:03:17.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: [09/Oct/2023:06:03:16] ENGINE Bus STARTING 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: [09/Oct/2023:06:03:17] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[212750]: mgrmap e33: y(active, since 1.00834s), standbys: x 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:17.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: [09/Oct/2023:06:03:16] ENGINE Bus STARTING 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: [09/Oct/2023:06:03:17] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:03:17.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:17 smithi107 ceph-mon[118900]: mgrmap e33: y(active, since 1.00834s), standbys: x 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: failed to encode map e77 with expected crc 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: [09/Oct/2023:06:03:16] ENGINE Bus STARTING 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: [09/Oct/2023:06:03:17] ENGINE Serving on https://172.21.15.107:7150 2023-10-09T06:03:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:17 smithi150 ceph-mon[105942]: mgrmap e33: y(active, since 1.00834s), standbys: x 2023-10-09T06:03:18.277 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T06:03:18.277 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T06:03:18.303 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:03:18.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[212750]: [09/Oct/2023:06:03:17] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[212750]: [09/Oct/2023:06:03:17] ENGINE Bus STARTED 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[212750]: from='client.25229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[212750]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[118900]: [09/Oct/2023:06:03:17] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[118900]: [09/Oct/2023:06:03:17] ENGINE Bus STARTED 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[118900]: from='client.25229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:18.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:18 smithi107 ceph-mon[118900]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:18.865 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:18 smithi150 ceph-mon[105942]: [09/Oct/2023:06:03:17] ENGINE Serving on http://172.21.15.107:8765 2023-10-09T06:03:18.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:18 smithi150 ceph-mon[105942]: [09/Oct/2023:06:03:17] ENGINE Bus STARTED 2023-10-09T06:03:18.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:18 smithi150 ceph-mon[105942]: from='client.25229 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:18.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:18 smithi150 ceph-mon[105942]: pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:18.989 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:19 smithi107 ceph-mon[212750]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:19 smithi107 ceph-mon[212750]: mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T06:03:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:19 smithi107 ceph-mon[118900]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:19 smithi107 ceph-mon[118900]: mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T06:03:19.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:19 smithi150 ceph-mon[105942]: pgmap v4: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:19.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:19 smithi150 ceph-mon[105942]: mgrmap e34: y(active, since 3s), standbys: x 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:20 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:20 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:20 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:20 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:20 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:21.970 INFO:teuthology.orchestra.run.smithi107.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:03:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:03:22] "GET /metrics HTTP/1.1" 200 33440 "" "Prometheus/2.43.0" 2023-10-09T06:03:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:21 smithi107 ceph-mon[212750]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:03:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:21 smithi107 ceph-mon[118900]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:21 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:03:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:21 smithi150 ceph-mon[105942]: pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:21 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:03:22.526 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_0' removed 2023-10-09T06:03:22.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 conmon[118876]: 2023-10-09T06:03:22.519+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 conmon[105918]: 2023-10-09T06:03:22.518+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e78 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[212750]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:03:23.064 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:03:23.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:22 smithi107 ceph-mon[118900]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:03:23.081 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:03:23.340 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:03:23.340 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:03:23.341 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T06:03:23.341 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T06:03:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:03:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:03:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:23.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/14491326' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:03:23.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:22 smithi150 ceph-mon[105942]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T06:03:23.460 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:23 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:23 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:24.575 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:24 smithi150 ceph-mon[105942]: failed to encode map e78 with expected crc 2023-10-09T06:03:25.362 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:24 smithi107 ceph-mon[212750]: failed to encode map e78 with expected crc 2023-10-09T06:03:25.362 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:24 smithi107 ceph-mon[118900]: failed to encode map e78 with expected crc 2023-10-09T06:03:26.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[212750]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:26.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:26.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:26.257 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[118900]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:26.257 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.257 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.257 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:26.257 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:25 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:25 smithi150 ceph-mon[105942]: pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:25 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:25 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:25 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:25 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:03:27.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:26 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.conf 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.conf 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi107:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:03:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:26 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T06:03:27.534 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_1' created 2023-10-09T06:03:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 conmon[118876]: 2023-10-09T06:03:27.520+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 conmon[105918]: 2023-10-09T06:03:27.518+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e79 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:28.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[212750]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:03:28.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:03:28.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:27 smithi107 ceph-mon[118900]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.380 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: Updating smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: Updating smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:27 smithi150 ceph-mon[105942]: failed to encode map e79 with expected crc 2023-10-09T06:03:28.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:28 smithi107 conmon[118876]: 2023-10-09T06:03:28.521+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:28.731 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:03:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:28 smithi150 conmon[105918]: 2023-10-09T06:03:28.520+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e80 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:28.904 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:03:28 smithi150 conmon[178500]: ts=2023-10-09T06:03:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:03:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:03:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[212750]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 ceph-mon[118900]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:29 smithi107 conmon[118876]: 2023-10-09T06:03:29.523+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 conmon[105918]: 2023-10-09T06:03:29.522+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: pgmap v11: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:29.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:29 smithi150 ceph-mon[105942]: failed to encode map e80 with expected crc 2023-10-09T06:03:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[212750]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[118900]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:30 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:30 smithi150 ceph-mon[105942]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T06:03:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:30 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:30 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:30 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:30 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.548 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T06:03:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[212750]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[212750]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:31.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[118900]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[118900]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:31.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:31 smithi107 conmon[118876]: 2023-10-09T06:03:31.540+0000 7fdb5e5fd700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 ceph-mon[105942]: failed to encode map e81 with expected crc 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 ceph-mon[105942]: pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:31 smithi150 conmon[105918]: 2023-10-09T06:03:31.539+0000 7f8dc4d1f700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T06:03:32.342 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:03:32] "GET /metrics HTTP/1.1" 200 36106 "" "Prometheus/2.43.0" 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[212750]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[212750]: failed to encode map e82 with expected crc 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[118900]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:03:32.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[118900]: failed to encode map e82 with expected crc 2023-10-09T06:03:32.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:32 smithi107 ceph-mon[118900]: failed to encode map e82 with expected crc 2023-10-09T06:03:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:32 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/182248616' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:03:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:32 smithi150 ceph-mon[105942]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:03:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:32 smithi150 ceph-mon[105942]: failed to encode map e82 with expected crc 2023-10-09T06:03:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:32 smithi150 ceph-mon[105942]: failed to encode map e82 with expected crc 2023-10-09T06:03:33.364 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T06:03:33.367 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:03:33.367 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:03:33.422 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:03:33.423 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:03:33.423 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_217624 2023-10-09T06:03:33.423 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:33.423 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:03:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:33 smithi107 ceph-mon[212750]: pgmap v16: 89 pgs: 7 unknown, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:33.940 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:33 smithi107 ceph-mon[118900]: pgmap v16: 89 pgs: 7 unknown, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:33.941 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:33 smithi150 ceph-mon[105942]: pgmap v16: 89 pgs: 7 unknown, 82 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:34.424 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 4143 4127 257.78 257.938 0.00369821 0.00386462 2023-10-09T06:03:35.424 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 7893 7877 246.019 234.375 0.00656565 0.00405792 2023-10-09T06:03:35.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:35 smithi107 ceph-mon[118900]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:35.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:35 smithi107 ceph-mon[212750]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:35.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:35 smithi150 ceph-mon[105942]: pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:36.424 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 11270 11254 234.364 211.062 0.00941918 0.00425834 2023-10-09T06:03:37.424 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 14222 14206 221.897 184.5 0.00456829 0.00450074 2023-10-09T06:03:37.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:37 smithi107 ceph-mon[118900]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:37 smithi107 ceph-mon[212750]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:37 smithi150 ceph-mon[105942]: pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:03:38.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 15 17229 17214 215.104 188 0.00413836 0.00464378 2023-10-09T06:03:39.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:03:38 smithi150 conmon[178500]: ts=2023-10-09T06:03:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:03:39.427 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 20063 20047 208.763 177.062 0.00420349 0.00478441 2023-10-09T06:03:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:39 smithi107 ceph-mon[212750]: pgmap v19: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:03:39.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:39 smithi107 ceph-mon[118900]: pgmap v19: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:03:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:39 smithi150 ceph-mon[105942]: pgmap v19: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:03:40.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 22739 22723 202.831 167.25 0.00604476 0.00492354 2023-10-09T06:03:41.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 25279 25263 197.32 158.75 0.00554934 0.00506267 2023-10-09T06:03:41.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:41 smithi107 ceph-mon[212750]: pgmap v20: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:41.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:41 smithi107 ceph-mon[118900]: pgmap v20: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:41 smithi150 ceph-mon[105942]: pgmap v20: 89 pgs: 89 active+clean; 890 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:03:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:03:42] "GET /metrics HTTP/1.1" 200 36106 "" "Prometheus/2.43.0" 2023-10-09T06:03:42.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 27703 27687 192.228 151.5 0.00628725 0.00519737 2023-10-09T06:03:43.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 29999 29983 187.354 143.5 0.00802766 0.00533249 2023-10-09T06:03:43.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:43 smithi107 ceph-mon[212750]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:03:43.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:43 smithi107 ceph-mon[118900]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:03:43.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:43 smithi150 ceph-mon[105942]: pgmap v21: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:03:44.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 32209 32193 182.878 138.125 0.00731342 0.00546307 2023-10-09T06:03:45.425 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 34215 34199 178.086 125.375 0.00612286 0.0056106 2023-10-09T06:03:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:45 smithi107 ceph-mon[118900]: pgmap v22: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:45.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:45 smithi107 ceph-mon[212750]: pgmap v22: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:45.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:45 smithi150 ceph-mon[105942]: pgmap v22: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:46.427 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 36275 36259 174.273 128.75 0.00795457 0.00573242 2023-10-09T06:03:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:46 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:46.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:46 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:03:47.427 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 38240 38224 170.588 122.812 0.00778939 0.00585684 2023-10-09T06:03:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:47 smithi107 ceph-mon[212750]: pgmap v23: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:47.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:47 smithi107 ceph-mon[118900]: pgmap v23: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:47 smithi150 ceph-mon[105942]: pgmap v23: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:03:48.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 40101 40085 166.97 116.312 0.00529233 0.00598411 2023-10-09T06:03:49.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:03:48 smithi150 conmon[178500]: ts=2023-10-09T06:03:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:03:49.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 41873 41857 163.456 110.75 0.00829864 0.00611394 2023-10-09T06:03:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:49 smithi107 ceph-mon[212750]: pgmap v24: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.19k op/s 2023-10-09T06:03:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:49 smithi107 ceph-mon[118900]: pgmap v24: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.19k op/s 2023-10-09T06:03:49.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:49 smithi150 ceph-mon[105942]: pgmap v24: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.19k op/s 2023-10-09T06:03:50.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 43742 43726 160.713 116.812 0.00315071 0.00621771 2023-10-09T06:03:51.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 45574 45558 158.145 114.5 0.0100885 0.00631907 2023-10-09T06:03:51.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:51 smithi107 ceph-mon[212750]: pgmap v25: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2023-10-09T06:03:51.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:51 smithi107 ceph-mon[118900]: pgmap v25: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2023-10-09T06:03:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:51 smithi150 ceph-mon[105942]: pgmap v25: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2k op/s 2023-10-09T06:03:52.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:03:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:03:52] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:03:52.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 47160 47144 155.039 99.125 0.00975307 0.00644435 2023-10-09T06:03:53.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:03:53.428890+0000 min lat: 0.00129016 max lat: 0.0794842 avg lat: 0.00654696 2023-10-09T06:03:53.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:03:53.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 48863 48847 152.609 106.438 0.0197561 0.00654696 2023-10-09T06:03:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:53 smithi107 ceph-mon[118900]: pgmap v26: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:03:53.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:53 smithi107 ceph-mon[212750]: pgmap v26: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:03:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:53 smithi150 ceph-mon[105942]: pgmap v26: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:03:54.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 50545 50529 150.347 105.125 0.00621499 0.00664687 2023-10-09T06:03:55.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 52185 52169 148.172 102.5 0.0052848 0.0067444 2023-10-09T06:03:56.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:55 smithi107 ceph-mon[118900]: pgmap v27: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-10-09T06:03:56.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:55 smithi107 ceph-mon[212750]: pgmap v27: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-10-09T06:03:56.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:55 smithi150 ceph-mon[105942]: pgmap v27: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2023-10-09T06:03:56.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 53805 53789 146.132 101.25 0.0128556 0.00683812 2023-10-09T06:03:57.428 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 55398 55382 144.192 99.5625 0.00956571 0.00693026 2023-10-09T06:03:58.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:57 smithi107 ceph-mon[118900]: pgmap v28: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:03:58.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:57 smithi107 ceph-mon[212750]: pgmap v28: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:03:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:57 smithi150 ceph-mon[105942]: pgmap v28: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:03:58.429 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 56950 56934 142.299 97 0.0103593 0.00702211 2023-10-09T06:03:59.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:03:58 smithi150 conmon[178500]: ts=2023-10-09T06:03:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:03:59.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 58490 58474 140.527 96.25 0.0099553 0.00711166 2023-10-09T06:04:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:03:59 smithi107 ceph-mon[118900]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:04:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:03:59 smithi107 ceph-mon[212750]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:04:00.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:03:59 smithi150 ceph-mon[105942]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2023-10-09T06:04:00.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 60007 59991 138.834 94.8125 0.00712187 0.0071985 2023-10-09T06:04:01.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 61484 61468 137.173 92.3125 0.00342454 0.00728564 2023-10-09T06:04:02.013 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:01 smithi107 ceph-mon[212750]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:04:02.013 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:02.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:01 smithi107 ceph-mon[118900]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:04:02.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:01 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:01 smithi150 ceph-mon[105942]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-09T06:04:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:01 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:02.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:02] "GET /metrics HTTP/1.1" 200 36160 "" "Prometheus/2.43.0" 2023-10-09T06:04:02.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 62924 62908 135.546 90 0.00364553 0.00737281 2023-10-09T06:04:03.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 64343 64327 133.984 88.6875 0.0147611 0.0074595 2023-10-09T06:04:04.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:03 smithi107 ceph-mon[212750]: pgmap v31: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:04:04.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:03 smithi150 ceph-mon[105942]: pgmap v31: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:04:04.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:03 smithi107 ceph-mon[118900]: pgmap v31: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-09T06:04:04.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 65748 65732 132.494 87.8125 0.0146068 0.00754291 2023-10-09T06:04:05.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 67134 67118 131.061 86.625 0.00799436 0.00762532 2023-10-09T06:04:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:05 smithi107 ceph-mon[212750]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:04:06.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:05 smithi107 ceph-mon[118900]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:04:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:05 smithi150 ceph-mon[105942]: pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:04:06.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 68496 68480 129.669 85.125 0.00685068 0.00770741 2023-10-09T06:04:07.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 69851 69835 128.346 84.6875 0.00787676 0.00778679 2023-10-09T06:04:08.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:07 smithi107 ceph-mon[212750]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:04:08.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:07 smithi107 ceph-mon[118900]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:04:08.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:07 smithi150 ceph-mon[105942]: pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T06:04:08.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 71192 71176 127.073 83.8125 0.00916083 0.00786418 2023-10-09T06:04:09.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:08 smithi150 conmon[178500]: ts=2023-10-09T06:04:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:09.430 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 72514 72498 125.839 82.625 0.0176995 0.00794179 2023-10-09T06:04:10.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:09 smithi107 ceph-mon[212750]: pgmap v34: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:04:10.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:09 smithi107 ceph-mon[118900]: pgmap v34: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:04:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:09 smithi150 ceph-mon[105942]: pgmap v34: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T06:04:10.432 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 73805 73789 124.612 80.6875 0.00791697 0.00801983 2023-10-09T06:04:11.439 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 75086 75070 123.427 80.0625 0.00564196 0.00809737 2023-10-09T06:04:12.015 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:12] "GET /metrics HTTP/1.1" 200 36160 "" "Prometheus/2.43.0" 2023-10-09T06:04:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:11 smithi107 ceph-mon[212750]: pgmap v35: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:04:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:11 smithi107 ceph-mon[118900]: pgmap v35: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:04:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:11 smithi150 ceph-mon[105942]: pgmap v35: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:04:12.439 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 76353 76337 122.284 79.1875 0.00957721 0.00817286 2023-10-09T06:04:13.439 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:04:13.440201+0000 min lat: 0.00129016 max lat: 0.0794842 avg lat: 0.0082464 2023-10-09T06:04:13.440 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:13.440 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 77609 77593 121.19 78.5 0.0103628 0.0082464 2023-10-09T06:04:14.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:13 smithi107 ceph-mon[118900]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T06:04:14.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:13 smithi107 ceph-mon[212750]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T06:04:14.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:13 smithi150 ceph-mon[105942]: pgmap v36: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T06:04:14.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 78845 78829 120.094 77.25 0.00810512 0.00832205 2023-10-09T06:04:15.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 80054 80038 119.034 75.5625 0.0149188 0.00839607 2023-10-09T06:04:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:15 smithi107 ceph-mon[212750]: pgmap v37: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:04:16.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:15 smithi107 ceph-mon[118900]: pgmap v37: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:04:16.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:15 smithi150 ceph-mon[105942]: pgmap v37: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-09T06:04:16.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 81233 81217 117.98 73.6875 0.015015 0.00847131 2023-10-09T06:04:17.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:17.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:16 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:17.449 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 82415 82399 116.978 73.875 0.0124072 0.00854314 2023-10-09T06:04:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:17 smithi107 ceph-mon[118900]: pgmap v38: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:18.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:17 smithi107 ceph-mon[212750]: pgmap v38: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:18.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:17 smithi150 ceph-mon[105942]: pgmap v38: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:04:18.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 83598 83582 116.022 73.9375 0.0117212 0.00861406 2023-10-09T06:04:19.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:18 smithi150 conmon[178500]: ts=2023-10-09T06:04:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:19.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 84756 84740 115.073 72.375 0.0112512 0.00868562 2023-10-09T06:04:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:19 smithi107 ceph-mon[118900]: pgmap v39: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:04:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:19 smithi107 ceph-mon[212750]: pgmap v39: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:04:20.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:19 smithi150 ceph-mon[105942]: pgmap v39: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:04:20.448 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 85918 85902 114.171 72.625 0.0163204 0.0087541 2023-10-09T06:04:21.449 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 87047 87031 113.261 70.5625 0.0150289 0.00882434 2023-10-09T06:04:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:22] "GET /metrics HTTP/1.1" 200 36159 "" "Prometheus/2.43.0" 2023-10-09T06:04:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:21 smithi107 ceph-mon[212750]: pgmap v40: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:04:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:21 smithi107 ceph-mon[118900]: pgmap v40: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:04:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:21 smithi150 ceph-mon[105942]: pgmap v40: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:04:22.450 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 88127 88111 112.326 67.5 0.018837 0.00889843 2023-10-09T06:04:23.451 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 89229 89213 111.455 68.875 0.0141978 0.0089681 2023-10-09T06:04:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:23 smithi107 ceph-mon[118900]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:04:24.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:23 smithi107 ceph-mon[212750]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:04:24.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:23 smithi150 ceph-mon[105942]: pgmap v41: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T06:04:24.451 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 90350 90334 110.644 70.0625 0.0242658 0.00903402 2023-10-09T06:04:25.451 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 91444 91428 109.831 68.375 0.00576096 0.00909975 2023-10-09T06:04:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:25 smithi107 ceph-mon[212750]: pgmap v42: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:04:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:25 smithi107 ceph-mon[118900]: pgmap v42: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:04:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:25 smithi150 ceph-mon[105942]: pgmap v42: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:04:26.451 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 92231 92215 108.687 49.1875 0.0142021 0.00919605 2023-10-09T06:04:26.774 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:04:27.451 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 93152 93136 107.741 57.5625 0.0186532 0.00927637 2023-10-09T06:04:27.600 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:27 smithi107 ceph-mon[212750]: pgmap v43: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T06:04:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:27 smithi107 ceph-mon[118900]: pgmap v43: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T06:04:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:27 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:27 smithi150 ceph-mon[105942]: pgmap v43: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-09T06:04:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:27 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:28.454 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 94082 94066 106.833 58.125 0.0150285 0.00935468 2023-10-09T06:04:29.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:28 smithi150 conmon[178500]: ts=2023-10-09T06:04:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:29.454 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 94869 94853 105.804 49.1875 0.00910095 0.00944601 2023-10-09T06:04:29.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:29 smithi107 ceph-mon[118900]: pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:04:29.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:29 smithi107 ceph-mon[212750]: pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:04:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:29 smithi150 ceph-mon[105942]: pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:04:30.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 95776 95760 104.942 56.6875 0.0202153 0.00952433 2023-10-09T06:04:30.737 INFO:teuthology.orchestra.run.smithi107.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:04:31.454 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 96664 96648 104.09 55.5 0.0188924 0.00960305 2023-10-09T06:04:31.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T06:04:32.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: pgmap v45: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:04:32.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:32.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:04:32.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: pgmap v45: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: pgmap v45: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:31 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:04:32.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:32] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:04:32.457 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 97563 97547 103.279 56.1875 0.00689002 0.00967865 2023-10-09T06:04:32.932 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:33.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:04:33.455434+0000 min lat: 0.00129016 max lat: 0.329823 avg lat: 0.00974938 2023-10-09T06:04:33.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:33.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 98482 98466 102.515 57.4375 0.0119751 0.00974938 2023-10-09T06:04:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:33 smithi107 ceph-mon[118900]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:33 smithi107 ceph-mon[212750]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:33 smithi150 ceph-mon[105942]: pgmap v46: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:34.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 99382 99366 101.757 56.25 0.00292341 0.00980866 2023-10-09T06:04:35.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 100126 100110 100.866 46.5 0.00610439 0.00990889 2023-10-09T06:04:35.794 INFO:teuthology.orchestra.run.smithi107.stdout:wait for majority of mons to upgrade 2023-10-09T06:04:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[212750]: pgmap v47: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:35.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:35.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[118900]: pgmap v47: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:35.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:35.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:35 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:35 smithi150 ceph-mon[105942]: pgmap v47: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T06:04:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:35 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:35 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:36.455 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 101048 101032 100.179 57.625 0.0221153 0.00997797 2023-10-09T06:04:36.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T06:04:37.466 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 101893 101877 99.425 52.8125 0.00723945 0.0100521 2023-10-09T06:04:37.559 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: pgmap v48: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:04:38.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[212750]: Upgrade: It appears safe to stop mon.c 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: pgmap v48: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T06:04:38.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:37 smithi107 ceph-mon[118900]: Upgrade: It appears safe to stop mon.c 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: pgmap v48: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:04:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:04:38.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T06:04:38.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:37 smithi150 ceph-mon[105942]: Upgrade: It appears safe to stop mon.c 2023-10-09T06:04:38.464 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 102761 102745 98.7303 54.25 0.0153513 0.010124 2023-10-09T06:04:39.156 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:38 smithi150 conmon[178500]: ts=2023-10-09T06:04:38.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:39.464 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 103568 103552 97.9989 50.4375 0.0161835 0.0101991 2023-10-09T06:04:39.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:39 smithi107 ceph-mon[212750]: pgmap v49: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:04:39.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:39 smithi107 ceph-mon[118900]: pgmap v49: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:04:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:39 smithi150 ceph-mon[105942]: pgmap v49: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:04:40.472 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 104318 104302 97.2264 46.875 0.0157532 0.0102791 2023-10-09T06:04:41.472 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 105146 105130 96.558 51.75 0.00775628 0.0103522 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: Upgrade: Updating mon.c 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:41.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: Deploying daemon mon.c on smithi107 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[212750]: pgmap v50: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: Upgrade: Updating mon.c 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: Deploying daemon mon.c on smithi107 2023-10-09T06:04:41.549 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:41 smithi107 ceph-mon[118900]: pgmap v50: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: Upgrade: Updating mon.c 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: from='mgr.34098 ' entity='mgr.y' 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: Deploying daemon mon.c on smithi107 2023-10-09T06:04:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:41 smithi150 ceph-mon[105942]: pgmap v50: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-10-09T06:04:42.351 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:42] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:04:42.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2469708971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:42.472 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 105874 105858 95.8185 45.5 0.0150532 0.0104315 2023-10-09T06:04:42.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:42 smithi107 ceph-mon[118900]: from='client.? 172.21.15.107:0/2469708971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:42 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/2469708971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:43.075 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:43 smithi107 systemd[1]: Stopping Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:04:43.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 106780 106764 95.2578 56.625 0.0147758 0.0104932 2023-10-09T06:04:43.636 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE Bus STOPPING 2023-10-09T06:04:43.637 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:43 smithi107 ceph-mon[212750]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T06:04:43.637 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:43 smithi107 ceph-mon[118900]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T06:04:43.637 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[118876]: 2023-10-09T06:04:43.556+0000 7fdb63e08700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:04:43.638 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[118876]: 2023-10-09T06:04:43.556+0000 7fdb63e08700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T06:04:43.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:43 smithi150 ceph-mon[105942]: pgmap v51: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-09T06:04:43.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:04:43.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE Bus STOPPED 2023-10-09T06:04:43.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE Bus STARTING 2023-10-09T06:04:43.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE Serving on http://:::9283 2023-10-09T06:04:43.901 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:43 smithi107 conmon[200038]: [09/Oct/2023:06:04:43] ENGINE Bus STARTED 2023-10-09T06:04:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:43 smithi107 bash[220479]: c53cec84dedf147ec9773ea41e35a9f309e76085977281b3b77e429aa82c65e9 2023-10-09T06:04:44.472 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 107591 107575 94.6304 50.6875 0.00604074 0.0105595 2023-10-09T06:04:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:45 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T06:04:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:45 smithi107 systemd[1]: Stopped Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:04:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:45 smithi107 systemd[1]: Starting Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:04:45.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 108380 108364 94.0011 49.3125 0.0155547 0.0106331 2023-10-09T06:04:46.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 109228 109212 93.4397 53 0.0209729 0.0106966 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 bash[220855]: 6c90cc5e902c4248629aeee86d9a59b278927dcf48dd15acf60daf19b58f0574 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 8 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: pidfile_write: ignore empty --pid-file 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: load: jerasure load: lrc 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: RocksDB version: 7.9.2 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Git sha 0 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:04:46.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: DB SUMMARY 2023-10-09T06:04:46.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: DB Session ID: 2HVYGG30KGU5FCR2KDLN 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: CURRENT file: CURRENT 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1087 Bytes 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7567931 ; 2023-10-09T06:04:46.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.error_if_exists: 0 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.create_if_missing: 0 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.paranoid_checks: 1 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.env: 0x55b007d99720 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.info_log: 0x55b00b31f420 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.statistics: (nil) 2023-10-09T06:04:46.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.use_fsync: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_log_file_size: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_fallocate: 1 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.use_direct_reads: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:04:46.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.db_log_dir: 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.wal_dir: 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.write_buffer_manager: 0x55b00a5aec80 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:04:46.665 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.unordered_write: 0 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:04:46.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.row_cache: None 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.wal_filter: None 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.two_write_queues: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.wal_compression: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.atomic_flush: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.log_readahead_size: 0 2023-10-09T06:04:46.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_background_jobs: 2 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_background_compactions: -1 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_subcompactions: 1 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:04:46.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_open_files: -1 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_background_flushes: -1 2023-10-09T06:04:46.669 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Compression algorithms supported: 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kZSTD supported: 0 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kXpressCompression supported: 0 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kZlibCompression supported: 1 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kSnappyCompression supported: 1 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kLZ4Compression supported: 1 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: kBZip2Compression supported: 0 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T06:04:46.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.merge_operator: 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_filter: None 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b00b31f520) 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout: cache_index_and_filter_blocks: 1 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:04:46.671 INFO:journalctl@ceph.mon.c.smithi107.stdout: pin_top_level_index_and_filter: 1 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_type: 0 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: data_block_index_type: 0 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_shortening: 1 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: checksum: 4 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: no_block_cache: 0 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache: 0x55b00a631350 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_name: BinnedLRUCache 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_options: 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: capacity : 536870912 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: num_shard_bits : 4 2023-10-09T06:04:46.672 INFO:journalctl@ceph.mon.c.smithi107.stdout: strict_capacity_limit : 0 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: high_pri_pool_ratio: 0.000 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_cache_compressed: (nil) 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: persistent_cache: (nil) 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_size: 4096 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_size_deviation: 10 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_restart_interval: 16 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: index_block_restart_interval: 1 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: metadata_block_size: 4096 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: partition_filters: 0 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: use_delta_encoding: 1 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: filter_policy: bloomfilter 2023-10-09T06:04:46.673 INFO:journalctl@ceph.mon.c.smithi107.stdout: whole_key_filtering: 1 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: verify_compression: 0 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: read_amp_bytes_per_bit: 0 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: format_version: 5 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: enable_index_compression: 1 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: block_align: 0 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: max_auto_readahead_size: 262144 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: prepopulate_block_cache: 0 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: initial_auto_readahead_size: 8192 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T06:04:46.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression: NoCompression 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.num_levels: 7 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:04:46.675 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:04:46.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:04:46.677 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:04:46.678 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:04:46.679 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.table_properties_collectors: 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.inplace_update_support: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.bloom_locality: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.max_successive_merges: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:04:46.680 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.ttl: 2592000 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enable_blob_files: false 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.min_blob_size: 0 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:04:46.681 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12815, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3c1baea1-e910-4ba2-ac0b-ff6de1d33536 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831486311813, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T06:04:46.682 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 systemd[1]: Started Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831486347554, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4591469, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12820, "largest_seqno": 16190, "table_properties": {"data_size": 4578979, "index_size": 7949, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3525, "raw_key_size": 30363, "raw_average_key_size": 21, "raw_value_size": 4551230, "raw_average_value_size": 3262, "num_data_blocks": 361, "num_entries": 1395, "num_filter_entries": 1395, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831486, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3c1baea1-e910-4ba2-ac0b-ff6de1d33536", "db_session_id": "2HVYGG30KGU5FCR2KDLN", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831486347681, "job": 1, "event": "recovery_finished"} 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b00a6d6000 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: DB pointer 0x55b00a6ba000 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** DB Stats ** 2023-10-09T06:04:46.683 INFO:journalctl@ceph.mon.c.smithi107.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: L0 1/0 4.38 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.8 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-10-09T06:04:46.684 INFO:journalctl@ceph.mon.c.smithi107.stdout: L6 1/0 5.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Sum 2/0 9.62 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.8 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.8 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** Compaction Stats [default] ** 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 215.8 0.02 0.00 1 0.020 0 0 0.0 0.0 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T06:04:46.685 INFO:journalctl@ceph.mon.c.smithi107.stdout: Flush(GB): cumulative 0.004, interval 0.004 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: Cumulative compaction: 0.00 GB write, 84.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: Interval compaction: 0.00 GB write, 84.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: Block cache BinnedLRUCache@0x55b00a631350#8 capacity: 512.00 MB usage: 12.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,8.08 KB,0.00154078%) Misc(1,0.00 KB,0%) 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: 2023-10-09T06:04:46.686 INFO:journalctl@ceph.mon.c.smithi107.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:04:46.687 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:46 smithi107 ceph-mon[220921]: starting mon.c rank 2 at public addrs [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] at bind addrs [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:04:47.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 110145 110129 92.9517 57.3125 0.0153829 0.0107539 2023-10-09T06:04:48.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 111096 111080 92.5051 59.4375 0.0226692 0.0108053 2023-10-09T06:04:49.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:48 smithi150 conmon[178500]: ts=2023-10-09T06:04:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:49.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 112043 112027 92.0669 59.1875 0.0120684 0.0108577 2023-10-09T06:04:50.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 112973 112957 91.6262 58.125 0.0186664 0.0109091 2023-10-09T06:04:51.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 113912 113896 91.2041 58.6875 0.0107879 0.0109593 2023-10-09T06:04:51.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:04:51.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:04:51.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:04:51.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1214845808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:51 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:04:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:51 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:04:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:51 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:04:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:51 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1214845808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:52.406 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:04:52] "GET /metrics HTTP/1.1" 200 36083 "" "Prometheus/2.43.0" 2023-10-09T06:04:52.473 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 114694 114678 90.6685 48.875 0.024647 0.0110243 2023-10-09T06:04:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:04:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:04:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:04:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1214845808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:53.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:04:53.474566+0000 min lat: 0.00129016 max lat: 0.329823 avg lat: 0.0110834 2023-10-09T06:04:53.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:04:53.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 115531 115515 90.189 52.3125 0.0182311 0.0110834 2023-10-09T06:04:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: mon.c calling monitor election 2023-10-09T06:04:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:04:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: mon.b calling monitor election 2023-10-09T06:04:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T06:04:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:04:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:04:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:04:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:04:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: mgrmap e34: y(active, since 96s), standbys: x 2023-10-09T06:04:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:04:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:04:54.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: mon.c calling monitor election 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: mon.b calling monitor election 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: mon.a calling monitor election 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:04:54.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: mgrmap e34: y(active, since 96s), standbys: x 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:53 smithi107 ceph-mon[220921]: overall HEALTH_OK 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: mon.c calling monitor election 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: pgmap v56: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: mon.b calling monitor election 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: mon.a calling monitor election 2023-10-09T06:04:54.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: monmap e3: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: mgrmap e34: y(active, since 96s), standbys: x 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: Cluster is now healthy 2023-10-09T06:04:54.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:53 smithi150 ceph-mon[105942]: overall HEALTH_OK 2023-10-09T06:04:54.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 116361 116345 89.7161 51.875 0.0167503 0.0111417 2023-10-09T06:04:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:54 smithi107 ceph-mon[212750]: mgrmap e35: y(active, since 97s), standbys: x 2023-10-09T06:04:55.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:54 smithi107 ceph-mon[220921]: mgrmap e35: y(active, since 97s), standbys: x 2023-10-09T06:04:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:54 smithi150 ceph-mon[105942]: mgrmap e35: y(active, since 97s), standbys: x 2023-10-09T06:04:55.241 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T06:04:55.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 117080 117064 89.1701 44.9375 0.0316217 0.0112091 2023-10-09T06:04:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:55 smithi107 ceph-mon[212750]: pgmap v57: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T06:04:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1895946735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:56.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:55 smithi107 ceph-mon[220921]: pgmap v57: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T06:04:56.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1895946735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:55 smithi150 ceph-mon[105942]: pgmap v57: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T06:04:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:55 smithi150 ceph-mon[105942]: from='client.? 172.21.15.107:0/1895946735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:04:56.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 117975 117959 88.7698 55.9375 0.010603 0.0112609 2023-10-09T06:04:56.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:04:56 smithi107 conmon[200038]: 2023-10-09T06:04:56.362+0000 7f5488b69700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T06:04:57.474 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 118558 118542 88.1471 36.4375 0.0138447 0.0113401 2023-10-09T06:04:57.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:57 smithi107 ceph-mon[212750]: pgmap v58: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T06:04:57.752 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:57 smithi107 ceph-mon[220921]: pgmap v58: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T06:04:57.788 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:04:57.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:57 smithi150 ceph-mon[105942]: pgmap v58: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-09T06:04:58.475 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 119351 119335 87.6933 49.5625 0.0228031 0.0113988 2023-10-09T06:04:58.828 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:04:59.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:04:58 smithi150 conmon[178500]: ts=2023-10-09T06:04:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:04:59.475 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 120083 120067 87.2058 45.75 0.0238044 0.0114622 2023-10-09T06:05:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[212750]: pgmap v59: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:05:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:59 smithi150 ceph-mon[105942]: pgmap v59: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:05:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:59 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:04:59 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[220921]: pgmap v59: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:05:00.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:04:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:00.475 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 120682 120666 86.6335 37.4375 0.00608062 0.0115384 2023-10-09T06:05:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[212750]: pgmap v60: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:05:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:01 smithi107 ceph-mon[220921]: pgmap v60: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:05:01.479 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 121450 121434 86.1935 48 0.028303 0.0115973 2023-10-09T06:05:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:01 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:01 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:01 smithi150 ceph-mon[105942]: pgmap v60: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s 2023-10-09T06:05:01.809 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:05:01.809 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (3m) - 8m 26.6M - 0.25.0 c8568f914cd2 0715bcb46fa1 2023-10-09T06:05:01.809 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (3m) - 7m 82.7M - 9.4.7 2c41d148cca3 96db62cb38d0 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (7m) - 7m 21.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (7m) - 7m 15.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443,9283,8765 running (2m) - 11m 449M - 18.2.0-598-gb4fa663f 3060897fdf97 7326a92d4fbc 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:8443,9283,8765 running (3m) - 13m 505M - 18.2.0-598-gb4fa663f 3060897fdf97 19d12e0e339c 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (115s) - 13m 52.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7f66f5126e5d 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (12m) - 12m 93.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 0a878f231494 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (15s) - 12m 30.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 6c90cc5e902c 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (3m) - 7m 18.7M - 1.5.0 0da6a335fe13 32eddd377f02 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (3m) - 7m 16.8M - 1.5.0 0da6a335fe13 6b4c6b51266f 2023-10-09T06:05:01.810 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (11m) - 11m 2766M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 8de9f1d99699 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (11m) - 11m 2012M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 674ef542e3e9 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (10m) - 10m 3641M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4f4ef8822b1f 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (10m) - 10m 2349M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e1f7d3e8c22d 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (10m) - 10m 2692M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (9m) - 9m 2976M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (9m) - 9m 1420M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (8m) - 8m 2422M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T06:05:01.811 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (2m) - 8m 53.6M - 2.43.0 a07b618ecd1d 3a1d67d52b64 2023-10-09T06:05:02.020 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:02] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:05:02.389 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:02.390 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:02 smithi107 ceph-mon[220921]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:02.391 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:02.391 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:02 smithi107 ceph-mon[212750]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:02.477 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 122219 122203 85.7647 48.0625 0.0174382 0.0116548 2023-10-09T06:05:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:02 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:02 smithi150 ceph-mon[105942]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:03.477 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0087 2023-10-09T06:05:03.477 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 122922 2023-10-09T06:05:03.477 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 85.3542 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 44.5262 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 257.938 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 36.4375 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1365 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 712.42 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 4127 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 583 2023-10-09T06:05:03.478 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0117122 2023-10-09T06:05:03.479 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0108445 2023-10-09T06:05:03.479 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 0.380738 2023-10-09T06:05:03.479 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00129016 2023-10-09T06:05:03.483 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:05:03.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:03 smithi107 ceph-mon[212750]: pgmap v61: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T06:05:03.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:03 smithi107 ceph-mon[220921]: pgmap v61: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T06:05:03.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:05:03.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:03 smithi150 ceph-mon[105942]: pgmap v61: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s 2023-10-09T06:05:04.566 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:05.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:05 smithi107 ceph-mon[220921]: pgmap v62: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T06:05:05.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:05 smithi107 ceph-mon[212750]: pgmap v62: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T06:05:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:05 smithi150 ceph-mon[105942]: pgmap v62: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-10-09T06:05:07.404 INFO:teuthology.orchestra.run.smithi107.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:05:07.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:07 smithi107 ceph-mon[212750]: pgmap v63: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s 2023-10-09T06:05:07.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:07.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:07 smithi107 ceph-mon[220921]: pgmap v63: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s 2023-10-09T06:05:07.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:07 smithi150 ceph-mon[105942]: pgmap v63: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s 2023-10-09T06:05:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:07 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:08 smithi107 ceph-mon[212750]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:08 smithi107 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:05:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:08 smithi107 ceph-mon[220921]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:08 smithi107 ceph-mon[220921]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:05:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:08 smithi150 ceph-mon[105942]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:05:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:08 smithi150 ceph-mon[105942]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:05:08.904 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:08 smithi150 conmon[178500]: ts=2023-10-09T06:05:08.668Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:09.079 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:05:09.587 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: pgmap v64: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 957 op/s 2023-10-09T06:05:09.587 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.587 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.587 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: pgmap v64: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 957 op/s 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:09.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.863 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: pgmap v64: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 957 op/s 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:09 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:10.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:10.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:10.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:10.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:10.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:10.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:10.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:10.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:10 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:05:10.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:10 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:05:10.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:10 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:05:10.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:10 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:05:11.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:11 smithi107 ceph-mon[220921]: pgmap v65: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 635 op/s 2023-10-09T06:05:11.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:11 smithi107 ceph-mon[212750]: pgmap v65: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 635 op/s 2023-10-09T06:05:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:11 smithi150 ceph-mon[105942]: pgmap v65: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 635 op/s 2023-10-09T06:05:12.161 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:12] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:05:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:13 smithi107 ceph-mon[212750]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 829 op/s 2023-10-09T06:05:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:13 smithi150 ceph-mon[105942]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 829 op/s 2023-10-09T06:05:14.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:13 smithi107 ceph-mon[220921]: pgmap v66: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 23 MiB/s wr, 829 op/s 2023-10-09T06:05:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:15 smithi107 ceph-mon[220921]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 793 op/s 2023-10-09T06:05:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:15 smithi107 ceph-mon[212750]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 793 op/s 2023-10-09T06:05:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:15 smithi150 ceph-mon[105942]: pgmap v67: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 793 op/s 2023-10-09T06:05:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:17 smithi107 ceph-mon[220921]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 642 op/s 2023-10-09T06:05:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:17 smithi107 ceph-mon[212750]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 642 op/s 2023-10-09T06:05:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:17 smithi150 ceph-mon[105942]: pgmap v68: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 642 op/s 2023-10-09T06:05:19.155 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:18 smithi150 conmon[178500]: ts=2023-10-09T06:05:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:20.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:19 smithi107 ceph-mon[212750]: pgmap v69: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 1.00k op/s 2023-10-09T06:05:20.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:19 smithi107 ceph-mon[220921]: pgmap v69: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 1.00k op/s 2023-10-09T06:05:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:19 smithi150 ceph-mon[105942]: pgmap v69: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 3.7 MiB/s wr, 1.00k op/s 2023-10-09T06:05:22.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:21 smithi107 ceph-mon[212750]: pgmap v70: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:05:22.015 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:22] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:05:22.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:21 smithi107 ceph-mon[220921]: pgmap v70: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:05:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:21 smithi150 ceph-mon[105942]: pgmap v70: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-09T06:05:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:23 smithi107 ceph-mon[220921]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:05:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:23 smithi107 ceph-mon[212750]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:05:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:23 smithi150 ceph-mon[105942]: pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 784 op/s 2023-10-09T06:05:26.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:25 smithi107 ceph-mon[220921]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:05:26.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:25 smithi107 ceph-mon[212750]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:05:26.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:25 smithi150 ceph-mon[105942]: pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-09T06:05:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:27 smithi107 ceph-mon[212750]: pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:05:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:27 smithi150 ceph-mon[105942]: pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:05:28.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:27 smithi107 ceph-mon[220921]: pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-09T06:05:29.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:28 smithi150 conmon[178500]: ts=2023-10-09T06:05:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:30.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:29 smithi107 ceph-mon[212750]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-10-09T06:05:30.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:29 smithi107 ceph-mon[220921]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-10-09T06:05:30.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:29 smithi150 ceph-mon[105942]: pgmap v74: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-10-09T06:05:32.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:32] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.43.0" 2023-10-09T06:05:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:31 smithi107 ceph-mon[212750]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:05:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:31 smithi107 ceph-mon[220921]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:05:32.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:31 smithi150 ceph-mon[105942]: pgmap v75: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:05:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:31 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:34.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:33 smithi107 ceph-mon[212750]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-09T06:05:34.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:33 smithi107 ceph-mon[220921]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-09T06:05:34.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:33 smithi150 ceph-mon[105942]: pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-09T06:05:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:35 smithi107 ceph-mon[212750]: pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:05:36.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:35 smithi107 ceph-mon[220921]: pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:05:36.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:35 smithi150 ceph-mon[105942]: pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:05:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:37 smithi107 ceph-mon[212750]: pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:05:38.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:37 smithi107 ceph-mon[220921]: pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:05:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:37 smithi150 ceph-mon[105942]: pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:05:39.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:38 smithi150 conmon[178500]: ts=2023-10-09T06:05:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:39 smithi150 ceph-mon[105942]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-10-09T06:05:40.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:39 smithi107 ceph-mon[212750]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-10-09T06:05:40.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:39 smithi107 ceph-mon[220921]: pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2023-10-09T06:05:42.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:42] "GET /metrics HTTP/1.1" 200 36168 "" "Prometheus/2.43.0" 2023-10-09T06:05:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:41 smithi107 ceph-mon[212750]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:05:42.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:41 smithi107 ceph-mon[220921]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:05:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:41 smithi150 ceph-mon[105942]: pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:05:44.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:43 smithi107 ceph-mon[212750]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:05:44.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:43 smithi107 ceph-mon[220921]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:05:44.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:43 smithi150 ceph-mon[105942]: pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2023-10-09T06:05:46.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:45 smithi107 ceph-mon[212750]: pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:05:46.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:45 smithi107 ceph-mon[220921]: pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:05:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:45 smithi150 ceph-mon[105942]: pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-10-09T06:05:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:46 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:05:48.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:47 smithi107 ceph-mon[212750]: pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T06:05:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:47 smithi107 ceph-mon[220921]: pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T06:05:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:47 smithi150 ceph-mon[105942]: pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-10-09T06:05:49.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:48 smithi150 conmon[178500]: ts=2023-10-09T06:05:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:05:50.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:49 smithi107 ceph-mon[212750]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-10-09T06:05:50.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:49 smithi107 ceph-mon[220921]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-10-09T06:05:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:49 smithi150 ceph-mon[105942]: pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-10-09T06:05:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:51 smithi150 ceph-mon[105942]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:52.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:05:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:05:52] "GET /metrics HTTP/1.1" 200 36165 "" "Prometheus/2.43.0" 2023-10-09T06:05:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:51 smithi107 ceph-mon[220921]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:51 smithi107 ceph-mon[212750]: pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:54.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:53 smithi107 ceph-mon[220921]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:53 smithi107 ceph-mon[212750]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:53 smithi150 ceph-mon[105942]: pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-09T06:05:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:55 smithi107 ceph-mon[212750]: pgmap v87: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T06:05:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:55 smithi107 ceph-mon[220921]: pgmap v87: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T06:05:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:55 smithi150 ceph-mon[105942]: pgmap v87: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T06:05:58.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:57 smithi107 ceph-mon[220921]: pgmap v88: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:05:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:57 smithi107 ceph-mon[212750]: pgmap v88: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:05:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:57 smithi150 ceph-mon[105942]: pgmap v88: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-09T06:05:59.154 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:05:58 smithi150 conmon[178500]: ts=2023-10-09T06:05:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:05:59 smithi107 ceph-mon[212750]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-10-09T06:06:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:05:59 smithi107 ceph-mon[220921]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-10-09T06:06:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:05:59 smithi150 ceph-mon[105942]: pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-10-09T06:06:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:01 smithi150 ceph-mon[105942]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:01 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:02.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:01 smithi107 ceph-mon[212750]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:02.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:02.405 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:02] "GET /metrics HTTP/1.1" 200 36164 "" "Prometheus/2.43.0" 2023-10-09T06:06:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:01 smithi107 ceph-mon[220921]: pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:04.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:03 smithi107 ceph-mon[220921]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:04.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:03 smithi107 ceph-mon[212750]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:03 smithi150 ceph-mon[105942]: pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:05 smithi107 ceph-mon[212750]: pgmap v92: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:06.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:05 smithi150 ceph-mon[105942]: pgmap v92: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:06.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:05 smithi107 ceph-mon[220921]: pgmap v92: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-09T06:06:08.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:07 smithi107 ceph-mon[212750]: pgmap v93: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:07 smithi150 ceph-mon[105942]: pgmap v93: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:07 smithi107 ceph-mon[220921]: pgmap v93: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-09T06:06:09.159 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:06:08 smithi150 conmon[178500]: ts=2023-10-09T06:06:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:10.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:10 smithi107 ceph-mon[212750]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T06:06:10.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:10 smithi107 ceph-mon[220921]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T06:06:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:10 smithi150 ceph-mon[105942]: pgmap v94: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2023-10-09T06:06:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:10 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:12 smithi107 ceph-mon[212750]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:12.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:12 smithi107 ceph-mon[220921]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:12.405 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:12] "GET /metrics HTTP/1.1" 200 36164 "" "Prometheus/2.43.0" 2023-10-09T06:06:12.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:12 smithi150 ceph-mon[105942]: pgmap v95: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:13.926 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:06:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:14 smithi107 ceph-mon[212750]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:14 smithi107 ceph-mon[220921]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:14 smithi150 ceph-mon[105942]: pgmap v96: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-10-09T06:06:14.733 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[212750]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:15 smithi107 ceph-mon[220921]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2023-10-09T06:06:15.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:15 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:15 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:15.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:15 smithi150 ceph-mon[105942]: pgmap v97: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2023-10-09T06:06:16.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:16 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:17.654 INFO:teuthology.orchestra.run.smithi107.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[212750]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[220921]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:17 smithi150 ceph-mon[105942]: pgmap v98: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-09T06:06:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:17 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.513 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T06:06:18.905 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:06:18 smithi150 conmon[178500]: ts=2023-10-09T06:06:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:18.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:18.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:06:18.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:06:18.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:18.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:18.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:18.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:18 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:18.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:18.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:18.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:18.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:06:18.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T06:06:18.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T06:06:19.257 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:20 smithi150 ceph-mon[105942]: Upgrade: It appears safe to stop mon.b 2023-10-09T06:06:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:20 smithi150 ceph-mon[105942]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-10-09T06:06:20.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:20 smithi107 ceph-mon[220921]: Upgrade: It appears safe to stop mon.b 2023-10-09T06:06:20.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:20 smithi107 ceph-mon[220921]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-10-09T06:06:20.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:20 smithi107 ceph-mon[212750]: Upgrade: It appears safe to stop mon.b 2023-10-09T06:06:20.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:20 smithi107 ceph-mon[212750]: pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-10-09T06:06:21.181 INFO:teuthology.orchestra.run.smithi107.stdout:wait for all mons to upgrade 2023-10-09T06:06:21.932 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T06:06:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:22 smithi150 ceph-mon[105942]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:22 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:22 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:22 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:22 smithi150 ceph-mon[105942]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[212750]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:22.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:22.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:22.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:22.406 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:22] "GET /metrics HTTP/1.1" 200 36092 "" "Prometheus/2.43.0" 2023-10-09T06:06:22.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[220921]: pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-09T06:06:22.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:22.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:22.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:22.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:22.668 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:23.214 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:23 smithi107 ceph-mon[220921]: Upgrade: Updating mon.b 2023-10-09T06:06:23.214 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:23 smithi107 ceph-mon[220921]: Deploying daemon mon.b on smithi150 2023-10-09T06:06:23.216 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:23 smithi107 ceph-mon[212750]: Upgrade: Updating mon.b 2023-10-09T06:06:23.216 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:23 smithi107 ceph-mon[212750]: Deploying daemon mon.b on smithi150 2023-10-09T06:06:23.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:23 smithi150 ceph-mon[105942]: Upgrade: Updating mon.b 2023-10-09T06:06:23.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:23 smithi150 ceph-mon[105942]: Deploying daemon mon.b on smithi150 2023-10-09T06:06:23.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:23 smithi150 systemd[1]: Stopping Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:24.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:23 smithi150 conmon[105918]: 2023-10-09T06:06:23.728+0000 7f8dca52a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:06:24.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:23 smithi150 conmon[105918]: 2023-10-09T06:06:23.728+0000 7f8dca52a700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T06:06:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:24 smithi150 bash[193772]: 0a878f23149465e5d975db267aa084b2a1b166f9f6d3c833402f731276140fa0 2023-10-09T06:06:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:24 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T06:06:25.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:24 smithi150 systemd[1]: Stopped Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:25.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 systemd[1]: Starting Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 bash[194116]: de70d36439206106a3262ce948fb55d14d907326a41d12bed39f5700dbdc5b57 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: pidfile_write: ignore empty --pid-file 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: load: jerasure load: lrc 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: RocksDB version: 7.9.2 2023-10-09T06:06:26.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Git sha 0 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: DB SUMMARY 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: DB Session ID: A1SOBR5ZSWPMFEF2JLWI 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: CURRENT file: CURRENT 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1087 Bytes 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9447058 ; 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.error_if_exists: 0 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.create_if_missing: 0 2023-10-09T06:06:26.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.paranoid_checks: 1 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.env: 0x558ccff91720 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.info_log: 0x558cd300d420 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.statistics: (nil) 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.use_fsync: 0 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_log_file_size: 0 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T06:06:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_fallocate: 1 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.use_direct_reads: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.db_log_dir: 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.wal_dir: 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T06:06:26.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.write_buffer_manager: 0x558cd229cc80 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T06:06:26.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.unordered_write: 0 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.row_cache: None 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.wal_filter: None 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T06:06:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.two_write_queues: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.wal_compression: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.atomic_flush: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.log_readahead_size: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T06:06:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_background_jobs: 2 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_background_compactions: -1 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_subcompactions: 1 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T06:06:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_open_files: -1 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_background_flushes: -1 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Compression algorithms supported: 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kZSTD supported: 0 2023-10-09T06:06:26.413 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kXpressCompression supported: 0 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kZlibCompression supported: 1 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kSnappyCompression supported: 1 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kLZ4Compression supported: 1 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: kBZip2Compression supported: 0 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.merge_operator: 2023-10-09T06:06:26.414 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_filter: None 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558cd300d520) 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks: 1 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: pin_top_level_index_and_filter: 1 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_type: 0 2023-10-09T06:06:26.415 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_index_type: 0 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_shortening: 1 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: checksum: 4 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: no_block_cache: 0 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache: 0x558cd231f350 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_name: BinnedLRUCache 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_options: 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: capacity : 536870912 2023-10-09T06:06:26.416 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_shard_bits : 4 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: strict_capacity_limit : 0 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: high_pri_pool_ratio: 0.000 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_cache_compressed: (nil) 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: persistent_cache: (nil) 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size: 4096 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_size_deviation: 10 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_restart_interval: 16 2023-10-09T06:06:26.417 INFO:journalctl@ceph.mon.b.smithi150.stdout: index_block_restart_interval: 1 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: metadata_block_size: 4096 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: partition_filters: 0 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: use_delta_encoding: 1 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: filter_policy: bloomfilter 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: whole_key_filtering: 1 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: verify_compression: 0 2023-10-09T06:06:26.418 INFO:journalctl@ceph.mon.b.smithi150.stdout: read_amp_bytes_per_bit: 0 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: format_version: 5 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_index_compression: 1 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: block_align: 0 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: max_auto_readahead_size: 262144 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: prepopulate_block_cache: 0 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: initial_auto_readahead_size: 8192 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T06:06:26.419 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression: NoCompression 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.num_levels: 7 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T06:06:26.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T06:06:26.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T06:06:26.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T06:06:26.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T06:06:26.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:06:26.422 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T06:06:26.423 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 systemd[1]: Started Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T06:06:26.424 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T06:06:26.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T06:06:26.426 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T06:06:26.427 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T06:06:26.428 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.table_properties_collectors: 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.inplace_update_support: 0 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.bloom_locality: 0 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.max_successive_merges: 0 2023-10-09T06:06:26.429 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.ttl: 2592000 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enable_blob_files: false 2023-10-09T06:06:26.430 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.min_blob_size: 0 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12896, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T06:06:26.431 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 954e5817-a921-4b9f-8b24-a68cf5f29f93 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831585923638, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831585960139, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5739516, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12901, "largest_seqno": 17547, "table_properties": {"data_size": 5723160, "index_size": 10855, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4485, "raw_key_size": 38125, "raw_average_key_size": 21, "raw_value_size": 5687528, "raw_average_value_size": 3216, "num_data_blocks": 489, "num_entries": 1768, "num_filter_entries": 1768, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696831585, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "954e5817-a921-4b9f-8b24-a68cf5f29f93", "db_session_id": "A1SOBR5ZSWPMFEF2JLWI", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696831585960276, "job": 1, "event": "recovery_finished"} 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558cd23ba000 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: DB pointer 0x558cd23a8000 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: starting mon.b rank 1 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:06:26.432 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** DB Stats ** 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T06:06:26.433 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: L0 1/0 5.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 306.3 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: L6 1/0 5.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: Sum 2/0 10.72 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 306.3 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-10-09T06:06:26.434 INFO:journalctl@ceph.mon.b.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 306.3 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** Compaction Stats [default] ** 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 306.3 0.02 0.00 1 0.018 0 0 0.0 0.0 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T06:06:26.435 INFO:journalctl@ceph.mon.b.smithi150.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: Cumulative compaction: 0.01 GB write, 112.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: Interval compaction: 0.01 GB write, 112.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: Block cache BinnedLRUCache@0x558cd231f350#7 capacity: 512.00 MB usage: 60.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.3e-05 secs_since: 0 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: Block cache entry stats(count,size,portion): DataBlock(4,22.81 KB,0.00435114%) FilterBlock(2,12.56 KB,0.00239611%) IndexBlock(2,25.16 KB,0.00479817%) Misc(1,0.00 KB,0%) 2023-10-09T06:06:26.436 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???) e3 preinit fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).mds e9 new map 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).mds e9 print_map 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: e9 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: legacy client fscid: 1 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: Filesystem 'foo' (1) 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: fs_name foo 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: epoch 8 2023-10-09T06:06:26.437 INFO:journalctl@ceph.mon.b.smithi150.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: created 2023-10-09T05:57:53.253294+0000 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: modified 2023-10-09T05:58:03.974541+0000 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: tableserver 0 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: root 0 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: session_timeout 60 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: session_autoclose 300 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: max_file_size 1099511627776 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: required_client_features {} 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: last_failure 0 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: last_failure_osd_epoch 0 2023-10-09T06:06:26.438 INFO:journalctl@ceph.mon.b.smithi150.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: max_mds 1 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: in 0 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: up {0=24560} 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: failed 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: damaged 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: stopped 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: data_pools [4] 2023-10-09T06:06:26.439 INFO:journalctl@ceph.mon.b.smithi150.stdout: metadata_pool 3 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: inline_data disabled 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: balancer 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: bal_rank_mask -1 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: standby_count_wanted 1 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: [mds.foo.smithi107.dqtsqc{0:24560} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.107:6834/3714925019,v1:172.21.15.107:6835/3714925019] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.440 INFO:journalctl@ceph.mon.b.smithi150.stdout: Standby daemons: 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout: 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout: [mds.foo.smithi150.znbnco{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.150:6832/900540054,v1:172.21.15.150:6833/900540054] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T06:06:26.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:25 smithi150 ceph-mon[194178]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-09T06:06:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:27 smithi107 ceph-mon[212750]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:06:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:27 smithi107 ceph-mon[220921]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:06:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:27 smithi150 ceph-mon[194178]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T06:06:27.900 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T06:06:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: pgmap v103: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: overall HEALTH_OK 2023-10-09T06:06:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: mon.a calling monitor election 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: mon.b calling monitor election 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: mon.c calling monitor election 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: monmap e4: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: overall HEALTH_OK 2023-10-09T06:06:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/450800866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: pgmap v103: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: overall HEALTH_OK 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:06:28.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: mon.a calling monitor election 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: mon.b calling monitor election 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: mon.c calling monitor election 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: monmap e4: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: overall HEALTH_OK 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/450800866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: pgmap v103: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: mon.a calling monitor election 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: mon.b calling monitor election 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: mon.c calling monitor election 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T06:06:28.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: monmap e4: 3 mons at {a=[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0],b=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],c=[v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]} removed_ranks: {} 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: fsmap foo:1 {0=foo.smithi107.dqtsqc=up:active} 1 up:standby 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: mgrmap e35: y(active, since 3m), standbys: x 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: overall HEALTH_OK 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:06:28.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/450800866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T06:06:29.126 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:06:29.159 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:06:28 smithi150 conmon[178500]: ts=2023-10-09T06:06:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi150.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.107\", device_class=\"ssd\", hostname=\"smithi107\", instance=\"smithi107:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.107\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T06:06:29.848 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:30 smithi150 ceph-mon[194178]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:30.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:30 smithi107 ceph-mon[220921]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:30.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:30 smithi107 ceph-mon[212750]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:31.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:30 smithi107 conmon[200038]: 2023-10-09T06:06:30.981+0000 7f5488b69700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T06:06:32.258 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:06:32.258 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (5m) - 9m 26.6M - 0.25.0 c8568f914cd2 0715bcb46fa1 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (4m) - 9m 91.6M - 9.4.7 2c41d148cca3 96db62cb38d0 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (8m) - 8m 22.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (8m) - 8m 15.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443,9283,8765 running (3m) - 13m 450M - 18.2.0-598-gb4fa663f 3060897fdf97 7326a92d4fbc 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:8443,9283,8765 running (4m) - 15m 507M - 18.2.0-598-gb4fa663f 3060897fdf97 19d12e0e339c 2023-10-09T06:06:32.259 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (3m) - 15m 59.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7f66f5126e5d 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (6s) - 13m 34.1M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 de70d3643920 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (106s) - 13m 43.5M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 6c90cc5e902c 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (5m) - 9m 18.6M - 1.5.0 0da6a335fe13 32eddd377f02 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (4m) - 9m 17.2M - 1.5.0 0da6a335fe13 6b4c6b51266f 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (13m) - 13m 3101M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 8de9f1d99699 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (12m) - 12m 2387M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 674ef542e3e9 2023-10-09T06:06:32.260 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (12m) - 12m 4047M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 4f4ef8822b1f 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (11m) - 11m 3012M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 e1f7d3e8c22d 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (11m) - 11m 3282M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (11m) - 11m 3706M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (10m) - 10m 1751M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (10m) - 10m 3232M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T06:06:32.261 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (4m) - 9m 63.2M - 2.43.0 a07b618ecd1d 3a1d67d52b64 2023-10-09T06:06:32.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[220921]: pgmap v105: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:32.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:32.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:32.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:32.305 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:32] "GET /metrics HTTP/1.1" 200 36162 "" "Prometheus/2.43.0" 2023-10-09T06:06:32.305 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[212750]: pgmap v105: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:32.305 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:32.305 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:32.305 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:32 smithi150 ceph-mon[194178]: pgmap v105: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.121 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:06:33.802 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:33.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:33 smithi150 ceph-mon[194178]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:33.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:33 smithi150 ceph-mon[194178]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:33.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:33 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:33 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[220921]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[220921]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[212750]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[212750]: pgmap v106: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:06:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:33.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:33 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:36.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:35 smithi150 ceph-mon[194178]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T06:06:36.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:35 smithi107 ceph-mon[212750]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T06:06:36.160 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:35 smithi107 ceph-mon[220921]: pgmap v107: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-09T06:06:36.374 INFO:teuthology.orchestra.run.smithi107.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:37.271 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T06:06:37.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[212750]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[212750]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[220921]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[220921]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:37.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:37 smithi107 ceph-mon[220921]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T06:06:37.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:37 smithi150 ceph-mon[194178]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:06:37.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:37 smithi150 ceph-mon[194178]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:06:37.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:37.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:37 smithi150 ceph-mon[194178]: pgmap v108: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T06:06:37.960 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: Reconfiguring mon.a (monmap changed)... 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:39.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[212750]: Reconfiguring daemon mon.a on smithi107 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:39.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: Reconfiguring mon.a (monmap changed)... 2023-10-09T06:06:39.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:39.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:39.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:39 smithi107 ceph-mon[220921]: Reconfiguring daemon mon.a on smithi107 2023-10-09T06:06:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:06:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: pgmap v109: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: Reconfiguring mon.a (monmap changed)... 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:39.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:39 smithi150 ceph-mon[194178]: Reconfiguring daemon mon.a on smithi107 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:41.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: pgmap v110: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:06:41.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:42.347 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:42] "GET /metrics HTTP/1.1" 200 36162 "" "Prometheus/2.43.0" 2023-10-09T06:06:42.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:42 smithi107 ceph-mon[212750]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:06:42.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:42 smithi107 ceph-mon[212750]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T06:06:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:42 smithi150 ceph-mon[194178]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:06:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:42 smithi150 ceph-mon[194178]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T06:06:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:42 smithi107 ceph-mon[220921]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T06:06:42.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:42 smithi107 ceph-mon[220921]: Reconfiguring daemon mgr.y on smithi107 2023-10-09T06:06:43.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:06:43.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:43.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:43.629 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:43.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:06:43.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:43.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:43.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:43.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: pgmap v111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T06:06:43.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:43.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:43.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:43.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:44 smithi107 ceph-mon[220921]: Reconfiguring mon.c (monmap changed)... 2023-10-09T06:06:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:44 smithi107 ceph-mon[220921]: Reconfiguring daemon mon.c on smithi107 2023-10-09T06:06:44.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:44 smithi107 ceph-mon[212750]: Reconfiguring mon.c (monmap changed)... 2023-10-09T06:06:44.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:44 smithi107 ceph-mon[212750]: Reconfiguring daemon mon.c on smithi107 2023-10-09T06:06:44.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:44 smithi150 ceph-mon[194178]: Reconfiguring mon.c (monmap changed)... 2023-10-09T06:06:44.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:44 smithi150 ceph-mon[194178]: Reconfiguring daemon mon.c on smithi107 2023-10-09T06:06:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:45 smithi107 ceph-mon[212750]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T06:06:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:45.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:45 smithi107 ceph-mon[220921]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T06:06:45.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:45 smithi150 ceph-mon[194178]: pgmap v112: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T06:06:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:46.776 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.0 on smithi107 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:06:46.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:46.778 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.0 on smithi107 2023-10-09T06:06:46.778 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T06:06:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:06:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.0 on smithi107 2023-10-09T06:06:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:06:48.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:47 smithi107 ceph-mon[220921]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T06:06:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:47 smithi150 ceph-mon[194178]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T06:06:48.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:47 smithi107 ceph-mon[212750]: pgmap v113: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:49.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.1 on smithi107 2023-10-09T06:06:49.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:06:49.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:06:49.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:49.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:48 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.1 on smithi107 2023-10-09T06:06:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T06:06:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:06:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:49.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:48 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.1 on smithi107 2023-10-09T06:06:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:49 smithi107 ceph-mon[220921]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:06:50.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:49 smithi107 ceph-mon[212750]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:06:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:49 smithi150 ceph-mon[194178]: pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-09T06:06:51.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:06:51.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:06:51.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:51.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.2 on smithi107 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:51.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:51 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.2 on smithi107 2023-10-09T06:06:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T06:06:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:06:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:51 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.2 on smithi107 2023-10-09T06:06:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:06:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:06:52] "GET /metrics HTTP/1.1" 200 36160 "" "Prometheus/2.43.0" 2023-10-09T06:06:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:52 smithi107 ceph-mon[220921]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T06:06:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:52 smithi107 ceph-mon[212750]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T06:06:52.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:52 smithi150 ceph-mon[194178]: pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-09T06:06:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:06:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:53.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:06:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:06:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:54 smithi150 ceph-mon[194178]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:06:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:54 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.3 on smithi107 2023-10-09T06:06:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:54 smithi150 ceph-mon[194178]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:06:54.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[212750]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:06:54.659 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.3 on smithi107 2023-10-09T06:06:54.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[212750]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:06:54.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[220921]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T06:06:54.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.3 on smithi107 2023-10-09T06:06:54.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:54 smithi107 ceph-mon[220921]: pgmap v116: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:06:55.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: Reconfiguring mds.foo.smithi107.dqtsqc (monmap changed)... 2023-10-09T06:06:55.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:06:55.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: Reconfiguring daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[212750]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: Reconfiguring mds.foo.smithi107.dqtsqc (monmap changed)... 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: Reconfiguring daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T06:06:55.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:55 smithi107 ceph-mon[220921]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: Reconfiguring mds.foo.smithi107.dqtsqc (monmap changed)... 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi107.dqtsqc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: Reconfiguring daemon mds.foo.smithi107.dqtsqc on smithi107 2023-10-09T06:06:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:55 smithi150 ceph-mon[194178]: pgmap v117: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2023-10-09T06:06:57.486 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: Reconfiguring mon.b (monmap changed)... 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: Reconfiguring daemon mon.b on smithi150 2023-10-09T06:06:57.487 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:57 smithi150 ceph-mon[194178]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: Reconfiguring mon.b (monmap changed)... 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: Reconfiguring daemon mon.b on smithi150 2023-10-09T06:06:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[212750]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: Reconfiguring mon.b (monmap changed)... 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: Reconfiguring daemon mon.b on smithi150 2023-10-09T06:06:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:57 smithi107 ceph-mon[220921]: pgmap v118: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T06:06:59.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:06:59 smithi107 ceph-mon[212750]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T06:06:59.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:06:59 smithi150 ceph-mon[194178]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T06:06:59.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:06:59 smithi107 ceph-mon[220921]: pgmap v119: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T06:07:01.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:01.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:01.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T06:07:01.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T06:07:01.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: Reconfiguring daemon mgr.x on smithi150 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: Reconfiguring daemon mgr.x on smithi150 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T06:07:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: Reconfiguring daemon mgr.x on smithi150 2023-10-09T06:07:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: pgmap v120: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-09T06:07:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:02.405 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:02] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.43.0" 2023-10-09T06:07:03.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[212750]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:07:03.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:07:03.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:03.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.4 on smithi150 2023-10-09T06:07:03.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[220921]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:07:03.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:07:03.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:03.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:03 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.4 on smithi150 2023-10-09T06:07:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:03 smithi150 ceph-mon[194178]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T06:07:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T06:07:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:03 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.4 on smithi150 2023-10-09T06:07:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:04 smithi150 ceph-mon[194178]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:07:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:07:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[212750]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[220921]: pgmap v121: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:04.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T06:07:04.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:05.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:05 smithi107 ceph-mon[212750]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:07:05.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:05 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.5 on smithi150 2023-10-09T06:07:05.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:05 smithi107 ceph-mon[220921]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:07:05.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:05 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.5 on smithi150 2023-10-09T06:07:05.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:05 smithi150 ceph-mon[194178]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T06:07:05.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:05 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.5 on smithi150 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[212750]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:06.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[220921]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:07:06.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:06 smithi150 ceph-mon[194178]: pgmap v122: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2023-10-09T06:07:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T06:07:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:07.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:07 smithi107 ceph-mon[212750]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:07:07.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:07 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.6 on smithi150 2023-10-09T06:07:07.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:07 smithi107 ceph-mon[220921]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:07:07.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:07 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.6 on smithi150 2023-10-09T06:07:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:07 smithi150 ceph-mon[194178]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T06:07:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:07 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.6 on smithi150 2023-10-09T06:07:08.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:07:08.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:07:08.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[212750]: Reconfiguring daemon osd.7 on smithi150 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:08.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:08 smithi107 ceph-mon[220921]: Reconfiguring daemon osd.7 on smithi150 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: pgmap v123: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:08 smithi150 ceph-mon[194178]: Reconfiguring daemon osd.7 on smithi150 2023-10-09T06:07:09.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:09 smithi150 ceph-mon[194178]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:07:09.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:09 smithi107 ceph-mon[220921]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:07:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:09 smithi107 ceph-mon[212750]: pgmap v124: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: Reconfiguring mds.foo.smithi150.znbnco (monmap changed)... 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:11.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:10 smithi150 ceph-mon[194178]: Reconfiguring daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: Reconfiguring mds.foo.smithi150.znbnco (monmap changed)... 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[212750]: Reconfiguring daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: Reconfiguring mds.foo.smithi150.znbnco (monmap changed)... 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.znbnco", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:11.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:10 smithi107 ceph-mon[220921]: Reconfiguring daemon mds.foo.smithi150.znbnco on smithi150 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:12.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:12.406 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:12] "GET /metrics HTTP/1.1" 200 36151 "" "Prometheus/2.43.0" 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: pgmap v125: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:12.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:12.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:13.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:13.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:14.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:14 smithi107 ceph-mon[212750]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T06:07:14.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:14 smithi107 ceph-mon[220921]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T06:07:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:14 smithi150 ceph-mon[194178]: pgmap v126: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T06:07:16.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:16 smithi107 ceph-mon[220921]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2023-10-09T06:07:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:16 smithi107 ceph-mon[212750]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2023-10-09T06:07:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:16 smithi150 ceph-mon[194178]: pgmap v127: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2023-10-09T06:07:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:17.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:17.409 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:18 smithi150 ceph-mon[194178]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:07:18.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:18 smithi107 ceph-mon[212750]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:07:18.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:18 smithi107 ceph-mon[220921]: pgmap v128: 89 pgs: 89 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-09T06:07:20.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:20 smithi107 ceph-mon[212750]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2023-10-09T06:07:20.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:20 smithi107 ceph-mon[220921]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2023-10-09T06:07:20.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:20 smithi150 ceph-mon[194178]: pgmap v129: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2023-10-09T06:07:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:22] "GET /metrics HTTP/1.1" 200 36156 "" "Prometheus/2.43.0" 2023-10-09T06:07:22.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:22 smithi107 ceph-mon[220921]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:07:22.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:22 smithi107 ceph-mon[212750]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:07:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:22 smithi150 ceph-mon[194178]: pgmap v130: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:07:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:24 smithi150 ceph-mon[194178]: pgmap v131: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:07:24.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:24 smithi107 ceph-mon[212750]: pgmap v131: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:07:24.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:24 smithi107 ceph-mon[220921]: pgmap v131: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:07:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:26 smithi107 ceph-mon[212750]: pgmap v132: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:07:26.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:26 smithi107 ceph-mon[220921]: pgmap v132: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:07:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:26 smithi150 ceph-mon[194178]: pgmap v132: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T06:07:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:28 smithi107 ceph-mon[212750]: pgmap v133: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:28 smithi107 ceph-mon[220921]: pgmap v133: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:28 smithi150 ceph-mon[194178]: pgmap v133: 89 pgs: 89 active+clean; 950 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:30.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:30 smithi107 ceph-mon[212750]: pgmap v134: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T06:07:30.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:30 smithi107 ceph-mon[220921]: pgmap v134: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T06:07:30.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:30 smithi150 ceph-mon[194178]: pgmap v134: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-10-09T06:07:32.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:32 smithi107 ceph-mon[220921]: pgmap v135: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:07:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:32.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:32] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:07:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:32 smithi150 ceph-mon[194178]: pgmap v135: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:07:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:32.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:32 smithi107 ceph-mon[212750]: pgmap v135: 89 pgs: 89 active+clean; 699 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-09T06:07:32.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:34 smithi150 ceph-mon[194178]: pgmap v136: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T06:07:34.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:34 smithi107 ceph-mon[212750]: pgmap v136: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T06:07:34.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:34 smithi107 ceph-mon[220921]: pgmap v136: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-10-09T06:07:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:36 smithi107 ceph-mon[212750]: pgmap v137: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-10-09T06:07:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:36 smithi107 ceph-mon[220921]: pgmap v137: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-10-09T06:07:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:36 smithi150 ceph-mon[194178]: pgmap v137: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2023-10-09T06:07:38.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:38 smithi107 ceph-mon[212750]: pgmap v138: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:38.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:38 smithi107 ceph-mon[220921]: pgmap v138: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:38.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:38 smithi150 ceph-mon[194178]: pgmap v138: 89 pgs: 89 active+clean; 450 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2023-10-09T06:07:40.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:40 smithi107 ceph-mon[212750]: pgmap v139: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2023-10-09T06:07:40.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:40 smithi107 ceph-mon[220921]: pgmap v139: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2023-10-09T06:07:40.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:40 smithi150 ceph-mon[194178]: pgmap v139: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2023-10-09T06:07:41.734 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 122922 objects 2023-10-09T06:07:41.734 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :158.251 2023-10-09T06:07:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:42 smithi107 ceph-mon[212750]: pgmap v140: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:07:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:42] "GET /metrics HTTP/1.1" 200 36154 "" "Prometheus/2.43.0" 2023-10-09T06:07:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:42 smithi107 ceph-mon[220921]: pgmap v140: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:07:42.607 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T06:07:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:42 smithi150 ceph-mon[194178]: pgmap v140: 89 pgs: 89 active+clean; 200 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2023-10-09T06:07:42.956 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:43.526 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T06:07:43.526 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T06:07:44.048 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:44.247 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:44 smithi107 ceph-mon[212750]: pgmap v141: 89 pgs: 89 active+clean; 175 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:07:44.247 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:44 smithi107 ceph-mon[220921]: pgmap v141: 89 pgs: 89 active+clean; 175 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:07:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:44 smithi150 ceph-mon[194178]: pgmap v141: 89 pgs: 89 active+clean; 175 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T06:07:44.837 INFO:teuthology.orchestra.run.smithi107.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:07:45.618 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T06:07:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[212750]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-09T06:07:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[212750]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:07:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[212750]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:07:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:45.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[220921]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-09T06:07:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[220921]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:07:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[220921]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:07:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:45.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:45 smithi150 ceph-mon[194178]: pgmap v142: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-09T06:07:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:45 smithi150 ceph-mon[194178]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:07:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:45 smithi150 ceph-mon[194178]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:07:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:46.175 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:46.873 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_1' removed 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1212210915' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1212210915' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1212210915' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:07:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:07:47.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[220921]: pgmap v143: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:07:47.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[220921]: failed to encode map e83 with expected crc 2023-10-09T06:07:47.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[220921]: failed to encode map e83 with expected crc 2023-10-09T06:07:47.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:47.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[220921]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:48.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[212750]: pgmap v143: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:07:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T06:07:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T06:07:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:47 smithi107 ceph-mon[212750]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:47 smithi150 ceph-mon[194178]: pgmap v143: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T06:07:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:47 smithi150 ceph-mon[194178]: failed to encode map e83 with expected crc 2023-10-09T06:07:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:47 smithi150 ceph-mon[194178]: failed to encode map e83 with expected crc 2023-10-09T06:07:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:07:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:47 smithi150 ceph-mon[194178]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T06:07:48.273 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:07:48.273 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:07:48.274 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T06:07:48.274 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T06:07:48.325 INFO:teuthology.orchestra.run.smithi107.stdout:wait for half of osds to upgrade 2023-10-09T06:07:48.989 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T06:07:49.125 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:49.362 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[212750]: failed to encode map e83 with expected crc 2023-10-09T06:07:49.362 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:49.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:49.363 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[220921]: failed to encode map e83 with expected crc 2023-10-09T06:07:49.363 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:49.363 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:49 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:49 smithi150 ceph-mon[194178]: failed to encode map e83 with expected crc 2023-10-09T06:07:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:49 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:49 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:50.366 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:50 smithi107 ceph-mon[220921]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:50.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:50 smithi107 ceph-mon[212750]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:50 smithi150 ceph-mon[194178]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:50.419 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:52.048 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_2' created 2023-10-09T06:07:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:07:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:07:52] "GET /metrics HTTP/1.1" 200 36106 "" "Prometheus/2.43.0" 2023-10-09T06:07:52.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/332638337' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/332638337' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:07:52.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: pgmap v146: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/332638337' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:52.731 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T06:07:53.183 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: Upgrade: Setting container_image for all mon 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: Upgrade: Setting container_image for all crash 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: Upgrade: osd.0 is safe to restart 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:07:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2746713983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: Upgrade: Setting container_image for all mon 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: Upgrade: Setting container_image for all crash 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: Upgrade: osd.0 is safe to restart 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:07:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2746713983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: Upgrade: Setting container_image for all mon 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: Upgrade: Setting container_image for all crash 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: Upgrade: osd.0 is safe to restart 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: failed to encode map e84 with expected crc 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T06:07:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T06:07:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2746713983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: pgmap v148: 89 pgs: 4 creating+peering, 73 active+clean, 12 unknown; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.100 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: pgmap v148: 89 pgs: 4 creating+peering, 73 active+clean, 12 unknown; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.101 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:54.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:07:54.103 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: pgmap v148: 89 pgs: 4 creating+peering, 73 active+clean, 12 unknown; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: failed to encode map e85 with expected crc 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T06:07:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[212750]: Upgrade: Updating osd.0 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[212750]: Deploying daemon osd.0 on smithi107 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[212750]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T06:07:55.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[220921]: Upgrade: Updating osd.0 2023-10-09T06:07:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[220921]: Deploying daemon osd.0 on smithi107 2023-10-09T06:07:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[220921]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[220921]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:55 smithi107 ceph-mon[220921]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T06:07:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:55 smithi150 ceph-mon[194178]: Upgrade: Updating osd.0 2023-10-09T06:07:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:55 smithi150 ceph-mon[194178]: Deploying daemon osd.0 on smithi107 2023-10-09T06:07:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:55 smithi150 ceph-mon[194178]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:55 smithi150 ceph-mon[194178]: failed to encode map e86 with expected crc 2023-10-09T06:07:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:55 smithi150 ceph-mon[194178]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T06:07:56.050 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[212750]: pgmap v151: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:56.051 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T06:07:56.051 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4226703785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:56.051 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[220921]: pgmap v151: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:56.051 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[220921]: failed to encode map e86 with expected crc 2023-10-09T06:07:56.051 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4226703785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:56 smithi150 ceph-mon[194178]: pgmap v151: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:56 smithi150 ceph-mon[194178]: failed to encode map e86 with expected crc 2023-10-09T06:07:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4226703785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:57.061 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[212750]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[220921]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[220921]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:57 smithi150 ceph-mon[194178]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:57 smithi150 ceph-mon[194178]: failed to encode map e86 with expected crc 2023-10-09T06:07:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:07:57.687 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:07:57 smithi107 systemd[1]: Stopping Ceph osd.0 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:07:58.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:58.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.062 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:07:57 smithi107 conmon[125135]: 2023-10-09T06:07:57.798+0000 7eff1c0cb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:07:58.063 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:07:57 smithi107 conmon[125135]: 2023-10-09T06:07:57.798+0000 7eff1c0cb700 -1 osd.0 86 *** Got signal Terminated *** 2023-10-09T06:07:58.063 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:07:57 smithi107 conmon[125135]: 2023-10-09T06:07:57.798+0000 7eff1c0cb700 -1 osd.0 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:07:58.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:58.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T06:07:58.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4069043995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:58.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[212750]: osd.0 marked itself down and dead 2023-10-09T06:07:58.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4069043995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:58 smithi107 ceph-mon[220921]: osd.0 marked itself down and dead 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: pgmap v152: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: failed to encode map e87 with expected crc 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1890961559' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4069043995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:07:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:58 smithi150 ceph-mon[194178]: osd.0 marked itself down and dead 2023-10-09T06:07:58.689 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T06:07:58.691 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:07:58.691 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:07:58.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:07:58.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:07:58.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_237467 2023-10-09T06:07:58.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:07:58.742 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:07:59.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:59.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[212750]: failed to encode map e88 with expected crc 2023-10-09T06:07:59.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[212750]: failed to encode map e87 with expected crc 2023-10-09T06:07:59.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[212750]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T06:07:59.071 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[220921]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:59.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[220921]: failed to encode map e88 with expected crc 2023-10-09T06:07:59.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[220921]: failed to encode map e87 with expected crc 2023-10-09T06:07:59.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:07:59 smithi107 ceph-mon[220921]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T06:07:59.072 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:07:58 smithi107 bash[237405]: 8de9f1d9969944fb295a6b51528e992a1f9a2e784918fb9976dbfa82740be056 2023-10-09T06:07:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:59 smithi150 ceph-mon[194178]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:07:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:59 smithi150 ceph-mon[194178]: failed to encode map e88 with expected crc 2023-10-09T06:07:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:59 smithi150 ceph-mon[194178]: failed to encode map e87 with expected crc 2023-10-09T06:07:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:07:59 smithi150 ceph-mon[194178]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T06:07:59.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2819 2803 175.178 175.188 0.00384415 0.00569458 2023-10-09T06:08:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: pgmap v155: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[212750]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: pgmap v155: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:00.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:00 smithi107 ceph-mon[220921]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: pgmap v155: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:00 smithi150 ceph-mon[194178]: failed to encode map e89 with expected crc 2023-10-09T06:08:00.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 6670 6654 207.923 240.688 0.00272999 0.00480236 2023-10-09T06:08:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3396208961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3396208961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3396208961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:01.742 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 10611 10595 220.713 246.312 0.00259306 0.00452269 2023-10-09T06:08:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[212750]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:02.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:02] "GET /metrics HTTP/1.1" 200 36149 "" "Prometheus/2.43.0" 2023-10-09T06:08:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[220921]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:02 smithi150 ceph-mon[194178]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:08:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:02.743 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 14142 14126 220.606 220.688 0.00347351 0.0045286 2023-10-09T06:08:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1060653688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1060653688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:03.152 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:02 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T06:08:03.152 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:02 smithi107 systemd[1]: Stopped Ceph osd.0 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:08:03.152 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:03 smithi107 systemd[1]: Starting Ceph osd.0 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:08:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1060653688' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:03.744 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 17259 17243 215.433 194.812 0.00238893 0.00463696 2023-10-09T06:08:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:04 smithi107 ceph-mon[212750]: pgmap v158: 89 pgs: 4 active+undersized+degraded, 12 stale+active+clean, 4 active+undersized, 69 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 2493/12693 objects degraded (19.641%) 2023-10-09T06:08:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:04 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 2493/12693 objects degraded (19.641%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:04 smithi107 ceph-mon[220921]: pgmap v158: 89 pgs: 4 active+undersized+degraded, 12 stale+active+clean, 4 active+undersized, 69 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 2493/12693 objects degraded (19.641%) 2023-10-09T06:08:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:04 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 2493/12693 objects degraded (19.641%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:04 smithi150 ceph-mon[194178]: pgmap v158: 89 pgs: 4 active+undersized+degraded, 12 stale+active+clean, 4 active+undersized, 69 active+clean; 264 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 2493/12693 objects degraded (19.641%) 2023-10-09T06:08:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:04 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 2493/12693 objects degraded (19.641%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:04.744 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 20229 20213 210.464 185.625 0.00460615 0.0047455 2023-10-09T06:08:04.901 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:08:04.901 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:08:04.901 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T06:08:04.901 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:08:04.902 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T06:08:04.902 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T06:08:04.902 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T06:08:04.902 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:04 smithi107 bash[238109]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T06:08:05.745 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 23158 23142 206.526 183.062 0.00449578 0.00483732 2023-10-09T06:08:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:06 smithi107 ceph-mon[220921]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2004690169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:06 smithi107 ceph-mon[212750]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:06.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2004690169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:06 smithi150 ceph-mon[194178]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2004690169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:06.745 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 26035 26019 203.186 179.812 0.00554511 0.00491602 2023-10-09T06:08:07.626 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[212750]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:07.626 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:07.627 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:07 smithi107 bash[238516]: f47405cf3d52a3deb64f63cce517eb6b5aa39884b62173f5bd10a3103d607c1c 2023-10-09T06:08:07.627 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-osd[238582]: -- 172.21.15.107:0/1826844522 <== mon.1 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55662dac24e0 con 0x55662da95000 2023-10-09T06:08:07.628 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:07 smithi107 systemd[1]: Started Ceph osd.0 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:08:07.745 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 28408 28392 197.089 148.312 0.00880495 0.00506876 2023-10-09T06:08:07.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[220921]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:07.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:07.881 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:07 smithi107 conmon[238556]: 2023-10-09T06:08:07.875+0000 7f98fb221640 -1 Falling back to public interface 2023-10-09T06:08:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:07 smithi150 ceph-mon[194178]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 866 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s; 6741/41610 objects degraded (16.200%) 2023-10-09T06:08:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:08.747 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 30736 30720 191.896 145.5 0.00541963 0.00520607 2023-10-09T06:08:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1910082199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1910082199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:08.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1910082199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:09.747 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 33019 33003 187.424 142.688 0.00886976 0.00533098 2023-10-09T06:08:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:09 smithi107 ceph-mon[212750]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:09 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14018/84762 objects degraded (16.538%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:09 smithi107 ceph-mon[220921]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:09 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14018/84762 objects degraded (16.538%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:09 smithi150 ceph-mon[194178]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:09 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14018/84762 objects degraded (16.538%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:10.747 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 34762 34746 180.884 108.938 0.00643318 0.00552246 2023-10-09T06:08:11.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3136016170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:11.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3136016170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3136016170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:11.748 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 36524 36508 175.434 110.125 0.00832351 0.00569485 2023-10-09T06:08:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:11 smithi107 ceph-mon[212750]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:11 smithi107 ceph-mon[220921]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:11 smithi150 ceph-mon[194178]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.7 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s; 14018/84762 objects degraded (16.538%) 2023-10-09T06:08:12.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:12] "GET /metrics HTTP/1.1" 200 36149 "" "Prometheus/2.43.0" 2023-10-09T06:08:12.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 38267 38251 170.655 108.938 0.00903486 0.00585627 2023-10-09T06:08:13.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.159 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.159 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:13.654 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:13 smithi107 conmon[238556]: 2023-10-09T06:08:13.189+0000 7f98fb221640 -1 osd.0 87 log_to_monitors true 2023-10-09T06:08:13.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 40051 40035 166.713 111.5 0.00574746 0.00599407 2023-10-09T06:08:14.349 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 170 MiB/s wr, 2.71k op/s; 17114/97848 objects degraded (17.490%) 2023-10-09T06:08:14.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3678488584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:14.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.350 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:08:14 smithi107 conmon[238556]: 2023-10-09T06:08:14.092+0000 7f98eae8a700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:08:14.351 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 170 MiB/s wr, 2.71k op/s; 17114/97848 objects degraded (17.490%) 2023-10-09T06:08:14.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3678488584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:14.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:14 smithi107 ceph-mon[220921]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 170 MiB/s wr, 2.71k op/s; 17114/97848 objects degraded (17.490%) 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3678488584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:14 smithi150 ceph-mon[194178]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T06:08:14.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 42139 42123 164.45 130.5 0.00400638 0.00607677 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[212750]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:08:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:08:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:15 smithi107 ceph-mon[220921]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: from='osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:08:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:15 smithi150 ceph-mon[194178]: failed to encode map e90 with expected crc 2023-10-09T06:08:15.751 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 42872 42856 157.474 45.8125 0.00247318 0.00610189 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 18968/114480 objects degraded (16.569%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168] boot 2023-10-09T06:08:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/919017561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 18968/114480 objects degraded (16.569%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168] boot 2023-10-09T06:08:16.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T06:08:16.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:08:16.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/919017561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 18968/114480 objects degraded (16.569%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: osd.0 [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168] boot 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: failed to encode map e91 with expected crc 2023-10-09T06:08:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/919017561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:16.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 42872 42856 148.729 0 - 0.00610189 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: failed to encode map e92 with expected crc 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:17.751 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 43288 43272 142.272 13 0.00885852 0.00702128 2023-10-09T06:08:18.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:18.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/412340601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/412340601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 18968/114480 objects degraded (16.569%) 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: failed to encode map e93 with expected crc 2023-10-09T06:08:18.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/412340601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:18.752 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:08:18.751582+0000 min lat: 0.00125711 max lat: 2.23592 avg lat: 0.00722606 2023-10-09T06:08:18.752 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:08:18.752 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 44278 44262 138.253 61.875 0.00960416 0.00722606 2023-10-09T06:08:19.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 45358 45342 134.861 67.5 0.0245742 0.00740913 2023-10-09T06:08:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:20 smithi107 ceph-mon[212750]: pgmap v170: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 21343/129741 objects degraded (16.450%); 255 KiB/s, 3 objects/s recovering 2023-10-09T06:08:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:20 smithi107 ceph-mon[220921]: pgmap v170: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 21343/129741 objects degraded (16.450%); 255 KiB/s, 3 objects/s recovering 2023-10-09T06:08:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:20 smithi150 ceph-mon[194178]: pgmap v170: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 21343/129741 objects degraded (16.450%); 255 KiB/s, 3 objects/s recovering 2023-10-09T06:08:20.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 46331 46315 131.491 60.8125 0.00563853 0.00759647 2023-10-09T06:08:21.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3085126548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:21.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3085126548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3085126548' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:21.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 47372 47356 128.604 65.0625 0.011966 0.00776869 2023-10-09T06:08:22.356 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:22] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.43.0" 2023-10-09T06:08:22.356 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:22 smithi107 ceph-mon[212750]: pgmap v171: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 21340/129741 objects degraded (16.448%); 192 KiB/s, 2 objects/s recovering 2023-10-09T06:08:22.357 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 21340/129741 objects degraded (16.448%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:22.357 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:22 smithi107 ceph-mon[220921]: pgmap v171: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 21340/129741 objects degraded (16.448%); 192 KiB/s, 2 objects/s recovering 2023-10-09T06:08:22.357 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 21340/129741 objects degraded (16.448%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:22 smithi150 ceph-mon[194178]: pgmap v171: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 21340/129741 objects degraded (16.448%); 192 KiB/s, 2 objects/s recovering 2023-10-09T06:08:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 21340/129741 objects degraded (16.448%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:22.759 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 48379 48363 125.85 62.9375 0.0194125 0.0079389 2023-10-09T06:08:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/666907197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:23.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/666907197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:23.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/666907197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:23.760 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 49401 49385 123.373 63.875 0.023404 0.00810043 2023-10-09T06:08:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:24 smithi150 ceph-mon[194178]: pgmap v172: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 21340/133689 objects degraded (15.962%); 153 KiB/s, 2 objects/s recovering 2023-10-09T06:08:24.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:24 smithi107 ceph-mon[212750]: pgmap v172: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 21340/133689 objects degraded (15.962%); 153 KiB/s, 2 objects/s recovering 2023-10-09T06:08:24.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:24 smithi107 ceph-mon[220921]: pgmap v172: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 21340/133689 objects degraded (15.962%); 153 KiB/s, 2 objects/s recovering 2023-10-09T06:08:24.761 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 50423 50407 121.079 63.875 0.0156407 0.00825241 2023-10-09T06:08:25.652 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:25 smithi107 conmon[200038]: 2023-10-09T06:08:25.184+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:25.763 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 51375 51359 118.792 59.5 0.023872 0.00841193 2023-10-09T06:08:26.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: pgmap v173: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 21104/144987 objects degraded (14.556%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:26.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:26.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:26.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:26.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:26.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1240850974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:26.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: pgmap v173: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 21104/144987 objects degraded (14.556%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:26.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:26.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:26.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:26.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:26.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1240850974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: pgmap v173: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 21104/144987 objects degraded (14.556%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:08:26.410 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:08:26.411 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:26.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1240850974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:26.764 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 52436 52420 116.915 66.3125 0.00974283 0.00854806 2023-10-09T06:08:27.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 21104/144987 objects degraded (14.556%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 21104/144987 objects degraded (14.556%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 21104/144987 objects degraded (14.556%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:27.764 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 53483 53467 115.141 65.4375 0.0126478 0.00867902 2023-10-09T06:08:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:27 smithi107 ceph-mon[212750]: pgmap v174: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 21104/144987 objects degraded (14.556%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:08:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:27 smithi107 ceph-mon[220921]: pgmap v174: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 21104/144987 objects degraded (14.556%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:08:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:27 smithi150 ceph-mon[194178]: pgmap v174: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 21104/144987 objects degraded (14.556%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:08:28.765 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 54436 54420 113.29 59.5625 0.0117584 0.00882076 2023-10-09T06:08:29.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/940587644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:29.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/940587644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/940587644' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:29.768 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 55478 55462 111.737 65.125 0.00848136 0.0089437 2023-10-09T06:08:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:29 smithi150 ceph-mon[194178]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 20848/160269 objects degraded (13.008%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:30.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:29 smithi107 ceph-mon[220921]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 20848/160269 objects degraded (13.008%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:30.412 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:29 smithi107 ceph-mon[212750]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 20848/160269 objects degraded (13.008%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:08:30.766 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 56455 56439 110.146 61.0625 0.015803 0.00907052 2023-10-09T06:08:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2583815911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2583815911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2583815911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:31.773 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 57470 57454 108.725 63.4375 0.00663262 0.00919265 2023-10-09T06:08:32.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:32] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T06:08:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[220921]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 20848/160269 objects degraded (13.008%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 20848/160269 objects degraded (13.008%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[212750]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 20848/160269 objects degraded (13.008%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:31 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 20848/160269 objects degraded (13.008%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:31 smithi150 ceph-mon[194178]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 20848/160269 objects degraded (13.008%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:31 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 20848/160269 objects degraded (13.008%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:32.769 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 58466 58450 107.359 62.25 0.00933539 0.00930913 2023-10-09T06:08:33.774 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 59417 59401 105.973 59.4375 0.0218704 0.00942872 2023-10-09T06:08:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:33 smithi107 ceph-mon[220921]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 20848/165003 objects degraded (12.635%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3079396266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:33 smithi107 ceph-mon[212750]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 20848/165003 objects degraded (12.635%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3079396266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:33 smithi150 ceph-mon[194178]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 20848/165003 objects degraded (12.635%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:08:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3079396266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:34.783 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 60452 60436 104.807 64.6875 0.0228016 0.00953534 2023-10-09T06:08:35.781 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 61413 61397 103.599 60.0625 0.024348 0.00964759 2023-10-09T06:08:36.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:36 smithi107 ceph-mon[220921]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 20577/175206 objects degraded (11.744%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:08:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:36 smithi107 ceph-mon[212750]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 20577/175206 objects degraded (11.744%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:08:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:36 smithi150 ceph-mon[194178]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 20577/175206 objects degraded (11.744%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:08:36.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 62344 62328 102.404 58.1875 0.0178238 0.00975861 2023-10-09T06:08:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1710071780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 20577/175206 objects degraded (11.744%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1710071780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:37.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 20577/175206 objects degraded (11.744%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1710071780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:37.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 20577/175206 objects degraded (11.744%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:37.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 63318 63302 101.34 60.875 0.0140833 0.00986235 2023-10-09T06:08:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:38 smithi150 ceph-mon[194178]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 20577/175206 objects degraded (11.744%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:38.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:38 smithi107 ceph-mon[220921]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 20577/175206 objects degraded (11.744%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:38.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:38 smithi107 ceph-mon[212750]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s; 20577/175206 objects degraded (11.744%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:08:38.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:08:38.782652+0000 min lat: 0.00125711 max lat: 2.23592 avg lat: 0.00997271 2023-10-09T06:08:38.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:08:38.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 64221 64205 100.219 56.4375 0.0165351 0.00997271 2023-10-09T06:08:39.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3943601981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:39.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3943601981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3943601981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:39.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 65193 65177 99.2568 60.75 0.0324063 0.0100693 2023-10-09T06:08:40.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:40 smithi107 conmon[200038]: 2023-10-09T06:08:40.202+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:40 smithi107 ceph-mon[212750]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 20281/189801 objects degraded (10.685%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:08:40.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:40 smithi107 ceph-mon[220921]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 20281/189801 objects degraded (10.685%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:08:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:40 smithi150 ceph-mon[194178]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s; 20281/189801 objects degraded (10.685%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:08:40.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 66121 66105 98.2753 58 0.00880414 0.0101645 2023-10-09T06:08:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:41 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:41 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:41 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:41.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 67005 66989 97.2754 55.25 0.00885386 0.0102742 2023-10-09T06:08:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:42] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T06:08:42.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[212750]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 20281/189801 objects degraded (10.685%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:42.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2259561387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:42.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 20281/189801 objects degraded (10.685%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:42 smithi150 ceph-mon[194178]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 20281/189801 objects degraded (10.685%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2259561387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 20281/189801 objects degraded (10.685%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[220921]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 20281/189801 objects degraded (10.685%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:42.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2259561387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:42.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 20281/189801 objects degraded (10.685%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:42.784 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 67953 67937 96.4118 59.25 0.0135803 0.0103663 2023-10-09T06:08:43.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 68821 68805 95.4689 54.25 0.0291261 0.0104685 2023-10-09T06:08:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:44 smithi107 ceph-mon[212750]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 20281/194109 objects degraded (10.448%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3055781896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:44.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:44 smithi107 ceph-mon[220921]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 20281/194109 objects degraded (10.448%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:44.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3055781896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:44 smithi150 ceph-mon[194178]: pgmap v182: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 20281/194109 objects degraded (10.448%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:08:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3055781896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:44.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 69745 69729 94.6495 57.75 0.00731637 0.0105599 2023-10-09T06:08:45.791 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 70681 70665 93.8693 58.5 0.0201448 0.0106475 2023-10-09T06:08:46.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:46 smithi107 ceph-mon[220921]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 19968/203721 objects degraded (9.802%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:08:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:46 smithi107 ceph-mon[212750]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 19968/203721 objects degraded (9.802%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:08:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:46 smithi150 ceph-mon[194178]: pgmap v183: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 19968/203721 objects degraded (9.802%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:08:46.794 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 71528 71512 93.0132 52.9375 0.00954503 0.010746 2023-10-09T06:08:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/566299567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19968/203721 objects degraded (9.802%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/566299567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19968/203721 objects degraded (9.802%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/566299567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:08:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19968/203721 objects degraded (9.802%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:47.798 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 72455 72439 92.2902 57.9375 0.0212008 0.0108298 2023-10-09T06:08:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:48 smithi107 ceph-mon[220921]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 19968/203721 objects degraded (9.802%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:48 smithi107 ceph-mon[212750]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 19968/203721 objects degraded (9.802%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:48.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:48 smithi150 ceph-mon[194178]: pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 19968/203721 objects degraded (9.802%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:08:48.798 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 73327 73311 91.5351 54.5 0.0148361 0.0109184 2023-10-09T06:08:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1778238037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1778238037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1778238037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:49.799 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 74213 74197 90.8242 55.375 0.00722809 0.0110046 2023-10-09T06:08:50.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:50 smithi107 ceph-mon[220921]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 19653/217143 objects degraded (9.051%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:08:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:50 smithi150 ceph-mon[194178]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 19653/217143 objects degraded (9.051%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:08:50.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:50 smithi107 ceph-mon[212750]: pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 19653/217143 objects degraded (9.051%); 4.8 MiB/s, 76 objects/s recovering 2023-10-09T06:08:50.799 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 75061 75045 90.0975 53 0.0101838 0.0110945 2023-10-09T06:08:51.800 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 75890 75874 89.3757 51.8125 0.0355965 0.0111819 2023-10-09T06:08:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[212750]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 19653/217143 objects degraded (9.051%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1087210954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:52.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19653/217143 objects degraded (9.051%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:52.408 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:08:52] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T06:08:52.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[220921]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 19653/217143 objects degraded (9.051%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:52.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1087210954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:52.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19653/217143 objects degraded (9.051%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:52 smithi150 ceph-mon[194178]: pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 19653/217143 objects degraded (9.051%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1087210954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19653/217143 objects degraded (9.051%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:52.800 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 76783 76767 88.7547 55.8125 0.00710893 0.0112609 2023-10-09T06:08:53.800 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 77678 77662 88.1585 55.9375 0.015863 0.0113375 2023-10-09T06:08:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:54 smithi107 ceph-mon[212750]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 19653/221184 objects degraded (8.885%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1480598283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:54 smithi107 ceph-mon[220921]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 19653/221184 objects degraded (8.885%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1480598283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:54 smithi150 ceph-mon[194178]: pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 19653/221184 objects degraded (8.885%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:08:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1480598283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:54.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 78530 78514 87.5323 53.25 0.0177763 0.011419 2023-10-09T06:08:55.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:08:55 smithi107 conmon[200038]: 2023-10-09T06:08:55.221+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:55.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 79434 79418 86.9883 56.5 0.0234446 0.0114915 2023-10-09T06:08:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[212750]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 19283/230130 objects degraded (8.379%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:08:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[220921]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 19283/230130 objects degraded (8.379%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:08:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:56 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:56 smithi150 ceph-mon[194178]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 19283/230130 objects degraded (8.379%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:08:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:56 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:56 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:08:56.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:56 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:08:56.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 80253 80237 86.3716 51.1875 0.00793552 0.011572 2023-10-09T06:08:57.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19283/230130 objects degraded (8.379%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:57.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1342828865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19283/230130 objects degraded (8.379%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1342828865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19283/230130 objects degraded (8.379%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:08:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1342828865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:08:57.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 81134 81118 85.8413 55.0625 0.00986209 0.0116442 2023-10-09T06:08:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:08:58 smithi107 ceph-mon[212750]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 19283/230130 objects degraded (8.379%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:08:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:08:58 smithi107 ceph-mon[220921]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 19283/230130 objects degraded (8.379%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:08:58.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:08:58 smithi150 ceph-mon[194178]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 733 op/s; 19283/230130 objects degraded (8.379%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:08:58.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:08:58.803960+0000 min lat: 0.00125711 max lat: 2.23592 avg lat: 0.0117134 2023-10-09T06:08:58.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:08:58.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 82020 82004 85.3329 55.375 0.0229294 0.0117134 2023-10-09T06:08:59.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 82820 82804 84.7541 50 0.0225687 0.0117932 2023-10-09T06:09:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:00 smithi107 ceph-mon[212750]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18905/243222 objects degraded (7.773%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:09:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1320075372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:00.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:00 smithi107 ceph-mon[220921]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18905/243222 objects degraded (7.773%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:09:00.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1320075372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:00 smithi150 ceph-mon[194178]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 18905/243222 objects degraded (7.773%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:09:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1320075372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:00.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 83387 83371 83.9593 35.4375 0.00358093 0.0118215 2023-10-09T06:09:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[220921]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:09:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[220921]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[220921]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[220921]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[220921]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[212750]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:01 smithi107 ceph-mon[212750]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:01 smithi150 ceph-mon[194178]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T06:09:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:01 smithi150 ceph-mon[194178]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:01 smithi150 ceph-mon[194178]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:01 smithi150 ceph-mon[194178]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:01 smithi150 ceph-mon[194178]: failed to encode map e94 with expected crc 2023-10-09T06:09:01.804 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 83812 83796 83.049 26.5625 0.0270333 0.012037 2023-10-09T06:09:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 18905/243222 objects degraded (7.773%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 18905/243222 objects degraded (7.773%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1794041999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 18905/243222 objects degraded (7.773%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 18905/243222 objects degraded (7.773%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1794041999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:02.405 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:02] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-10-09T06:09:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 18905/243222 objects degraded (7.773%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T06:09:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T06:09:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: failed to encode map e95 with expected crc 2023-10-09T06:09:02.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:02.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 18905/243222 objects degraded (7.773%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:02.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1794041999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:02.809 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 84648 84632 82.5614 52.25 0.0319636 0.0121072 2023-10-09T06:09:03.809 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 85487 85471 82.0984 52.4375 0.0178265 0.0121754 2023-10-09T06:09:04.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:04 smithi107 ceph-mon[212750]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 18905/246504 objects degraded (7.669%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:04.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:04 smithi107 ceph-mon[220921]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 18905/246504 objects degraded (7.669%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:04 smithi150 ceph-mon[194178]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 18905/246504 objects degraded (7.669%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:04.809 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 86250 86234 81.5768 47.6875 0.0164703 0.0122529 2023-10-09T06:09:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/382767786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:05.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/382767786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:05.659 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/382767786' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:05.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 87105 87089 81.1539 53.4375 0.0100402 0.0123167 2023-10-09T06:09:06.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:06 smithi107 ceph-mon[212750]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 18525/253941 objects degraded (7.295%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:09:06.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:06 smithi107 ceph-mon[220921]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 18525/253941 objects degraded (7.295%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:09:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:06 smithi150 ceph-mon[194178]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 18525/253941 objects degraded (7.295%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:09:06.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 87902 87886 80.6933 49.8125 0.0152295 0.0123867 2023-10-09T06:09:07.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 18525/253941 objects degraded (7.295%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:07.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/211405888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 18525/253941 objects degraded (7.295%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/211405888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:07.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 18525/253941 objects degraded (7.295%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:07.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/211405888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:07.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 88695 88679 80.2401 49.5625 0.013966 0.0124563 2023-10-09T06:09:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:08 smithi107 ceph-mon[212750]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 28 MiB/s wr, 446 op/s; 18525/253941 objects degraded (7.295%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:08.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:08 smithi107 ceph-mon[220921]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 28 MiB/s wr, 446 op/s; 18525/253941 objects degraded (7.295%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:08.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:08 smithi150 ceph-mon[194178]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 28 MiB/s wr, 446 op/s; 18525/253941 objects degraded (7.295%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:09:08.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 89525 89509 79.8352 51.875 0.018266 0.012521 2023-10-09T06:09:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:09 smithi150 ceph-mon[194178]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 18311/265374 objects degraded (6.900%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:09:09.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:09 smithi107 ceph-mon[220921]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 18311/265374 objects degraded (6.900%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:09:09.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:09 smithi107 ceph-mon[212750]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 18311/265374 objects degraded (6.900%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:09:09.817 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 90292 90276 79.3837 47.9375 0.0158354 0.0125913 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:10 smithi107 conmon[200038]: 2023-10-09T06:09:10.242+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1733024033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1733024033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:10.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:10 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:10.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1733024033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:10.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:10 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:10.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:10 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:10.817 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 91098 91082 78.9811 50.375 0.0256103 0.0126558 2023-10-09T06:09:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:11 smithi107 ceph-mon[220921]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 18311/266208 objects degraded (6.878%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:09:11.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:11 smithi107 ceph-mon[212750]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 18311/266208 objects degraded (6.878%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:09:11.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:11 smithi150 ceph-mon[194178]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 18311/266208 objects degraded (6.878%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:09:11.820 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 91910 91894 78.5921 50.75 0.0256091 0.0127189 2023-10-09T06:09:12.398 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:12] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-10-09T06:09:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 18311/266208 objects degraded (6.878%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3041592632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:12.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 18311/266208 objects degraded (6.878%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:12.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3041592632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 18311/266208 objects degraded (6.878%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3041592632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:12.819 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 92676 92660 78.1774 47.875 0.013227 0.0127861 2023-10-09T06:09:13.820 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 93471 93455 77.7978 49.6875 0.0198895 0.0128478 2023-10-09T06:09:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:13 smithi107 ceph-mon[220921]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 18311/269985 objects degraded (6.782%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:13 smithi107 ceph-mon[212750]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 18311/269985 objects degraded (6.782%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:13 smithi150 ceph-mon[194178]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 18311/269985 objects degraded (6.782%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:09:14.820 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 94212 94196 77.3839 46.3125 0.0233039 0.0129168 2023-10-09T06:09:15.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 95006 94990 77.0211 49.625 0.0317352 0.0129781 2023-10-09T06:09:16.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:15 smithi107 ceph-mon[220921]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 18097/278049 objects degraded (6.509%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:16.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/398283121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:15 smithi150 ceph-mon[194178]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 18097/278049 objects degraded (6.509%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/398283121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:16.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:15 smithi107 ceph-mon[212750]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 18097/278049 objects degraded (6.509%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:16.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/398283121' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:16.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 95724 95708 76.6093 44.875 0.023481 0.0130479 2023-10-09T06:09:17.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:17.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:16 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 18097/278049 objects degraded (6.509%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:17.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:17.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:16 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 18097/278049 objects degraded (6.509%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:16 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 18097/278049 objects degraded (6.509%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:09:17.824 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 96348 96332 76.1329 39 0.0180008 0.0131289 2023-10-09T06:09:18.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:17 smithi107 ceph-mon[212750]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18097/278049 objects degraded (6.509%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:09:18.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1273909990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:18.062 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:17 smithi107 ceph-mon[220921]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18097/278049 objects degraded (6.509%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:09:18.062 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1273909990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:17 smithi150 ceph-mon[194178]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 18097/278049 objects degraded (6.509%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:09:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1273909990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:18.831 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:09:18.827433+0000 min lat: 0.00125711 max lat: 2.23592 avg lat: 0.0132146 2023-10-09T06:09:18.832 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:09:18.832 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 96943 96927 75.6436 37.1875 0.016242 0.0132146 2023-10-09T06:09:19.829 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 97501 97485 75.1407 34.875 0.0539068 0.0133013 2023-10-09T06:09:19.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:19 smithi107 ceph-mon[212750]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 17870/288450 objects degraded (6.195%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:09:19.979 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:19 smithi107 ceph-mon[220921]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 17870/288450 objects degraded (6.195%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:09:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:19 smithi150 ceph-mon[194178]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 17870/288450 objects degraded (6.195%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:09:20.827 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 98053 98037 74.6455 34.5 0.0321612 0.0133907 2023-10-09T06:09:20.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1132021600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1132021600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1132021600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:21.827 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 98695 98679 74.2299 40.125 0.0218442 0.0134659 2023-10-09T06:09:22.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:21 smithi107 ceph-mon[212750]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 17870/289089 objects degraded (6.181%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:22.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:21 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 17870/289089 objects degraded (6.181%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:22.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:21 smithi107 ceph-mon[220921]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 17870/289089 objects degraded (6.181%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:22.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:21 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 17870/289089 objects degraded (6.181%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:21 smithi150 ceph-mon[194178]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 658 op/s; 17870/289089 objects degraded (6.181%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:21 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 17870/289089 objects degraded (6.181%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:22.407 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:22] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T06:09:22.832 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 99310 99294 73.7997 38.4375 0.00814527 0.0135436 2023-10-09T06:09:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1914480155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:23.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1914480155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:23.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1914480155' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:23.832 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 99978 99962 73.4229 41.75 0.0335361 0.0136138 2023-10-09T06:09:24.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:23 smithi107 ceph-mon[212750]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 17870/291828 objects degraded (6.123%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:23 smithi107 ceph-mon[220921]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 17870/291828 objects degraded (6.123%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:23 smithi150 ceph-mon[194178]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 17870/291828 objects degraded (6.123%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:09:24.832 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 100618 100602 73.0346 40 0.0103803 0.0136855 2023-10-09T06:09:25.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 101226 101210 72.6289 38 0.0241737 0.0137624 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[212750]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 17696/297888 objects degraded (5.940%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1911209379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[220921]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 17696/297888 objects degraded (5.940%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1911209379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:25.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:25 smithi150 ceph-mon[194178]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 17696/297888 objects degraded (5.940%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:09:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1911209379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:26.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 101813 101797 72.2209 36.6875 0.0423927 0.0138411 2023-10-09T06:09:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 17696/297888 objects degraded (5.940%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 17696/297888 objects degraded (5.940%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.163 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 17696/297888 objects degraded (5.940%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:27.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 102355 102339 71.7901 33.875 0.0215222 0.0139247 2023-10-09T06:09:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:27 smithi107 ceph-mon[212750]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 17696/297888 objects degraded (5.940%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:09:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2276328528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:28.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:27 smithi107 ceph-mon[220921]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 17696/297888 objects degraded (5.940%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:09:28.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2276328528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:27 smithi150 ceph-mon[194178]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 550 op/s; 17696/297888 objects degraded (5.940%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:09:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2276328528' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:28.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0211 2023-10-09T06:09:28.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 102886 2023-10-09T06:09:28.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 71.4318 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 48.897 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 246.312 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1142 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 782.352 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3941 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0139949 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0314018 2023-10-09T06:09:28.838 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.23592 2023-10-09T06:09:28.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00125711 2023-10-09T06:09:28.844 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:09:30.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[212750]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 17497/306558 objects degraded (5.708%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:30.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:30.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[220921]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 17497/306558 objects degraded (5.708%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:30 smithi150 ceph-mon[194178]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 17497/306558 objects degraded (5.708%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.083 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/567548729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:31.083 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.083 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/567548729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/567548729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 17497/306972 objects degraded (5.700%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 17497/306972 objects degraded (5.700%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:31 smithi107 conmon[200038]: 2023-10-09T06:09:31.864+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:32] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-10-09T06:09:32.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 17497/306972 objects degraded (5.700%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 17497/306972 objects degraded (5.700%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 17497/306972 objects degraded (5.700%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:09:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 17497/306972 objects degraded (5.700%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:09:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:09:32.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:33.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:33.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:33.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/987021764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:33.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:33.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:33.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/987021764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:33.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:33 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:33 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/987021764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:34 smithi150 ceph-mon[194178]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 574 op/s; 17497/305556 objects degraded (5.726%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:34.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:34 smithi107 ceph-mon[212750]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 574 op/s; 17497/305556 objects degraded (5.726%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:34.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:34 smithi107 ceph-mon[220921]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 574 op/s; 17497/305556 objects degraded (5.726%); 1.9 MiB/s, 31 objects/s recovering 2023-10-09T06:09:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:36 smithi107 ceph-mon[212750]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 672 op/s; 17281/301410 objects degraded (5.733%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3878217818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:36 smithi107 ceph-mon[220921]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 672 op/s; 17281/301410 objects degraded (5.733%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3878217818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:36 smithi150 ceph-mon[194178]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 672 op/s; 17281/301410 objects degraded (5.733%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:09:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3878217818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 17281/301410 objects degraded (5.733%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:37.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 17281/301410 objects degraded (5.733%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:37.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 17281/301410 objects degraded (5.733%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:38 smithi150 ceph-mon[194178]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 504 op/s; 17281/301410 objects degraded (5.733%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:09:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4057271208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:38.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:38 smithi107 ceph-mon[212750]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 504 op/s; 17281/301410 objects degraded (5.733%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:09:38.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4057271208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:38.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:38 smithi107 ceph-mon[220921]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 504 op/s; 17281/301410 objects degraded (5.733%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:09:38.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4057271208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:40.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:40 smithi107 ceph-mon[212750]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s; 17049/292089 objects degraded (5.837%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:09:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:40 smithi107 ceph-mon[220921]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s; 17049/292089 objects degraded (5.837%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:09:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:40 smithi150 ceph-mon[194178]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 762 op/s; 17049/292089 objects degraded (5.837%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:09:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3427671284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:41.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3427671284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:41.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3427671284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:42.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:42 smithi107 ceph-mon[212750]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.8 MiB/s wr, 537 op/s; 17049/291546 objects degraded (5.848%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:42.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 17049/291546 objects degraded (5.848%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:42.383 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:42] "GET /metrics HTTP/1.1" 200 36231 "" "Prometheus/2.43.0" 2023-10-09T06:09:42.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:42 smithi107 ceph-mon[220921]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.8 MiB/s wr, 537 op/s; 17049/291546 objects degraded (5.848%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:42.383 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 17049/291546 objects degraded (5.848%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:42 smithi150 ceph-mon[194178]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.8 MiB/s wr, 537 op/s; 17049/291546 objects degraded (5.848%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 17049/291546 objects degraded (5.848%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:43.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1868659148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1868659148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:43.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1868659148' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:44 smithi150 ceph-mon[194178]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 609 op/s; 17049/288468 objects degraded (5.910%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:44.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:44 smithi107 ceph-mon[220921]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 609 op/s; 17049/288468 objects degraded (5.910%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:44.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:44 smithi107 ceph-mon[212750]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 3.0 MiB/s wr, 609 op/s; 17049/288468 objects degraded (5.910%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:09:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:46 smithi107 ceph-mon[212750]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.8 MiB/s wr, 724 op/s; 16816/281538 objects degraded (5.973%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:09:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/75063555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:46 smithi107 ceph-mon[220921]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.8 MiB/s wr, 724 op/s; 16816/281538 objects degraded (5.973%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:09:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/75063555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:46 smithi150 ceph-mon[194178]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 1.8 MiB/s wr, 724 op/s; 16816/281538 objects degraded (5.973%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:09:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/75063555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:47.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:46 smithi107 conmon[200038]: 2023-10-09T06:09:46.883+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16816/281538 objects degraded (5.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16816/281538 objects degraded (5.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:09:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16816/281538 objects degraded (5.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[212750]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16816/281538 objects degraded (5.973%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:09:48.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:48.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[220921]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16816/281538 objects degraded (5.973%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:09:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:48.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:48 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:48 smithi150 ceph-mon[194178]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16816/281538 objects degraded (5.973%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:09:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:48 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:09:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:48 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:09:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1163813830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:49.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1163813830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:49.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1163813830' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:50 smithi150 ceph-mon[194178]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16544/271137 objects degraded (6.102%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:09:50.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:50 smithi107 ceph-mon[212750]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16544/271137 objects degraded (6.102%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:09:50.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:50 smithi107 ceph-mon[220921]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 16544/271137 objects degraded (6.102%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:09:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4168808166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4168808166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:51.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4168808166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:52.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:52 smithi107 ceph-mon[212750]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 16544/270480 objects degraded (6.117%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:52.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16544/270480 objects degraded (6.117%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:52.341 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:09:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:09:52] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-10-09T06:09:52.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:52 smithi107 ceph-mon[220921]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 16544/270480 objects degraded (6.117%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:52.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16544/270480 objects degraded (6.117%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:52 smithi150 ceph-mon[194178]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 16544/270480 objects degraded (6.117%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16544/270480 objects degraded (6.117%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:53.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:53 smithi107 ceph-mon[212750]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 16544/267162 objects degraded (6.192%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:53.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:53 smithi107 ceph-mon[220921]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 16544/267162 objects degraded (6.192%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:53 smithi150 ceph-mon[194178]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 16544/267162 objects degraded (6.192%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:09:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/354904316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:54.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/354904316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:54.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/354904316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:55.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:55 smithi107 ceph-mon[220921]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16240/259695 objects degraded (6.253%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:55.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:55 smithi107 ceph-mon[212750]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16240/259695 objects degraded (6.253%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:55 smithi150 ceph-mon[194178]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 16240/259695 objects degraded (6.253%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:09:56.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1123649549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:56.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1123649549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1123649549' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T06:09:57.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 16240/259695 objects degraded (6.253%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:09:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16240/259695 objects degraded (6.253%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[220921]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T06:09:57.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 16240/259695 objects degraded (6.253%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16240/259695 objects degraded (6.253%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:57.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:57 smithi107 ceph-mon[212750]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: failed to encode map e96 with expected crc 2023-10-09T06:09:57.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 16240/259695 objects degraded (6.253%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:09:57.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16240/259695 objects degraded (6.253%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T06:09:57.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:57 smithi150 ceph-mon[194178]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T06:09:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[220921]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:58 smithi150 ceph-mon[194178]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:58.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:58 smithi107 ceph-mon[212750]: failed to encode map e97 with expected crc 2023-10-09T06:09:59.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:59 smithi107 ceph-mon[220921]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15905/251259 objects degraded (6.330%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:09:59.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:09:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/731545894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:59 smithi107 ceph-mon[212750]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15905/251259 objects degraded (6.330%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:09:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:09:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/731545894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:09:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:59 smithi150 ceph-mon[194178]: pgmap v224: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15905/251259 objects degraded (6.330%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:09:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:09:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/731545894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: Health detail: HEALTH_WARN Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: [WRN] PG_DEGRADED: Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.4 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.5 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.6 is stuck undersized for 101s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.7 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.d is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-09T06:10:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[212750]: pg 7.f is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: Health detail: HEALTH_WARN Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: [WRN] PG_DEGRADED: Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.4 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.5 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.6 is stuck undersized for 101s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.7 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.d is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:00 smithi107 ceph-mon[220921]: pg 7.f is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: Health detail: HEALTH_WARN Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: [WRN] PG_DEGRADED: Degraded data redundancy: 15905/251259 objects degraded (6.330%), 6 pgs degraded, 6 pgs undersized 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.4 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-09T06:10:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.5 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-09T06:10:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.6 is stuck undersized for 101s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.7 is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-09T06:10:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.d is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-09T06:10:00.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:00 smithi150 ceph-mon[194178]: pg 7.f is stuck undersized for 101s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-09T06:10:01.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:01 smithi107 ceph-mon[212750]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15905/250698 objects degraded (6.344%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:10:01.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3358772341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:01.902 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:01 smithi107 conmon[200038]: 2023-10-09T06:10:01.898+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:01.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:01 smithi107 ceph-mon[220921]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15905/250698 objects degraded (6.344%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:10:01.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3358772341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:01.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:01 smithi150 ceph-mon[194178]: pgmap v225: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 15905/250698 objects degraded (6.344%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:10:01.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3358772341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:02.157 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:02] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T06:10:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 15905/250698 objects degraded (6.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 15905/250698 objects degraded (6.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:02 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:02 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:02.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:02.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 15905/250698 objects degraded (6.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:02.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:02.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:02 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:03 smithi107 ceph-mon[212750]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15905/247290 objects degraded (6.432%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:10:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:03 smithi107 ceph-mon[220921]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15905/247290 objects degraded (6.432%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:10:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:03 smithi150 ceph-mon[194178]: pgmap v226: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 15905/247290 objects degraded (6.432%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:10:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4150128536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4150128536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4150128536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:05 smithi107 ceph-mon[220921]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 15741/240288 objects degraded (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:05 smithi107 ceph-mon[212750]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 15741/240288 objects degraded (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:05.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:05 smithi150 ceph-mon[194178]: pgmap v227: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 15741/240288 objects degraded (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4059254166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:06.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4059254166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:06.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4059254166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:07.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:07 smithi107 ceph-mon[220921]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 15741/240288 objects degraded (6.551%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:10:07.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 15741/240288 objects degraded (6.551%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:07.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:07 smithi107 ceph-mon[212750]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 15741/240288 objects degraded (6.551%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:10:07.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 15741/240288 objects degraded (6.551%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:07.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:07 smithi150 ceph-mon[194178]: pgmap v228: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 15741/240288 objects degraded (6.551%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:10:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 15741/240288 objects degraded (6.551%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:10.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:09 smithi107 ceph-mon[212750]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 15563/230511 objects degraded (6.752%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:10.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1346818195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:09 smithi150 ceph-mon[194178]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 15563/230511 objects degraded (6.752%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1346818195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:10.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:09 smithi107 ceph-mon[220921]: pgmap v229: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 15563/230511 objects degraded (6.752%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T06:10:10.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1346818195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:12.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[220921]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15563/229872 objects degraded (6.770%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:12.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/625674337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:12.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 15563/229872 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:12.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[212750]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15563/229872 objects degraded (6.770%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:12.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/625674337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:12.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:11 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 15563/229872 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:12.019 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:12] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T06:10:12.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:11 smithi150 ceph-mon[194178]: pgmap v230: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15563/229872 objects degraded (6.770%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:12.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/625674337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:12.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:11 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 15563/229872 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:14.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:13 smithi107 ceph-mon[220921]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 15563/226497 objects degraded (6.871%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:13 smithi107 ceph-mon[212750]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 15563/226497 objects degraded (6.871%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:13 smithi150 ceph-mon[194178]: pgmap v231: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 15563/226497 objects degraded (6.871%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:10:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3635307677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:15.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3635307677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:15.473 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3635307677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:15 smithi107 ceph-mon[212750]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15377/219384 objects degraded (7.009%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:10:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:15 smithi150 ceph-mon[194178]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15377/219384 objects degraded (7.009%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:10:16.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:15 smithi107 ceph-mon[220921]: pgmap v232: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15377/219384 objects degraded (7.009%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:10:17.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:16 smithi107 conmon[200038]: 2023-10-09T06:10:16.920+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3205250982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 15377/219384 objects degraded (7.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3205250982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:16 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 15377/219384 objects degraded (7.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:17.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:17.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3205250982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:17.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:16 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 15377/219384 objects degraded (7.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:17 smithi150 ceph-mon[194178]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15377/219384 objects degraded (7.009%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:17 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:17 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:18.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[212750]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15377/219384 objects degraded (7.009%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:18.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:18.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[220921]: pgmap v233: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 15377/219384 objects degraded (7.009%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:18.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:18.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:17 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:19 smithi107 ceph-mon[212750]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 15196/209691 objects degraded (7.247%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:10:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2773039872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:19 smithi150 ceph-mon[194178]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 15196/209691 objects degraded (7.247%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:10:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2773039872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:20.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:19 smithi107 ceph-mon[220921]: pgmap v234: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 15196/209691 objects degraded (7.247%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:10:20.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2773039872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:22] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-10-09T06:10:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[220921]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15196/209076 objects degraded (7.268%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 15196/209076 objects degraded (7.268%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4150190355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:21 smithi150 ceph-mon[194178]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15196/209076 objects degraded (7.268%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:21 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 15196/209076 objects degraded (7.268%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4150190355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[212750]: pgmap v235: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 15196/209076 objects degraded (7.268%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 15196/209076 objects degraded (7.268%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4150190355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:24.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:23 smithi107 ceph-mon[212750]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15196/205926 objects degraded (7.379%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:23 smithi150 ceph-mon[194178]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15196/205926 objects degraded (7.379%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:24.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:23 smithi107 ceph-mon[220921]: pgmap v236: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 15196/205926 objects degraded (7.379%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:10:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3581238211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3581238211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3581238211' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:25 smithi150 ceph-mon[194178]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14998/198882 objects degraded (7.541%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:10:26.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:25 smithi107 ceph-mon[212750]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14998/198882 objects degraded (7.541%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:10:26.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:25 smithi107 ceph-mon[220921]: pgmap v237: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 14998/198882 objects degraded (7.541%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:10:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14998/198882 objects degraded (7.541%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14998/198882 objects degraded (7.541%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14998/198882 objects degraded (7.541%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:27 smithi107 ceph-mon[212750]: pgmap v238: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14998/198882 objects degraded (7.541%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:10:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2576619467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:27 smithi107 ceph-mon[220921]: pgmap v238: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14998/198882 objects degraded (7.541%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:10:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2576619467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:27 smithi150 ceph-mon[194178]: pgmap v238: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 14998/198882 objects degraded (7.541%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:10:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2576619467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:29 smithi107 ceph-mon[212750]: pgmap v239: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 14796/189282 objects degraded (7.817%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:10:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2694760778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:29 smithi107 ceph-mon[220921]: pgmap v239: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 14796/189282 objects degraded (7.817%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:10:30.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2694760778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:29 smithi150 ceph-mon[194178]: pgmap v239: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 14796/189282 objects degraded (7.817%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:10:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2694760778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:32.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:32] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-10-09T06:10:32.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[212750]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 14796/188562 objects degraded (7.847%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:32.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14796/188562 objects degraded (7.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:31 smithi150 ceph-mon[194178]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 14796/188562 objects degraded (7.847%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:31 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14796/188562 objects degraded (7.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:32.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[220921]: pgmap v240: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 14796/188562 objects degraded (7.847%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:32.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:32.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14796/188562 objects degraded (7.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:32.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3439362597' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3439362597' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:33.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3439362597' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:34.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:33 smithi107 ceph-mon[220921]: pgmap v241: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 14796/185505 objects degraded (7.976%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:34.264 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:33 smithi107 ceph-mon[212750]: pgmap v241: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 14796/185505 objects degraded (7.976%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:33 smithi150 ceph-mon[194178]: pgmap v241: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 14796/185505 objects degraded (7.976%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:10:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/702045769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/702045769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:35.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/702045769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:36 smithi107 ceph-mon[212750]: pgmap v242: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14593/178857 objects degraded (8.159%); 2.9 MiB/s, 50 objects/s recovering 2023-10-09T06:10:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:36 smithi107 ceph-mon[220921]: pgmap v242: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14593/178857 objects degraded (8.159%); 2.9 MiB/s, 50 objects/s recovering 2023-10-09T06:10:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:36 smithi150 ceph-mon[194178]: pgmap v242: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 14593/178857 objects degraded (8.159%); 2.9 MiB/s, 50 objects/s recovering 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14593/178857 objects degraded (8.159%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14593/178857 objects degraded (8.159%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14593/178857 objects degraded (8.159%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:38.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:38 smithi107 ceph-mon[212750]: pgmap v243: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 14593/178857 objects degraded (8.159%); 1.9 MiB/s, 33 objects/s recovering 2023-10-09T06:10:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2091383943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:38 smithi150 ceph-mon[194178]: pgmap v243: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 14593/178857 objects degraded (8.159%); 1.9 MiB/s, 33 objects/s recovering 2023-10-09T06:10:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2091383943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:38.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:38 smithi107 ceph-mon[220921]: pgmap v243: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 14593/178857 objects degraded (8.159%); 1.9 MiB/s, 33 objects/s recovering 2023-10-09T06:10:38.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2091383943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:39.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.655 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:39 smithi107 conmon[200038]: 2023-10-09T06:10:39.367+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:39.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:39.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:10:39.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:10:39.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: pgmap v244: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14378/170157 objects degraded (8.450%); 1.9 MiB/s, 51 objects/s recovering 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1983525538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: pgmap v244: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14378/170157 objects degraded (8.450%); 1.9 MiB/s, 51 objects/s recovering 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:40.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1983525538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: pgmap v244: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 14378/170157 objects degraded (8.450%); 1.9 MiB/s, 51 objects/s recovering 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1983525538' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:42.275 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:42] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-10-09T06:10:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:42 smithi107 ceph-mon[220921]: pgmap v245: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 14378/169590 objects degraded (8.478%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14378/169590 objects degraded (8.478%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:42.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:42 smithi107 ceph-mon[212750]: pgmap v245: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 14378/169590 objects degraded (8.478%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:42.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14378/169590 objects degraded (8.478%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:42.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:42 smithi150 ceph-mon[194178]: pgmap v245: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 14378/169590 objects degraded (8.478%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14378/169590 objects degraded (8.478%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:43.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2084887926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2084887926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:43.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2084887926' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:44 smithi150 ceph-mon[194178]: pgmap v246: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14378/166524 objects degraded (8.634%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:44.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:44 smithi107 ceph-mon[212750]: pgmap v246: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14378/166524 objects degraded (8.634%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:44.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:44 smithi107 ceph-mon[220921]: pgmap v246: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 14378/166524 objects degraded (8.634%); 831 KiB/s, 34 objects/s recovering 2023-10-09T06:10:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4187641025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4187641025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:45.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4187641025' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:46 smithi107 ceph-mon[220921]: pgmap v247: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 14157/159675 objects degraded (8.866%); 831 KiB/s, 53 objects/s recovering 2023-10-09T06:10:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:46 smithi107 ceph-mon[212750]: pgmap v247: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 14157/159675 objects degraded (8.866%); 831 KiB/s, 53 objects/s recovering 2023-10-09T06:10:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:46 smithi150 ceph-mon[194178]: pgmap v247: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 14157/159675 objects degraded (8.866%); 831 KiB/s, 53 objects/s recovering 2023-10-09T06:10:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:47.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 14157/159675 objects degraded (8.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 14157/159675 objects degraded (8.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:47.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:10:47.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 14157/159675 objects degraded (8.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:48.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:48 smithi107 ceph-mon[212750]: pgmap v248: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 14157/159675 objects degraded (8.866%); 0 B/s, 36 objects/s recovering 2023-10-09T06:10:48.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2398317362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:48.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:48 smithi107 ceph-mon[220921]: pgmap v248: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 14157/159675 objects degraded (8.866%); 0 B/s, 36 objects/s recovering 2023-10-09T06:10:48.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2398317362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:48.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:48 smithi150 ceph-mon[194178]: pgmap v248: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 14157/159675 objects degraded (8.866%); 0 B/s, 36 objects/s recovering 2023-10-09T06:10:48.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2398317362' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:49 smithi107 ceph-mon[212750]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 13925/150231 objects degraded (9.269%); 0 B/s, 55 objects/s recovering 2023-10-09T06:10:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:49 smithi150 ceph-mon[194178]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 13925/150231 objects degraded (9.269%); 0 B/s, 55 objects/s recovering 2023-10-09T06:10:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:49 smithi107 ceph-mon[220921]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 13925/150231 objects degraded (9.269%); 0 B/s, 55 objects/s recovering 2023-10-09T06:10:50.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/626541048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/626541048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/626541048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:51 smithi150 ceph-mon[194178]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 13925/149622 objects degraded (9.307%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:51.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:51 smithi107 ceph-mon[212750]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 13925/149622 objects degraded (9.307%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:51.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:51 smithi107 ceph-mon[220921]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 13925/149622 objects degraded (9.307%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:52.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:10:52] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-10-09T06:10:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13925/149622 objects degraded (9.307%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13925/149622 objects degraded (9.307%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13925/149622 objects degraded (9.307%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:53.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:53 smithi107 ceph-mon[212750]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 13925/146544 objects degraded (9.502%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:53.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2252837202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:53 smithi107 ceph-mon[220921]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 13925/146544 objects degraded (9.502%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2252837202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:53 smithi150 ceph-mon[194178]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 13925/146544 objects degraded (9.502%); 0 B/s, 37 objects/s recovering 2023-10-09T06:10:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2252837202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:54.655 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:10:54 smithi107 conmon[200038]: 2023-10-09T06:10:54.396+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:54.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:54 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:54 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:10:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:54 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:10:55.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:55 smithi107 ceph-mon[212750]: pgmap v252: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 58 objects/s recovering 2023-10-09T06:10:55.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1582880609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:55.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:55 smithi107 ceph-mon[220921]: pgmap v252: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 58 objects/s recovering 2023-10-09T06:10:55.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1582880609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:55 smithi150 ceph-mon[194178]: pgmap v252: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 58 objects/s recovering 2023-10-09T06:10:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1582880609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:57.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:56 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13681/139557 objects degraded (9.803%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:57.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:56 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13681/139557 objects degraded (9.803%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:56 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13681/139557 objects degraded (9.803%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:10:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:57 smithi107 ceph-mon[212750]: pgmap v253: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 39 objects/s recovering 2023-10-09T06:10:58.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:57 smithi107 ceph-mon[220921]: pgmap v253: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 39 objects/s recovering 2023-10-09T06:10:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:57 smithi150 ceph-mon[194178]: pgmap v253: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 13681/139557 objects degraded (9.803%); 0 B/s, 39 objects/s recovering 2023-10-09T06:10:59.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3199137316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:59.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3199137316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:10:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3199137316' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:10:59 smithi107 ceph-mon[212750]: pgmap v254: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 13435/129963 objects degraded (10.338%); 0 B/s, 60 objects/s recovering 2023-10-09T06:11:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:10:59 smithi107 ceph-mon[220921]: pgmap v254: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 13435/129963 objects degraded (10.338%); 0 B/s, 60 objects/s recovering 2023-10-09T06:11:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:10:59 smithi150 ceph-mon[194178]: pgmap v254: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 13435/129963 objects degraded (10.338%); 0 B/s, 60 objects/s recovering 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1694339900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1694339900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: failed to encode map e98 with expected crc 2023-10-09T06:11:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1694339900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: pgmap v256: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13435/129309 objects degraded (10.390%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13435/129309 objects degraded (10.390%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:02] "GET /metrics HTTP/1.1" 200 36287 "" "Prometheus/2.43.0" 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: pgmap v256: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13435/129309 objects degraded (10.390%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:01 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13435/129309 objects degraded (10.390%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: pgmap v256: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13435/129309 objects degraded (10.390%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: failed to encode map e99 with expected crc 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:02.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:01 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13435/129309 objects degraded (10.390%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:04.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:03 smithi107 ceph-mon[212750]: pgmap v258: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13192/126564 objects degraded (10.423%); 0 B/s, 61 objects/s recovering 2023-10-09T06:11:04.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3977223238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:03 smithi107 ceph-mon[220921]: pgmap v258: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13192/126564 objects degraded (10.423%); 0 B/s, 61 objects/s recovering 2023-10-09T06:11:04.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3977223238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:04.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:03 smithi150 ceph-mon[194178]: pgmap v258: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13192/126564 objects degraded (10.423%); 0 B/s, 61 objects/s recovering 2023-10-09T06:11:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3977223238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:05 smithi107 ceph-mon[212750]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 70 objects/s recovering 2023-10-09T06:11:06.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1341330059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:05 smithi107 ceph-mon[220921]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 70 objects/s recovering 2023-10-09T06:11:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1341330059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:05 smithi150 ceph-mon[194178]: pgmap v259: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 70 objects/s recovering 2023-10-09T06:11:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1341330059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:07.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13120/120498 objects degraded (10.888%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:07.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13120/120498 objects degraded (10.888%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13120/120498 objects degraded (10.888%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:08.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:07 smithi107 ceph-mon[212750]: pgmap v260: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:08.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:07 smithi107 ceph-mon[220921]: pgmap v260: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:08.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:07 smithi150 ceph-mon[194178]: pgmap v260: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 13120/120498 objects degraded (10.888%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:09.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1792444883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:09.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1792444883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:09.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1792444883' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:09.903 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:09 smithi107 conmon[200038]: 2023-10-09T06:11:09.412+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:10.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:09 smithi107 ceph-mon[212750]: pgmap v261: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 12927/111699 objects degraded (11.573%); 0 B/s, 57 objects/s recovering 2023-10-09T06:11:10.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:10.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:09 smithi107 ceph-mon[220921]: pgmap v261: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 12927/111699 objects degraded (11.573%); 0 B/s, 57 objects/s recovering 2023-10-09T06:11:10.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:09 smithi150 ceph-mon[194178]: pgmap v261: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 12927/111699 objects degraded (11.573%); 0 B/s, 57 objects/s recovering 2023-10-09T06:11:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:09 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2028262837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:11.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2028262837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:11.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:10 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:11.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:10 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:11.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2028262837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:11 smithi107 ceph-mon[212750]: pgmap v262: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12927/110346 objects degraded (11.715%); 0 B/s, 50 objects/s recovering 2023-10-09T06:11:12.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:11 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12927/110346 objects degraded (11.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:12.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:12] "GET /metrics HTTP/1.1" 200 36287 "" "Prometheus/2.43.0" 2023-10-09T06:11:12.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:11 smithi107 ceph-mon[220921]: pgmap v262: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12927/110346 objects degraded (11.715%); 0 B/s, 50 objects/s recovering 2023-10-09T06:11:12.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:11 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12927/110346 objects degraded (11.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:11 smithi150 ceph-mon[194178]: pgmap v262: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12927/110346 objects degraded (11.715%); 0 B/s, 50 objects/s recovering 2023-10-09T06:11:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:11 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12927/110346 objects degraded (11.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:14.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:13 smithi107 ceph-mon[212750]: pgmap v263: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 12740/107250 objects degraded (11.879%); 0 B/s, 58 objects/s recovering 2023-10-09T06:11:14.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/37013524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:14.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:13 smithi107 ceph-mon[220921]: pgmap v263: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 12740/107250 objects degraded (11.879%); 0 B/s, 58 objects/s recovering 2023-10-09T06:11:14.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/37013524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:14.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:13 smithi150 ceph-mon[194178]: pgmap v263: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 12740/107250 objects degraded (11.879%); 0 B/s, 58 objects/s recovering 2023-10-09T06:11:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/37013524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:16 smithi107 ceph-mon[212750]: pgmap v264: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 37 objects/s recovering 2023-10-09T06:11:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2503781627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:16.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:16 smithi107 ceph-mon[220921]: pgmap v264: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 37 objects/s recovering 2023-10-09T06:11:16.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2503781627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:16 smithi150 ceph-mon[194178]: pgmap v264: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 37 objects/s recovering 2023-10-09T06:11:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2503781627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12740/100197 objects degraded (12.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:17.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:17.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12740/100197 objects degraded (12.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:17.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:17.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12740/100197 objects degraded (12.715%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:18 smithi107 ceph-mon[212750]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:18.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:18 smithi107 ceph-mon[220921]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:18 smithi150 ceph-mon[194178]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12740/100197 objects degraded (12.715%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2024312266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:19.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2024312266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:19.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2024312266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:20 smithi107 ceph-mon[212750]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12543/91371 objects degraded (13.728%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:20.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:20 smithi107 ceph-mon[220921]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12543/91371 objects degraded (13.728%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:20 smithi150 ceph-mon[194178]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12543/91371 objects degraded (13.728%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:21.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/187344372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/187344372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:21.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/187344372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:22.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:22 smithi107 ceph-mon[220921]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12543/90123 objects degraded (13.918%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:22.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12543/90123 objects degraded (13.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:22.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:22] "GET /metrics HTTP/1.1" 200 36283 "" "Prometheus/2.43.0" 2023-10-09T06:11:22.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:22 smithi107 ceph-mon[212750]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12543/90123 objects degraded (13.918%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:22.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12543/90123 objects degraded (13.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:22 smithi150 ceph-mon[194178]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12543/90123 objects degraded (13.918%); 0 B/s, 31 objects/s recovering 2023-10-09T06:11:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12543/90123 objects degraded (13.918%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:24 smithi150 ceph-mon[194178]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 12341/87021 objects degraded (14.182%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1907208078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:24.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:24 smithi107 ceph-mon[220921]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 12341/87021 objects degraded (14.182%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:24.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1907208078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:24.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:24 smithi107 ceph-mon[212750]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 12341/87021 objects degraded (14.182%); 0 B/s, 48 objects/s recovering 2023-10-09T06:11:24.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1907208078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:24.903 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:24 smithi107 conmon[200038]: 2023-10-09T06:11:24.449+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:25.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:25.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[220921]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1425658679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:26.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:26.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[212750]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:26.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1425658679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:26 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:26 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:26 smithi150 ceph-mon[194178]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1425658679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12341/80031 objects degraded (15.420%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12341/80031 objects degraded (15.420%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12341/80031 objects degraded (15.420%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:28.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:28 smithi107 ceph-mon[212750]: pgmap v270: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:28.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:28 smithi107 ceph-mon[220921]: pgmap v270: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:28.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:28 smithi150 ceph-mon[194178]: pgmap v270: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 12341/80031 objects degraded (15.420%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:29.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/882715338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:29.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/882715338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:29.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/882715338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:30.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:30 smithi107 ceph-mon[212750]: pgmap v271: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 12140/71055 objects degraded (17.085%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:30.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:30 smithi107 ceph-mon[220921]: pgmap v271: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 12140/71055 objects degraded (17.085%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:30.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:30 smithi150 ceph-mon[194178]: pgmap v271: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 12140/71055 objects degraded (17.085%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:31.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/971188463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:31.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/971188463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:31.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/971188463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:32.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:32] "GET /metrics HTTP/1.1" 200 36282 "" "Prometheus/2.43.0" 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[212750]: pgmap v272: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12140/69897 objects degraded (17.368%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12140/69897 objects degraded (17.368%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[220921]: pgmap v272: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12140/69897 objects degraded (17.368%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:32.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12140/69897 objects degraded (17.368%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:32 smithi150 ceph-mon[194178]: pgmap v272: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12140/69897 objects degraded (17.368%); 0 B/s, 33 objects/s recovering 2023-10-09T06:11:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12140/69897 objects degraded (17.368%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:34.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:34 smithi107 ceph-mon[220921]: pgmap v273: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 11946/66789 objects degraded (17.886%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:34.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/739928585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:34.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:34 smithi150 ceph-mon[194178]: pgmap v273: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 11946/66789 objects degraded (17.886%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:34.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/739928585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:34.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:34 smithi107 ceph-mon[212750]: pgmap v273: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 11946/66789 objects degraded (17.886%); 0 B/s, 49 objects/s recovering 2023-10-09T06:11:34.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/739928585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:36 smithi107 ceph-mon[212750]: pgmap v274: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:36 smithi107 ceph-mon[220921]: pgmap v274: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:36.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:36 smithi150 ceph-mon[194178]: pgmap v274: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:37.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1230838163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:37.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11946/59847 objects degraded (19.961%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1230838163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11946/59847 objects degraded (19.961%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1230838163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11946/59847 objects degraded (19.961%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:38.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:38 smithi107 ceph-mon[212750]: pgmap v275: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:38 smithi150 ceph-mon[194178]: pgmap v275: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:38.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:38 smithi107 ceph-mon[220921]: pgmap v275: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11946/59847 objects degraded (19.961%); 0 B/s, 32 objects/s recovering 2023-10-09T06:11:39.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/593987526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:39.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/593987526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/593987526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:40.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:40 smithi107 ceph-mon[220921]: pgmap v276: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 11721/51024 objects degraded (22.972%); 0 B/s, 51 objects/s recovering 2023-10-09T06:11:40.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:40 smithi150 ceph-mon[194178]: pgmap v276: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 11721/51024 objects degraded (22.972%); 0 B/s, 51 objects/s recovering 2023-10-09T06:11:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:40.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:40 smithi107 ceph-mon[212750]: pgmap v276: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 11721/51024 objects degraded (22.972%); 0 B/s, 51 objects/s recovering 2023-10-09T06:11:40.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:42.301 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:42] "GET /metrics HTTP/1.1" 200 36282 "" "Prometheus/2.43.0" 2023-10-09T06:11:42.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[212750]: pgmap v277: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11721/49845 objects degraded (23.515%); 0 B/s, 34 objects/s recovering 2023-10-09T06:11:42.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/760849312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:42.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11721/49845 objects degraded (23.515%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:42 smithi150 ceph-mon[194178]: pgmap v277: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11721/49845 objects degraded (23.515%); 0 B/s, 34 objects/s recovering 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/760849312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11721/49845 objects degraded (23.515%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[220921]: pgmap v277: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 11721/49845 objects degraded (23.515%); 0 B/s, 34 objects/s recovering 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/760849312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:42.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11721/49845 objects degraded (23.515%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:44.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[212750]: pgmap v278: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 974 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 11509/46803 objects degraded (24.590%); 0 B/s, 52 objects/s recovering 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1960914401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[220921]: pgmap v278: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 974 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 11509/46803 objects degraded (24.590%); 0 B/s, 52 objects/s recovering 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1960914401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:44.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:44 smithi150 ceph-mon[194178]: pgmap v278: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 974 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 11509/46803 objects degraded (24.590%); 0 B/s, 52 objects/s recovering 2023-10-09T06:11:44.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:44.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1960914401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: pgmap v279: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.155 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:45 smithi107 conmon[200038]: 2023-10-09T06:11:45.932+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: pgmap v279: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:46.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: pgmap v279: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:11:46.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3322817556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:46 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11509/40443 objects degraded (28.457%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:47.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3322817556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11509/40443 objects degraded (28.457%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:11:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:11:47.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3322817556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:47.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:46 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11509/40443 objects degraded (28.457%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:47.657 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:11:47 smithi150 conmon[174357]: logger=cleanup t=2023-10-09T06:11:47.267385436Z level=info msg="Completed cleanup jobs" duration=120.081532ms 2023-10-09T06:11:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:47 smithi150 ceph-mon[194178]: pgmap v280: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:48.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:47 smithi107 ceph-mon[212750]: pgmap v280: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:48.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:47 smithi107 ceph-mon[220921]: pgmap v280: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 842 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 11509/40443 objects degraded (28.457%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:50.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:49 smithi107 ceph-mon[212750]: pgmap v281: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 669 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11277/32157 objects degraded (35.069%); 0 B/s, 55 objects/s recovering 2023-10-09T06:11:50.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1002183137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:49 smithi150 ceph-mon[194178]: pgmap v281: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 669 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11277/32157 objects degraded (35.069%); 0 B/s, 55 objects/s recovering 2023-10-09T06:11:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1002183137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:50.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:49 smithi107 ceph-mon[220921]: pgmap v281: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 669 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11277/32157 objects degraded (35.069%); 0 B/s, 55 objects/s recovering 2023-10-09T06:11:50.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1002183137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:52.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:11:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:11:52] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2023-10-09T06:11:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[220921]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 646 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 11277/31032 objects degraded (36.340%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1578894068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11277/31032 objects degraded (36.340%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:52.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[212750]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 646 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 11277/31032 objects degraded (36.340%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:52.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1578894068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:52.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:51 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11277/31032 objects degraded (36.340%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:52.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:51 smithi150 ceph-mon[194178]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 646 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 11277/31032 objects degraded (36.340%); 0 B/s, 36 objects/s recovering 2023-10-09T06:11:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1578894068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:51 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11277/31032 objects degraded (36.340%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:54.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:53 smithi107 ceph-mon[220921]: pgmap v283: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11040/27981 objects degraded (39.455%); 0 B/s, 56 objects/s recovering 2023-10-09T06:11:54.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:53 smithi107 ceph-mon[212750]: pgmap v283: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11040/27981 objects degraded (39.455%); 0 B/s, 56 objects/s recovering 2023-10-09T06:11:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:53 smithi150 ceph-mon[194178]: pgmap v283: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11040/27981 objects degraded (39.455%); 0 B/s, 56 objects/s recovering 2023-10-09T06:11:55.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/146006999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/146006999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/146006999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:55 smithi107 ceph-mon[220921]: pgmap v284: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:56.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:55 smithi107 ceph-mon[212750]: pgmap v284: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:56.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:55 smithi150 ceph-mon[194178]: pgmap v284: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:57.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:56 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11040/20910 objects degraded (52.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:57.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/851788334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:56 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11040/20910 objects degraded (52.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/851788334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:57.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:56 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11040/20910 objects degraded (52.798%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:11:57.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/851788334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:11:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:57 smithi107 ceph-mon[212750]: pgmap v285: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:57 smithi150 ceph-mon[194178]: pgmap v285: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:11:58.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:57 smithi107 ceph-mon[220921]: pgmap v285: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 435 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 11040/20910 objects degraded (52.798%); 0 B/s, 39 objects/s recovering 2023-10-09T06:12:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:59 smithi107 ceph-mon[212750]: pgmap v286: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 253 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10786/12192 objects degraded (88.468%); 0 B/s, 60 objects/s recovering 2023-10-09T06:12:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:11:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2328635949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:59 smithi107 ceph-mon[220921]: pgmap v286: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 253 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10786/12192 objects degraded (88.468%); 0 B/s, 60 objects/s recovering 2023-10-09T06:12:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:11:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2328635949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:59 smithi150 ceph-mon[194178]: pgmap v286: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 253 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10786/12192 objects degraded (88.468%); 0 B/s, 60 objects/s recovering 2023-10-09T06:12:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:11:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2328635949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:01.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:00 smithi107 conmon[200038]: 2023-10-09T06:12:00.951+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:12:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: pgmap v287: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10786/10884 objects degraded (99.100%); 0 B/s, 40 objects/s recovering 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 10786/10884 objects degraded (99.100%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/742766322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: pgmap v287: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10786/10884 objects degraded (99.100%); 0 B/s, 40 objects/s recovering 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T06:12:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 10786/10884 objects degraded (99.100%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/742766322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:02.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:02] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: pgmap v287: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 226 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10786/10884 objects degraded (99.100%); 0 B/s, 40 objects/s recovering 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: failed to encode map e100 with expected crc 2023-10-09T06:12:02.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:02.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 10786/10884 objects degraded (99.100%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:02.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/742766322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T06:12:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:02 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:03.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:02 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:04.041 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 102886 objects 2023-10-09T06:12:04.042 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :155.197 2023-10-09T06:12:04.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:03 smithi107 ceph-mon[212750]: failed to encode map e101 with expected crc 2023-10-09T06:12:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:03 smithi107 ceph-mon[212750]: pgmap v290: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 167 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 10478/8070 objects degraded (129.839%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:03 smithi107 ceph-mon[220921]: failed to encode map e101 with expected crc 2023-10-09T06:12:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:03 smithi107 ceph-mon[220921]: pgmap v290: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 167 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 10478/8070 objects degraded (129.839%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:03 smithi150 ceph-mon[194178]: failed to encode map e101 with expected crc 2023-10-09T06:12:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:03 smithi150 ceph-mon[194178]: pgmap v290: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 167 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 10478/8070 objects degraded (129.839%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1052324753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1052324753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1052324753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:06.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:05 smithi107 ceph-mon[212750]: pgmap v291: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:05 smithi107 ceph-mon[220921]: pgmap v291: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:05 smithi150 ceph-mon[194178]: pgmap v291: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 70 objects/s recovering 2023-10-09T06:12:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3991006371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 10478/2178 objects degraded (481.084%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3991006371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 10478/2178 objects degraded (481.084%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3991006371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 10478/2178 objects degraded (481.084%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:08.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:07 smithi107 ceph-mon[212750]: pgmap v292: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 38 objects/s recovering 2023-10-09T06:12:08.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:07 smithi107 ceph-mon[220921]: pgmap v292: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 38 objects/s recovering 2023-10-09T06:12:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:07 smithi150 ceph-mon[194178]: pgmap v292: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 44 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 10478/2178 objects degraded (481.084%); 0 B/s, 38 objects/s recovering 2023-10-09T06:12:08.845 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T06:12:08.846 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T06:12:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3033745120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:09.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3033745120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:09.222 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:09.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3033745120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:10.246 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:09 smithi107 ceph-mon[212750]: pgmap v293: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 134 objects/s recovering 2023-10-09T06:12:10.247 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:09 smithi107 ceph-mon[220921]: pgmap v293: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 134 objects/s recovering 2023-10-09T06:12:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:09 smithi150 ceph-mon[194178]: pgmap v293: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 134 objects/s recovering 2023-10-09T06:12:11.839 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_2' removed 2023-10-09T06:12:12.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:12] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-10-09T06:12:12.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: pgmap v294: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 109 objects/s recovering 2023-10-09T06:12:12.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2977118109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[220921]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: pgmap v294: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 109 objects/s recovering 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2977118109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:12.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:12:12.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:12 smithi107 ceph-mon[212750]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T06:12:12.394 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:12:12.395 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:12:12.395 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T06:12:12.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: pgmap v294: 89 pgs: 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 9710/72 objects degraded (13486.111%); 0 B/s, 109 objects/s recovering 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2977118109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3896329393' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:12:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:12 smithi150 ceph-mon[194178]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T06:12:12.733 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:13.220 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:13 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:13 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:13 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:13 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/299187008' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T06:12:14.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:12:14.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/299187008' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: failed to encode map e102 with expected crc 2023-10-09T06:12:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9710/72 objects degraded (13486.111%), 4 pgs degraded, 4 pgs undersized) 2023-10-09T06:12:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:12:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/299187008' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:15.027 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_3' created 2023-10-09T06:12:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/105716379' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:15 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/105716379' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:15 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/105716379' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:15 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T06:12:15.653 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T06:12:16.000 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:12:16.228 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:15 smithi107 conmon[200038]: 2023-10-09T06:12:15.967+0000 7f5488b69700 -1 mgr.server reply reply (11) Resource temporarily unavailable 16 pgs have unknown state; cannot draw any conclusions 2023-10-09T06:12:16.228 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:15 smithi107 conmon[200038]: 2023-10-09T06:12:15.967+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy 16 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2023-10-09T06:12:16.228 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3476251705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.229 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.230 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.230 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:12:16.230 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:12:16.230 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3476251705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:16.230 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: failed to encode map e103 with expected crc 2023-10-09T06:12:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T06:12:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T06:12:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3476251705' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.274 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.275 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.275 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:17.275 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-10-09T06:12:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: failed to encode map e104 with expected crc 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: pgmap v300: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: failed to encode map e104 with expected crc 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: pgmap v300: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:12:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: failed to encode map e104 with expected crc 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: pgmap v300: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: failed to encode map e105 with expected crc 2023-10-09T06:12:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:12:18.544 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T06:12:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[212750]: failed to encode map e105 with expected crc 2023-10-09T06:12:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2478983801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:12:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[212750]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:12:19.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[220921]: failed to encode map e105 with expected crc 2023-10-09T06:12:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2478983801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:12:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:19 smithi107 ceph-mon[220921]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:12:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:19 smithi150 ceph-mon[194178]: failed to encode map e105 with expected crc 2023-10-09T06:12:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2478983801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1035254425' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:12:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:19 smithi150 ceph-mon[194178]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T06:12:20.117 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T06:12:20.120 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:12:20.120 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:12:20.176 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:12:20.176 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:12:20.176 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_251897 2023-10-09T06:12:20.177 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:12:20.177 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[212750]: pgmap v303: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[220921]: pgmap v303: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:20.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:20 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:20 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:20 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:20 smithi150 ceph-mon[194178]: pgmap v303: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:20 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:20 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.176 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 3770 3754 234.613 234.625 0.00507031 0.00425032 2023-10-09T06:12:21.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[220921]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/849640407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:21.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[212750]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/849640407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:21 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:21 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:21 smithi150 ceph-mon[194178]: failed to encode map e106 with expected crc 2023-10-09T06:12:21.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/849640407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:22.178 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 7455 7439 232.29 230.312 0.00276257 0.00429418 2023-10-09T06:12:22.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:22 smithi107 ceph-mon[220921]: pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:22] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2023-10-09T06:12:22.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:22 smithi107 ceph-mon[212750]: pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:22 smithi150 ceph-mon[194178]: pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:12:23.178 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 10719 10703 222.858 204 0.00681904 0.00448053 2023-10-09T06:12:24.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 13781 13765 214.949 191.375 0.00525572 0.00464626 2023-10-09T06:12:24.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:24 smithi107 ceph-mon[212750]: pgmap v305: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s 2023-10-09T06:12:24.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2664999762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:24 smithi107 ceph-mon[220921]: pgmap v305: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s 2023-10-09T06:12:24.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2664999762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:24 smithi150 ceph-mon[194178]: pgmap v305: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 246 op/s 2023-10-09T06:12:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2664999762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:25.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 16768 16752 209.279 186.688 0.00451451 0.00477116 2023-10-09T06:12:26.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 19343 19327 201.223 160.938 0.00468004 0.00496324 2023-10-09T06:12:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:26 smithi107 ceph-mon[220921]: pgmap v306: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:12:26.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2781941920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:26.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:26 smithi107 ceph-mon[212750]: pgmap v306: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:12:26.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2781941920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:26 smithi150 ceph-mon[194178]: pgmap v306: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:12:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2781941920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:27.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 21983 21967 196.048 165 0.00735967 0.00509475 2023-10-09T06:12:28.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:27 smithi107 ceph-mon[220921]: pgmap v307: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:12:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:27 smithi107 ceph-mon[212750]: pgmap v307: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:12:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:27 smithi150 ceph-mon[194178]: pgmap v307: 89 pgs: 89 active+clean; 671 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:12:28.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 24450 24434 190.815 154.188 0.00520086 0.00523523 2023-10-09T06:12:29.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/732448587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:29.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/732448587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/732448587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:29.179 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 26749 26733 185.579 143.688 0.00490904 0.00538454 2023-10-09T06:12:30.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:29 smithi107 ceph-mon[212750]: pgmap v308: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T06:12:30.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:29 smithi107 ceph-mon[220921]: pgmap v308: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T06:12:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:29 smithi150 ceph-mon[194178]: pgmap v308: 89 pgs: 89 active+clean; 1.3 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-10-09T06:12:30.182 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 29077 29061 181.519 145.5 0.00422638 0.00550374 2023-10-09T06:12:31.182 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 31093 31077 176.473 126 0.00734402 0.00566106 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: pgmap v309: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: Upgrade: osd.1 is safe to restart 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3983101254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:32.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:32] "GET /metrics HTTP/1.1" 200 36273 "" "Prometheus/2.43.0" 2023-10-09T06:12:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: pgmap v309: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:12:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: Upgrade: osd.1 is safe to restart 2023-10-09T06:12:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3983101254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:32.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: pgmap v309: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: Upgrade: osd.1 is safe to restart 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3983101254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:32.184 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 15 32955 32940 171.444 116.438 0.01499 0.00582783 2023-10-09T06:12:33.185 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 34907 34891 167.637 121.938 0.00916225 0.00596016 2023-10-09T06:12:34.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:33 smithi107 ceph-mon[212750]: pgmap v310: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T06:12:34.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1927641966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:34.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:33 smithi107 ceph-mon[220921]: pgmap v310: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T06:12:34.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1927641966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:33 smithi150 ceph-mon[194178]: pgmap v310: 89 pgs: 89 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-09T06:12:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1927641966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:34.185 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 36801 36785 164.118 118.375 0.0041859 0.00608833 2023-10-09T06:12:35.185 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 38780 38764 161.417 123.688 0.00673282 0.00619021 2023-10-09T06:12:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:35 smithi107 ceph-mon[212750]: pgmap v311: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T06:12:36.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:35 smithi107 ceph-mon[220921]: pgmap v311: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T06:12:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:35 smithi150 ceph-mon[194178]: pgmap v311: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T06:12:36.185 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 40585 40569 158.38 112.812 0.00838584 0.0063103 2023-10-09T06:12:37.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3785648294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3785648294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3785648294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:37.185 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 42387 42371 155.689 112.625 0.00261575 0.00641724 2023-10-09T06:12:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:37 smithi107 ceph-mon[212750]: pgmap v312: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:12:38.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:37 smithi107 ceph-mon[220921]: pgmap v312: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:12:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:37 smithi150 ceph-mon[194178]: pgmap v312: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:12:38.186 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 44066 44050 152.87 104.938 0.00645621 0.00653649 2023-10-09T06:12:39.186 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 45728 45712 150.292 103.875 0.0123523 0.00664959 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: pgmap v313: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4181829333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: Upgrade: Updating osd.1 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[212750]: Deploying daemon osd.1 on smithi107 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: pgmap v313: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T06:12:40.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4181829333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: Upgrade: Updating osd.1 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:39 smithi107 ceph-mon[220921]: Deploying daemon osd.1 on smithi107 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: pgmap v313: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 165 MiB/s wr, 2.64k op/s 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4181829333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: Upgrade: Updating osd.1 2023-10-09T06:12:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T06:12:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:12:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:39 smithi150 ceph-mon[194178]: Deploying daemon osd.1 on smithi107 2023-10-09T06:12:40.187 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:12:40.188628+0000 min lat: 0.00118625 max lat: 0.103625 avg lat: 0.00676552 2023-10-09T06:12:40.188 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:12:40.188 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 47309 47293 147.705 98.8125 0.011397 0.00676552 2023-10-09T06:12:41.188 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 48686 48670 144.771 86.0625 0.00358686 0.00685167 2023-10-09T06:12:41.870 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:41 smithi107 ceph-mon[212750]: pgmap v314: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T06:12:42.132 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:42] "GET /metrics HTTP/1.1" 200 36273 "" "Prometheus/2.43.0" 2023-10-09T06:12:42.132 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/227079507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:42.133 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:41 smithi107 ceph-mon[220921]: pgmap v314: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T06:12:42.133 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/227079507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:41 smithi150 ceph-mon[194178]: pgmap v314: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-10-09T06:12:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/227079507' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:42.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 50076 50060 142.129 86.875 0.00511562 0.00703036 2023-10-09T06:12:43.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 51595 51579 140.078 94.9375 0.00285446 0.00713076 2023-10-09T06:12:44.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:43 smithi107 ceph-mon[212750]: pgmap v315: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:44.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1374824718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:44.144 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:43 smithi107 ceph-mon[220921]: pgmap v315: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:44.144 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1374824718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:44.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:43 smithi150 ceph-mon[194178]: pgmap v315: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-09T06:12:44.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1374824718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:44.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 52712 52696 137.151 69.8125 0.0068418 0.00728636 2023-10-09T06:12:44.402 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:44 smithi107 systemd[1]: Stopping Ceph osd.1 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:44.891 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:44 smithi107 conmon[134831]: 2023-10-09T06:12:44.607+0000 7f946b27c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:12:44.891 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:44 smithi107 conmon[134831]: 2023-10-09T06:12:44.607+0000 7f946b27c700 -1 osd.1 106 *** Got signal Terminated *** 2023-10-09T06:12:44.891 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:44 smithi107 conmon[134831]: 2023-10-09T06:12:44.607+0000 7f946b27c700 -1 osd.1 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:12:45.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:44 smithi107 ceph-mon[212750]: osd.1 marked itself down and dead 2023-10-09T06:12:45.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:44 smithi107 ceph-mon[220921]: osd.1 marked itself down and dead 2023-10-09T06:12:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:44 smithi150 ceph-mon[194178]: osd.1 marked itself down and dead 2023-10-09T06:12:45.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 53771 53755 134.314 66.1875 0.00264215 0.0073547 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: pgmap v316: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: pgmap v316: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:45.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: osdmap e107: 8 total, 7 up, 8 in 2023-10-09T06:12:45.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[212750]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: osdmap e107: 8 total, 7 up, 8 in 2023-10-09T06:12:46.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:45 smithi107 ceph-mon[220921]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: pgmap v316: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-09T06:12:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:12:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: osdmap e107: 8 total, 7 up, 8 in 2023-10-09T06:12:46.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:45 smithi150 ceph-mon[194178]: failed to encode map e107 with expected crc 2023-10-09T06:12:46.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 54248 54232 130.296 29.8125 0.0113408 0.00767078 2023-10-09T06:12:46.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:46 smithi107 bash[252925]: 674ef542e3e91392db620390dfe94507cf075e5754e2290b452fdd217fd793ea 2023-10-09T06:12:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: osdmap e108: 8 total, 7 up, 8 in 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3730043535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:47.190 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 55965 55949 129.445 107.312 0.010181 0.00772065 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: osdmap e108: 8 total, 7 up, 8 in 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3730043535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: osdmap e108: 8 total, 7 up, 8 in 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: failed to encode map e108 with expected crc 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3730043535' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:12:48.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 57639 57623 128.554 104.625 0.00638277 0.00777337 2023-10-09T06:12:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:47 smithi107 ceph-mon[212750]: pgmap v319: 89 pgs: 14 stale+active+clean, 75 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:12:48.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:47 smithi107 ceph-mon[220921]: pgmap v319: 89 pgs: 14 stale+active+clean, 75 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:12:48.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:47 smithi150 ceph-mon[194178]: pgmap v319: 89 pgs: 14 stale+active+clean, 75 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:12:49.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 59327 59311 127.759 105.5 0.0141729 0.00782251 2023-10-09T06:12:49.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2787697419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:49.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2787697419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2787697419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:49.651 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:49 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T06:12:49.652 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:49 smithi107 systemd[1]: Stopped Ceph osd.1 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:49.652 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:49 smithi107 systemd[1]: Starting Ceph osd.1 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:12:50.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 60920 60904 126.82 99.5625 0.00832471 0.00788162 2023-10-09T06:12:50.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:49 smithi107 ceph-mon[212750]: pgmap v320: 89 pgs: 8 active+undersized+degraded, 2 stale+active+clean, 27 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21816/168276 objects degraded (12.964%) 2023-10-09T06:12:50.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:49 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 21816/168276 objects degraded (12.964%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:50.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:49 smithi107 ceph-mon[220921]: pgmap v320: 89 pgs: 8 active+undersized+degraded, 2 stale+active+clean, 27 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21816/168276 objects degraded (12.964%) 2023-10-09T06:12:50.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:49 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 21816/168276 objects degraded (12.964%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:50.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:49 smithi150 ceph-mon[194178]: pgmap v320: 89 pgs: 8 active+undersized+degraded, 2 stale+active+clean, 27 active+undersized, 52 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21816/168276 objects degraded (12.964%) 2023-10-09T06:12:50.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:49 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 21816/168276 objects degraded (12.964%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T06:12:51.154 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T06:12:51.155 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:50 smithi107 bash[253581]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T06:12:51.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 62431 62415 125.775 94.4375 0.0125476 0.00794614 2023-10-09T06:12:52.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 63601 63585 124.13 73.125 0.0114603 0.00805128 2023-10-09T06:12:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:12:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:12:52] "GET /metrics HTTP/1.1" 200 36277 "" "Prometheus/2.43.0" 2023-10-09T06:12:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:51 smithi107 ceph-mon[220921]: pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 25608/173856 objects degraded (14.729%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3870645331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:51 smithi150 ceph-mon[194178]: pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 25608/173856 objects degraded (14.729%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3870645331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:52.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:51 smithi107 ceph-mon[212750]: pgmap v321: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 25608/173856 objects degraded (14.729%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:52.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3870645331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:53.101 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:52 smithi107 bash[254014]: a3526ce24032259533e57873885a0d6ca44fe0856e142f63a434ce3cd6ac2c25 2023-10-09T06:12:53.102 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:52 smithi107 systemd[1]: Started Ceph osd.1 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:12:53.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 65098 65082 123.204 93.5625 0.00863712 0.00811259 2023-10-09T06:12:53.401 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:12:53 smithi107 conmon[254053]: 2023-10-09T06:12:53.099+0000 7f4733850640 -1 Falling back to public interface 2023-10-09T06:12:54.191 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 66437 66421 122.042 83.6875 0.00864146 0.00819014 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 25608/177981 objects degraded (14.388%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4146992527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 25608/177981 objects degraded (14.388%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4146992527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: pgmap v322: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s; 25608/177981 objects degraded (14.388%); 8.0 KiB/s, 0 objects/s recovering 2023-10-09T06:12:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:12:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4146992527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:55.194 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 67818 67802 121.014 86.3125 0.00611322 0.00825944 2023-10-09T06:12:55.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:54 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 28893/195984 objects degraded (14.743%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:54 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 28893/195984 objects degraded (14.743%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:55.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:54 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 28893/195984 objects degraded (14.743%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:12:56.194 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 69055 69039 119.8 77.3125 0.0155719 0.00834226 2023-10-09T06:12:56.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:55 smithi107 ceph-mon[212750]: pgmap v323: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 28893/195984 objects degraded (14.743%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T06:12:56.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:55 smithi107 ceph-mon[220921]: pgmap v323: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 28893/195984 objects degraded (14.743%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T06:12:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:55 smithi150 ceph-mon[194178]: pgmap v323: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 28893/195984 objects degraded (14.743%); 6.5 KiB/s, 0 objects/s recovering 2023-10-09T06:12:57.194 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 70335 70319 118.725 80 0.0129542 0.00841791 2023-10-09T06:12:57.344 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2072168482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:57.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2072168482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:57.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2072168482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:58.194 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 71609 71593 117.696 79.625 0.0136173 0.00849241 2023-10-09T06:12:58.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[220921]: pgmap v324: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 28893/195984 objects degraded (14.743%); 5.9 KiB/s, 0 objects/s recovering 2023-10-09T06:12:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[212750]: pgmap v324: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 28893/195984 objects degraded (14.743%); 5.9 KiB/s, 0 objects/s recovering 2023-10-09T06:12:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:57 smithi150 ceph-mon[194178]: pgmap v324: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s; 28893/195984 objects degraded (14.743%); 5.9 KiB/s, 0 objects/s recovering 2023-10-09T06:12:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.194 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 72760 72744 116.523 71.9375 0.0184727 0.00857598 2023-10-09T06:12:59.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: pgmap v325: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 31312/210804 objects degraded (14.854%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4054786777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: pgmap v325: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 31312/210804 objects degraded (14.854%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:12:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:12:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4054786777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:12:59.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: pgmap v325: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 31312/210804 objects degraded (14.854%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:12:59.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:12:59.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:12:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4054786777' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:00.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:13:00.196066+0000 min lat: 0.00118625 max lat: 1.03991 avg lat: 0.00865152 2023-10-09T06:13:00.196 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:13:00.196 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 73987 73971 115.525 76.6875 0.00744051 0.00865152 2023-10-09T06:13:00.303 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:13:00 smithi107 conmon[254053]: 2023-10-09T06:13:00.297+0000 7f4733850640 -1 osd.1 106 log_to_monitors true 2023-10-09T06:13:01.151 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:13:00 smithi107 conmon[254053]: 2023-10-09T06:13:00.818+0000 7f4724cbc700 -1 osd.1 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:13:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:00 smithi107 ceph-mon[212750]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:00 smithi107 ceph-mon[220921]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:00 smithi150 ceph-mon[194178]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T06:13:01.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 75394 75378 114.852 87.9375 0.0101811 0.00870288 2023-10-09T06:13:02.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:02] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.43.0" 2023-10-09T06:13:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: pgmap v326: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 31705/215403 objects degraded (14.719%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:13:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: osdmap e109: 8 total, 7 up, 8 in 2023-10-09T06:13:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 31705/215403 objects degraded (14.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2396712524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: pgmap v326: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 31705/215403 objects degraded (14.719%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: osdmap e109: 8 total, 7 up, 8 in 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 31705/215403 objects degraded (14.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2396712524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:02.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: pgmap v326: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 31705/215403 objects degraded (14.719%); 5.3 KiB/s, 0 objects/s recovering 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: osdmap e109: 8 total, 7 up, 8 in 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: from='osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: failed to encode map e109 with expected crc 2023-10-09T06:13:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 31705/215403 objects degraded (14.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2396712524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:02.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 76011 75995 113.036 38.5625 0.00301697 0.00879342 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293] boot 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T06:13:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293] boot 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T06:13:03.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: failed to encode map e110 with expected crc 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: osd.1 [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293] boot 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T06:13:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T06:13:03.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 76011 75995 110.408 0 - 0.00879342 2023-10-09T06:13:04.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: pgmap v329: 89 pgs: 11 peering, 9 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 31704/218250 objects degraded (14.526%) 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[212750]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: pgmap v329: 89 pgs: 11 peering, 9 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 31704/218250 objects degraded (14.526%) 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:03 smithi107 ceph-mon[220921]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: pgmap v329: 89 pgs: 11 peering, 9 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 31704/218250 objects degraded (14.526%) 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:03 smithi150 ceph-mon[194178]: failed to encode map e111 with expected crc 2023-10-09T06:13:04.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 76085 76069 108.005 2.3125 0.00866211 0.0091883 2023-10-09T06:13:04.829 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T06:13:04.829 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4184273929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4184273929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: failed to encode map e112 with expected crc 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:04.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:04.833 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:05.154 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:04 smithi107 conmon[200038]: 2023-10-09T06:13:04.885+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-10-09T06:13:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T06:13:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4184273929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: failed to encode map e112 with expected crc 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:05.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:13:05.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:13:05.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:05.195 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 77190 77174 107.14 69.0625 0.0109975 0.00932921 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: pgmap v332: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 10 active+undersized, 58 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-10-09T06:13:06.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: pgmap v332: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 10 active+undersized, 58 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:05 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-10-09T06:13:06.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: pgmap v332: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 10 active+undersized, 58 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:06.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:05 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-10-09T06:13:06.197 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 78260 78244 106.261 66.875 0.0179255 0.00940632 2023-10-09T06:13:07.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2613525290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:07.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 10337/226617 objects degraded (4.561%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:07.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2613525290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:07.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 10337/226617 objects degraded (4.561%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2613525290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 10337/226617 objects degraded (4.561%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:07.198 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 79299 79283 105.381 64.9375 0.0126574 0.00948473 2023-10-09T06:13:08.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:07 smithi107 ceph-mon[212750]: pgmap v333: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 68 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:08.193 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:07 smithi107 ceph-mon[220921]: pgmap v333: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 68 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:08.194 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:07 smithi150 ceph-mon[194178]: pgmap v333: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 14 peering, 2 active+undersized+degraded, 68 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 10337/226617 objects degraded (4.561%) 2023-10-09T06:13:08.198 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 80402 80386 104.622 68.9375 0.0157122 0.00955318 2023-10-09T06:13:09.198 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 81399 81383 103.759 62.3125 0.0119266 0.00963255 2023-10-09T06:13:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:09 smithi107 ceph-mon[212750]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9486/237765 objects degraded (3.990%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:13:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3119811330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:09 smithi107 ceph-mon[220921]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9486/237765 objects degraded (3.990%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:13:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3119811330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:09 smithi150 ceph-mon[194178]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 9486/237765 objects degraded (3.990%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:13:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3119811330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:10.198 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 82455 82439 103.004 66 0.0105805 0.00970357 2023-10-09T06:13:11.198 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 83497 83481 102.261 65.125 0.00697657 0.00977349 2023-10-09T06:13:12.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[220921]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 9486/241773 objects degraded (3.924%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:13:12.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4156104714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:12.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 9486/241773 objects degraded (3.924%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:11 smithi150 ceph-mon[194178]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 9486/241773 objects degraded (3.924%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:13:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4156104714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:11 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 9486/241773 objects degraded (3.924%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:12.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[212750]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 9486/241773 objects degraded (3.924%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:13:12.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4156104714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:12.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:11 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 9486/241773 objects degraded (3.924%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:12.159 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:12] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.43.0" 2023-10-09T06:13:12.199 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 84508 84492 101.509 63.1875 0.012293 0.00984569 2023-10-09T06:13:13.200 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 85545 85529 100.815 64.8125 0.0114965 0.0099141 2023-10-09T06:13:14.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:13 smithi107 ceph-mon[212750]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 9486/244626 objects degraded (3.878%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:13:14.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:13 smithi107 ceph-mon[220921]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 9486/244626 objects degraded (3.878%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:13:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:13 smithi150 ceph-mon[194178]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s; 9486/244626 objects degraded (3.878%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:13:14.207 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 86518 86502 100.07 60.8125 0.00720033 0.00998829 2023-10-09T06:13:15.202 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 87538 87522 99.4093 63.75 0.00680836 0.010055 2023-10-09T06:13:15.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1014646233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:15.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1014646233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:15.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1014646233' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:16.203 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 88551 88535 98.7649 63.3125 0.00964113 0.0101199 2023-10-09T06:13:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:15 smithi107 ceph-mon[212750]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9092/257157 objects degraded (3.536%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:13:16.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:15 smithi107 ceph-mon[220921]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9092/257157 objects degraded (3.536%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:13:16.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:15 smithi150 ceph-mon[194178]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9092/257157 objects degraded (3.536%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:13:17.203 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 89488 89472 98.0597 58.5625 0.0161259 0.0101935 2023-10-09T06:13:17.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:17.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:17.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2087032872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[220921]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9092/257157 objects degraded (3.536%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:13:17.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 9092/257157 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2087032872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:17 smithi150 ceph-mon[194178]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9092/257157 objects degraded (3.536%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 9092/257157 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:17.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2087032872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:17.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[212750]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 9092/257157 objects degraded (3.536%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:13:17.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 9092/257157 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:18.203 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 90486 90470 97.4446 62.375 0.0210457 0.0102572 2023-10-09T06:13:19.204 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 91418 91402 96.7803 58.25 0.0238569 0.010328 2023-10-09T06:13:20.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:19 smithi107 conmon[200038]: 2023-10-09T06:13:19.910+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:19 smithi107 ceph-mon[212750]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 8674/268074 objects degraded (3.236%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:13:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/352556364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:20.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:19 smithi107 ceph-mon[220921]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 8674/268074 objects degraded (3.236%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:13:20.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/352556364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:20.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:19 smithi150 ceph-mon[194178]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 8674/268074 objects degraded (3.236%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:13:20.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/352556364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:20.204 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:13:20.204753+0000 min lat: 0.00118625 max lat: 2.21722 avg lat: 0.0103918 2023-10-09T06:13:20.204 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:13:20.204 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 92390 92374 96.1785 60.75 0.0106055 0.0103918 2023-10-09T06:13:21.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:20 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:20 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:20 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:21.204 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 93207 93191 95.4391 51.0625 0.00349023 0.0104332 2023-10-09T06:13:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 8674/271995 objects degraded (3.189%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:13:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T06:13:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/591604387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8674/271995 objects degraded (3.189%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:22.153 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:22] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 8674/271995 objects degraded (3.189%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/591604387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:21 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8674/271995 objects degraded (3.189%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 8674/271995 objects degraded (3.189%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: failed to encode map e113 with expected crc 2023-10-09T06:13:22.161 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/591604387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:22.161 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:21 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8674/271995 objects degraded (3.189%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:22.207 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 93508 93492 94.1989 18.8125 0.00826548 0.0106101 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[212750]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:22 smithi107 ceph-mon[220921]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:22 smithi150 ceph-mon[194178]: failed to encode map e114 with expected crc 2023-10-09T06:13:23.207 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 94472 94456 93.6601 60.25 0.00726516 0.0106721 2023-10-09T06:13:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:23 smithi107 ceph-mon[212750]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8674/274215 objects degraded (3.163%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:13:24.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:23 smithi150 ceph-mon[194178]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8674/274215 objects degraded (3.163%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:13:24.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:23 smithi107 ceph-mon[220921]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8674/274215 objects degraded (3.163%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:13:24.207 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 95347 95331 93.0514 54.6875 0.0103411 0.0107414 2023-10-09T06:13:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2444797838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2444797838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:25.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2444797838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:25.208 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 96282 96266 92.5184 58.4375 0.0211403 0.0108031 2023-10-09T06:13:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:25 smithi107 ceph-mon[220921]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 8300/283140 objects degraded (2.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:25 smithi150 ceph-mon[194178]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 8300/283140 objects degraded (2.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:26.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:25 smithi107 ceph-mon[212750]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 8300/283140 objects degraded (2.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:26.209 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 97218 97202 92.0031 58.5 0.0192981 0.0108649 2023-10-09T06:13:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2344540546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8300/283140 objects degraded (2.931%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:27.208 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 15 98097 98082 91.4509 55 0.011715 0.0109294 2023-10-09T06:13:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2344540546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:27.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8300/283140 objects degraded (2.931%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:27.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2344540546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:27.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8300/283140 objects degraded (2.931%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:28.208 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 98961 98945 90.8994 53.9375 0.00482767 0.0109966 2023-10-09T06:13:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:27 smithi107 ceph-mon[220921]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8015/284442 objects degraded (2.818%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:27 smithi107 ceph-mon[212750]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8015/284442 objects degraded (2.818%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:27 smithi150 ceph-mon[194178]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 8015/284442 objects degraded (2.818%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:29.208 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 99827 99811 90.3666 54.125 0.0248723 0.0110615 2023-10-09T06:13:30.209 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 100684 100668 89.8399 53.5625 0.0097459 0.0111254 2023-10-09T06:13:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:29 smithi107 ceph-mon[220921]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8015/293925 objects degraded (2.727%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1975533257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:29 smithi107 ceph-mon[212750]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8015/293925 objects degraded (2.727%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1975533257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:30.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:29 smithi150 ceph-mon[194178]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8015/293925 objects degraded (2.727%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:13:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1975533257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:31.209 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 101616 101600 89.3951 58.25 0.015526 0.011182 2023-10-09T06:13:32.209 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 102439 102423 88.868 51.4375 0.0240227 0.0112467 2023-10-09T06:13:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[220921]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 7665/297189 objects degraded (2.579%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:13:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7665/297189 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:32.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/712398851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[212750]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 7665/297189 objects degraded (2.579%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:13:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7665/297189 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/712398851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:32.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:32] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-10-09T06:13:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:31 smithi150 ceph-mon[194178]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 7665/297189 objects degraded (2.579%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:13:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:31 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7665/297189 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/712398851' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:33.209 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 103346 103330 88.4273 56.6875 0.00852545 0.0113033 2023-10-09T06:13:34.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 104208 104192 87.9589 53.875 0.0236257 0.0113643 2023-10-09T06:13:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:33 smithi107 ceph-mon[212750]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 7665/300459 objects degraded (2.551%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:13:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:33 smithi107 ceph-mon[220921]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 7665/300459 objects degraded (2.551%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:13:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:33 smithi150 ceph-mon[194178]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 7665/300459 objects degraded (2.551%); 5.8 MiB/s, 92 objects/s recovering 2023-10-09T06:13:35.211 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 105061 105045 87.4971 53.3125 0.0127336 0.011425 2023-10-09T06:13:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1629256087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:34 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:35.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:34 smithi107 conmon[200038]: 2023-10-09T06:13:34.931+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1629256087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:34 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:35.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1629256087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:35.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:34 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:36.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 105966 105950 87.0877 56.5625 0.0104931 0.0114781 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[212750]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7305/310194 objects degraded (2.355%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[220921]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7305/310194 objects degraded (2.355%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:35 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:35 smithi150 ceph-mon[194178]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 7305/310194 objects degraded (2.355%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:13:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:35 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:35 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T06:13:37.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 106786 106770 86.6224 51.25 0.0117913 0.0115396 2023-10-09T06:13:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7305/310194 objects degraded (2.355%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2561857832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7305/310194 objects degraded (2.355%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2561857832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:37.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7305/310194 objects degraded (2.355%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2561857832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:38.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 107691 107675 86.2371 56.5625 0.0123687 0.0115913 2023-10-09T06:13:38.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:38 smithi107 ceph-mon[220921]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7305/311028 objects degraded (2.349%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:13:38.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:38 smithi107 ceph-mon[212750]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7305/311028 objects degraded (2.349%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:13:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:38 smithi150 ceph-mon[194178]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7305/311028 objects degraded (2.349%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:13:39.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 108536 108520 85.8141 52.8125 0.00976426 0.0116407 2023-10-09T06:13:40.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:13:40.214252+0000 min lat: 0.00118625 max lat: 2.21722 avg lat: 0.0117781 2023-10-09T06:13:40.213 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:13:40.214 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 108699 108683 84.8691 10.1875 0.0164316 0.0117781 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 7305/319908 objects degraded (2.283%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3977576751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 7305/319908 objects degraded (2.283%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3977576751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 7305/319908 objects degraded (2.283%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: osdmap e115: 8 total, 8 up, 8 in 2023-10-09T06:13:40.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: failed to encode map e115 with expected crc 2023-10-09T06:13:40.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3977576751' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:41.214 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 109564 109548 84.4879 54.0625 0.011654 0.0118313 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[212750]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T06:13:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:41 smithi107 ceph-mon[220921]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: osdmap e116: 8 total, 8 up, 8 in 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: failed to encode map e116 with expected crc 2023-10-09T06:13:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:41 smithi150 ceph-mon[194178]: failed to encode map e116 with expected crc 2023-10-09T06:13:42.215 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 110371 110355 84.0719 50.4375 0.0109673 0.0118899 2023-10-09T06:13:42.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:42] "GET /metrics HTTP/1.1" 200 36299 "" "Prometheus/2.43.0" 2023-10-09T06:13:42.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[212750]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7305/322176 objects degraded (2.267%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7305/322176 objects degraded (2.267%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3267234944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[220921]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7305/322176 objects degraded (2.267%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7305/322176 objects degraded (2.267%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3267234944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:42 smithi150 ceph-mon[194178]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7305/322176 objects degraded (2.267%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:13:42.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7305/322176 objects degraded (2.267%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:42.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3267234944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:43.217 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 111216 111200 83.6934 52.8125 0.0190773 0.0119436 2023-10-09T06:13:44.217 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 112050 112034 83.3177 52.125 0.0165659 0.0119973 2023-10-09T06:13:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:44 smithi107 ceph-mon[212750]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 649 op/s; 6985/325788 objects degraded (2.144%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:13:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:44 smithi107 ceph-mon[220921]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 649 op/s; 6985/325788 objects degraded (2.144%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:13:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:44 smithi150 ceph-mon[194178]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 649 op/s; 6985/325788 objects degraded (2.144%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:13:45.217 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 112878 112862 82.9464 51.75 0.0058764 0.012051 2023-10-09T06:13:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2631606692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:45.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2631606692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2631606692' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:46.217 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 113722 113706 82.5954 52.75 0.0175021 0.0121021 2023-10-09T06:13:46.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:46 smithi107 ceph-mon[212750]: pgmap v356: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s; 6585/333264 objects degraded (1.976%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:46.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:46 smithi107 ceph-mon[220921]: pgmap v356: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s; 6585/333264 objects degraded (1.976%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:46.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:46 smithi150 ceph-mon[194178]: pgmap v356: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s; 6585/333264 objects degraded (1.976%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:47.219 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 114487 114471 82.1943 47.8125 0.0126157 0.0121616 2023-10-09T06:13:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6585/333264 objects degraded (1.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/874334947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6585/333264 objects degraded (1.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/874334947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:13:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6585/333264 objects degraded (1.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/874334947' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:48.219 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 115330 115314 81.8591 52.6875 0.0185917 0.0122113 2023-10-09T06:13:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:48 smithi107 ceph-mon[212750]: pgmap v357: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 6585/334881 objects degraded (1.966%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:48 smithi107 ceph-mon[220921]: pgmap v357: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 6585/334881 objects degraded (1.966%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:48.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:48 smithi150 ceph-mon[194178]: pgmap v357: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 6585/334881 objects degraded (1.966%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:13:49.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 116155 116139 81.5186 51.5625 0.0150022 0.0122619 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0072 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 116911 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 81.1817 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 44.8775 2023-10-09T06:13:50.220 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 234.625 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1298 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 718.04 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3754 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0123142 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0322089 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.21722 2023-10-09T06:13:50.221 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00118625 2023-10-09T06:13:50.224 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:13:50.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[220921]: pgmap v358: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 6248/343188 objects degraded (1.821%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:13:50.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3320734047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:50.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:50.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:49 smithi107 conmon[200038]: 2023-10-09T06:13:49.947+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:13:50.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[212750]: pgmap v358: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 6248/343188 objects degraded (1.821%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:13:50.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3320734047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:50.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:50 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:50 smithi150 ceph-mon[194178]: pgmap v358: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s; 6248/343188 objects degraded (1.821%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:13:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3320734047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:50 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:51.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:51 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:51 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:13:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:51 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:51 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:13:51.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:51 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:13:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:51 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:13:52.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[212750]: pgmap v359: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 6248/345405 objects degraded (1.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:52.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6248/345405 objects degraded (1.809%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2007789537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[220921]: pgmap v359: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 6248/345405 objects degraded (1.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6248/345405 objects degraded (1.809%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:52.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2007789537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:52.407 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:13:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:13:52] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.43.0" 2023-10-09T06:13:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:52 smithi150 ceph-mon[194178]: pgmap v359: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 721 op/s; 6248/345405 objects degraded (1.809%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:13:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6248/345405 objects degraded (1.809%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2007789537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:54 smithi107 ceph-mon[212750]: pgmap v360: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 749 op/s; 6248/346497 objects degraded (1.803%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:13:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:54 smithi107 ceph-mon[220921]: pgmap v360: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 749 op/s; 6248/346497 objects degraded (1.803%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:13:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:54 smithi150 ceph-mon[194178]: pgmap v360: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 749 op/s; 6248/346497 objects degraded (1.803%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:13:55.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/108337602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:55.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/108337602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/108337602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:56 smithi107 ceph-mon[220921]: pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 888 op/s; 5891/343836 objects degraded (1.713%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:13:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:56 smithi150 ceph-mon[194178]: pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 888 op/s; 5891/343836 objects degraded (1.713%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:13:56.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:56 smithi107 ceph-mon[212750]: pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 888 op/s; 5891/343836 objects degraded (1.713%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:13:57.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5891/343836 objects degraded (1.713%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:57.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5891/343836 objects degraded (1.713%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5891/343836 objects degraded (1.713%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 725 op/s; 5891/342228 objects degraded (1.721%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/323906191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 725 op/s; 5891/342228 objects degraded (1.721%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/323906191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:58 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 30 MiB/s wr, 725 op/s; 5891/342228 objects degraded (1.721%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/323906191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: osdmap e117: 8 total, 8 up, 8 in 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:58 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:13:59 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e117 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:13:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:13:59 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:00 smithi150 ceph-mon[194178]: pgmap v364: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 800 op/s; 5796/334419 objects degraded (1.733%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:14:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:00 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:00 smithi150 ceph-mon[194178]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2057945073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[212750]: pgmap v364: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 800 op/s; 5796/334419 objects degraded (1.733%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[212750]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2057945073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[220921]: pgmap v364: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 800 op/s; 5796/334419 objects degraded (1.733%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[220921]: failed to encode map e118 with expected crc 2023-10-09T06:14:00.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2057945073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:02.320 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:02] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T06:14:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:02 smithi150 ceph-mon[194178]: pgmap v366: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 832 op/s; 5796/332445 objects degraded (1.743%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:02.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5796/332445 objects degraded (1.743%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[212750]: pgmap v366: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 832 op/s; 5796/332445 objects degraded (1.743%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5796/332445 objects degraded (1.743%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[220921]: pgmap v366: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 832 op/s; 5796/332445 objects degraded (1.743%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:02.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5796/332445 objects degraded (1.743%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:14:03.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3543454909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:03.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3543454909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:03.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3543454909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:04.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:04 smithi107 ceph-mon[212750]: pgmap v367: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5483/328011 objects degraded (1.672%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:04 smithi107 ceph-mon[220921]: pgmap v367: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5483/328011 objects degraded (1.672%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:04.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:04 smithi150 ceph-mon[194178]: pgmap v367: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5483/328011 objects degraded (1.672%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1538258668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:05.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:05.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1538258668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:05.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:05.660 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1538258668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:06.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:06 smithi107 ceph-mon[220921]: pgmap v368: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5186/320739 objects degraded (1.617%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:14:06.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:06 smithi150 ceph-mon[194178]: pgmap v368: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5186/320739 objects degraded (1.617%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:14:06.660 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:06 smithi107 ceph-mon[212750]: pgmap v368: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 5186/320739 objects degraded (1.617%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:14:07.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5186/320739 objects degraded (1.617%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:07.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5186/320739 objects degraded (1.617%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:07.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5186/320739 objects degraded (1.617%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:08 smithi107 ceph-mon[212750]: pgmap v369: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5186/318558 objects degraded (1.628%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:14:08.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4126693614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:08.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:08 smithi150 ceph-mon[194178]: pgmap v369: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5186/318558 objects degraded (1.628%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:14:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4126693614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:08.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:08 smithi107 ceph-mon[220921]: pgmap v369: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 5186/318558 objects degraded (1.628%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T06:14:08.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4126693614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[212750]: pgmap v370: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4938/311445 objects degraded (1.586%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T06:14:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1874633313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:10.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[220921]: pgmap v370: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4938/311445 objects degraded (1.586%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T06:14:10.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1874633313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:10.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:10 smithi150 ceph-mon[194178]: pgmap v370: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4938/311445 objects degraded (1.586%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T06:14:10.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:10.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1874633313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:12.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[212750]: pgmap v371: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4938/309444 objects degraded (1.596%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:12.318 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:12] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T06:14:12.584 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:12 smithi107 conmon[200038]: 2023-10-09T06:14:12.583+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4938/309444 objects degraded (1.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[220921]: pgmap v371: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4938/309444 objects degraded (1.596%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.585 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4938/309444 objects degraded (1.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:12 smithi150 ceph-mon[194178]: pgmap v371: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4938/309444 objects degraded (1.596%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4938/309444 objects degraded (1.596%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2678545869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.667 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2678545869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:14:13.668 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.669 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.669 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:14:13.669 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:13.669 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2678545869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:14.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:14.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:14.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[212750]: pgmap v372: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4938/305559 objects degraded (1.616%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:14 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:14 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:14 smithi150 ceph-mon[194178]: pgmap v372: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4938/305559 objects degraded (1.616%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:14.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:14 smithi107 ceph-mon[220921]: pgmap v372: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 4938/305559 objects degraded (1.616%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T06:14:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2193330696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:15.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2193330696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:15.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2193330696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:16 smithi107 ceph-mon[212750]: pgmap v373: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 4698/298608 objects degraded (1.573%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:14:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:16 smithi107 ceph-mon[220921]: pgmap v373: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 4698/298608 objects degraded (1.573%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:14:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:16 smithi150 ceph-mon[194178]: pgmap v373: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 4698/298608 objects degraded (1.573%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:14:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4698/298608 objects degraded (1.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4698/298608 objects degraded (1.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4698/298608 objects degraded (1.573%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:18.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:18 smithi107 ceph-mon[212750]: pgmap v374: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4698/296376 objects degraded (1.585%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:14:18.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2309344220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:18.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:18 smithi107 ceph-mon[220921]: pgmap v374: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4698/296376 objects degraded (1.585%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:14:18.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2309344220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:18 smithi150 ceph-mon[194178]: pgmap v374: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 4698/296376 objects degraded (1.585%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:14:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2309344220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:20.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:20 smithi107 ceph-mon[212750]: pgmap v375: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 4411/289080 objects degraded (1.526%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:14:20.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:20 smithi107 ceph-mon[220921]: pgmap v375: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 4411/289080 objects degraded (1.526%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:14:20.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:20 smithi150 ceph-mon[194178]: pgmap v375: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 4411/289080 objects degraded (1.526%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:14:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2258863162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:21.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2258863162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:21.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2258863162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:22] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-10-09T06:14:22.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:22 smithi107 ceph-mon[212750]: pgmap v376: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4411/286809 objects degraded (1.538%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:22.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4411/286809 objects degraded (1.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:22 smithi107 ceph-mon[220921]: pgmap v376: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4411/286809 objects degraded (1.538%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4411/286809 objects degraded (1.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:22.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:22 smithi150 ceph-mon[194178]: pgmap v376: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4411/286809 objects degraded (1.538%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:22.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4411/286809 objects degraded (1.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: pgmap v377: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4411/282237 objects degraded (1.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:14:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3629387792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:24.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: pgmap v377: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4411/282237 objects degraded (1.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:24.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:14:24.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3629387792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: pgmap v377: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 4411/282237 objects degraded (1.563%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T06:14:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: failed to encode map e119 with expected crc 2023-10-09T06:14:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3629387792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:24 smithi150 ceph-mon[194178]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[220921]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:25.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:24 smithi107 ceph-mon[212750]: failed to encode map e120 with expected crc 2023-10-09T06:14:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:25 smithi107 ceph-mon[220921]: pgmap v380: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4061/275652 objects degraded (1.473%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:14:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4247326934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:25 smithi107 ceph-mon[212750]: pgmap v380: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4061/275652 objects degraded (1.473%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:14:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4247326934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:26.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:25 smithi150 ceph-mon[194178]: pgmap v380: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4061/275652 objects degraded (1.473%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:14:26.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4247326934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:27.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4061/275652 objects degraded (1.473%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:27.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4061/275652 objects degraded (1.473%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4061/275652 objects degraded (1.473%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:27.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:27 smithi107 ceph-mon[212750]: pgmap v381: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4061/273711 objects degraded (1.484%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:27.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:27.898 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:27 smithi107 conmon[200038]: 2023-10-09T06:14:27.611+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:28.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:27 smithi107 ceph-mon[220921]: pgmap v381: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4061/273711 objects degraded (1.484%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:27 smithi150 ceph-mon[194178]: pgmap v381: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 4061/273711 objects degraded (1.484%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:28 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:29.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:28 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:29.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1323334252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:29.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:29.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:29.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1323334252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:29.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:29.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:14:29.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1323334252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:29 smithi107 ceph-mon[212750]: pgmap v382: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 3751/267261 objects degraded (1.403%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:30.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:29 smithi107 ceph-mon[220921]: pgmap v382: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 3751/267261 objects degraded (1.403%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:30.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:29 smithi150 ceph-mon[194178]: pgmap v382: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 3751/267261 objects degraded (1.403%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/850712415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:31.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/850712415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:31.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/850712415' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:32.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[212750]: pgmap v383: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 3751/265134 objects degraded (1.415%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:32.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:32.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3751/265134 objects degraded (1.415%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:32.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:32] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T06:14:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[220921]: pgmap v383: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 3751/265134 objects degraded (1.415%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:31 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3751/265134 objects degraded (1.415%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:31 smithi150 ceph-mon[194178]: pgmap v383: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 3751/265134 objects degraded (1.415%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T06:14:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:31 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3751/265134 objects degraded (1.415%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:33 smithi107 ceph-mon[212750]: pgmap v384: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3751/260652 objects degraded (1.439%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:14:34.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/487616776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:33 smithi107 ceph-mon[220921]: pgmap v384: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3751/260652 objects degraded (1.439%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:14:34.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/487616776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:34.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:33 smithi150 ceph-mon[194178]: pgmap v384: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3751/260652 objects degraded (1.439%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:14:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/487616776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:35 smithi107 ceph-mon[212750]: pgmap v385: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3503/253422 objects degraded (1.382%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/621281713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:36.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:35 smithi150 ceph-mon[194178]: pgmap v385: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3503/253422 objects degraded (1.382%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/621281713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:36.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:35 smithi107 ceph-mon[220921]: pgmap v385: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3503/253422 objects degraded (1.382%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:14:36.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/621281713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:37.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3503/253422 objects degraded (1.382%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:37.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3503/253422 objects degraded (1.382%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3503/253422 objects degraded (1.382%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:38 smithi150 ceph-mon[194178]: pgmap v386: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 3503/251232 objects degraded (1.394%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:14:38.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:38 smithi107 ceph-mon[212750]: pgmap v386: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 3503/251232 objects degraded (1.394%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:14:38.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:38 smithi107 ceph-mon[220921]: pgmap v386: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 3503/251232 objects degraded (1.394%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:14:39.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3700639901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:39.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3700639901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3700639901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:40 smithi150 ceph-mon[194178]: pgmap v387: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 3232/244002 objects degraded (1.325%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:14:40.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:40 smithi107 ceph-mon[212750]: pgmap v387: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 3232/244002 objects degraded (1.325%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:14:40.409 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:40 smithi107 ceph-mon[220921]: pgmap v387: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 3232/244002 objects degraded (1.325%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:14:41.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1701849088' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1701849088' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:41.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1701849088' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:42.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:42 smithi107 ceph-mon[220921]: pgmap v388: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 3232/241866 objects degraded (1.336%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:42.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3232/241866 objects degraded (1.336%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:42 smithi107 ceph-mon[212750]: pgmap v388: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 3232/241866 objects degraded (1.336%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3232/241866 objects degraded (1.336%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:42.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:42] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-10-09T06:14:42.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:42 smithi150 ceph-mon[194178]: pgmap v388: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 3232/241866 objects degraded (1.336%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3232/241866 objects degraded (1.336%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:43.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:44.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:44.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[212750]: Upgrade: osd.2 is safe to restart 2023-10-09T06:14:44.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[212750]: pgmap v389: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3232/237444 objects degraded (1.361%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:44.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/845483230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:44.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:44.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[220921]: Upgrade: osd.2 is safe to restart 2023-10-09T06:14:44.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[220921]: pgmap v389: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3232/237444 objects degraded (1.361%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:44.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/845483230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:44 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T06:14:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:44 smithi150 ceph-mon[194178]: Upgrade: osd.2 is safe to restart 2023-10-09T06:14:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:44 smithi150 ceph-mon[194178]: pgmap v389: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3232/237444 objects degraded (1.361%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:14:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/845483230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:46.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[212750]: pgmap v390: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 2951/230277 objects degraded (1.282%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:14:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:14:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1666873144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[220921]: pgmap v390: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 2951/230277 objects degraded (1.282%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:14:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:14:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1666873144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:46 smithi150 ceph-mon[194178]: pgmap v390: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 843 op/s; 2951/230277 objects degraded (1.282%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:14:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:14:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T06:14:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:14:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1666873144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[212750]: Upgrade: Updating osd.2 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[212750]: Deploying daemon osd.2 on smithi107 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2951/230277 objects degraded (1.282%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[220921]: Upgrade: Updating osd.2 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[220921]: Deploying daemon osd.2 on smithi107 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:47.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2951/230277 objects degraded (1.282%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:47 smithi150 ceph-mon[194178]: Upgrade: Updating osd.2 2023-10-09T06:14:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:47 smithi150 ceph-mon[194178]: Deploying daemon osd.2 on smithi107 2023-10-09T06:14:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:14:47.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2951/230277 objects degraded (1.282%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:48 smithi107 ceph-mon[212750]: pgmap v391: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 2951/228237 objects degraded (1.293%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:14:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:48 smithi107 ceph-mon[220921]: pgmap v391: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 2951/228237 objects degraded (1.293%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:14:48.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:48 smithi150 ceph-mon[194178]: pgmap v391: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 2951/228237 objects degraded (1.293%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3685332149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3685332149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:49.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T06:14:49.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: failed to encode map e121 with expected crc 2023-10-09T06:14:49.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3685332149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: pgmap v393: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 2905/222516 objects degraded (1.306%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[212750]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: pgmap v393: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 2905/222516 objects degraded (1.306%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.331 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:50 smithi107 ceph-mon[220921]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.652 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:50 smithi107 systemd[1]: Stopping Ceph osd.2 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:14:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: pgmap v393: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 2905/222516 objects degraded (1.306%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: failed to encode map e122 with expected crc 2023-10-09T06:14:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:50 smithi150 ceph-mon[194178]: failed to encode map e122 with expected crc 2023-10-09T06:14:51.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:51 smithi107 ceph-mon[212750]: osd.2 marked itself down and dead 2023-10-09T06:14:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2759413333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:51 smithi107 ceph-mon[220921]: osd.2 marked itself down and dead 2023-10-09T06:14:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2759413333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:51.401 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:51 smithi107 conmon[143874]: 2023-10-09T06:14:51.013+0000 7fc633b43700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:14:51.401 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:51 smithi107 conmon[143874]: 2023-10-09T06:14:51.013+0000 7fc633b43700 -1 osd.2 122 *** Got signal Terminated *** 2023-10-09T06:14:51.402 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:51 smithi107 conmon[143874]: 2023-10-09T06:14:51.013+0000 7fc633b43700 -1 osd.2 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:14:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:51 smithi150 ceph-mon[194178]: osd.2 marked itself down and dead 2023-10-09T06:14:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2759413333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:14:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:14:52] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: pgmap v395: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 2905/220827 objects degraded (1.316%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2905/220827 objects degraded (1.316%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:52.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: pgmap v395: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 2905/220827 objects degraded (1.316%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:14:52.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2905/220827 objects degraded (1.316%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: pgmap v395: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 2905/220827 objects degraded (1.316%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: osdmap e123: 8 total, 7 up, 8 in 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: failed to encode map e123 with expected crc 2023-10-09T06:14:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2905/220827 objects degraded (1.316%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[212750]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.403 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:53 smithi107 bash[263284]: 4f4ef8822b1faf5119fc77cd5806cb4ec70605cd1bf784615ac267844b7566b8 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:53 smithi107 ceph-mon[220921]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: osdmap e124: 8 total, 7 up, 8 in 2023-10-09T06:14:53.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:53.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:53 smithi150 ceph-mon[194178]: failed to encode map e124 with expected crc 2023-10-09T06:14:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:54 smithi107 ceph-mon[212750]: pgmap v398: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2691/216927 objects degraded (1.241%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2279641804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:54.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:54 smithi107 ceph-mon[220921]: pgmap v398: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2691/216927 objects degraded (1.241%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:54.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2279641804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:54.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:54 smithi150 ceph-mon[194178]: pgmap v398: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 2691/216927 objects degraded (1.241%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:14:54.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2279641804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:56 smithi107 ceph-mon[212750]: pgmap v399: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+undersized, 69 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 19767/212733 objects degraded (9.292%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T06:14:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:56 smithi107 ceph-mon[220921]: pgmap v399: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+undersized, 69 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 19767/212733 objects degraded (9.292%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T06:14:56.401 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:55 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T06:14:56.402 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:55 smithi107 systemd[1]: Stopped Ceph osd.2 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:56.402 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:56 smithi107 systemd[1]: Starting Ceph osd.2 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:14:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:56 smithi150 ceph-mon[194178]: pgmap v399: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+undersized, 69 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 19767/212733 objects degraded (9.292%); 7.1 MiB/s, 113 objects/s recovering 2023-10-09T06:14:57.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/435072965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:57.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19767/212733 objects degraded (9.292%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:57.555 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/435072965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:57.555 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19767/212733 objects degraded (9.292%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:57.555 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:14:57.555 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T06:14:57.555 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T06:14:57.555 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T06:14:57.555 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T06:14:57.556 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T06:14:57.556 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T06:14:57.556 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:57 smithi107 bash[263935]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T06:14:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/435072965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19767/212733 objects degraded (9.292%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:14:58.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:58 smithi107 ceph-mon[220921]: pgmap v400: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28392/210411 objects degraded (13.494%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:14:58.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:58 smithi107 ceph-mon[212750]: pgmap v400: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28392/210411 objects degraded (13.494%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:14:58.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:58 smithi150 ceph-mon[194178]: pgmap v400: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28392/210411 objects degraded (13.494%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:14:59.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:14:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/611356024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:59.651 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:59 smithi107 bash[264345]: eea84a85a04b98a7842bc47d2086af7d3dd3bb2e4ce1aaff367bc837803bb3e0 2023-10-09T06:14:59.651 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:59 smithi107 ceph-osd[264409]: -- 172.21.15.107:0/3693366512 <== mon.2 v2:172.21.15.107:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559dfaafe4e0 con 0x559dfaad1400 2023-10-09T06:14:59.651 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:59 smithi107 systemd[1]: Started Ceph osd.2 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:14:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:14:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/611356024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:14:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:14:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/611356024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:00.401 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:14:59 smithi107 conmon[264385]: 2023-10-09T06:14:59.969+0000 7f0aefe75640 -1 Falling back to public interface 2023-10-09T06:15:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:00 smithi150 ceph-mon[194178]: pgmap v401: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27328/203643 objects degraded (13.420%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[212750]: pgmap v401: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27328/203643 objects degraded (13.420%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[220921]: pgmap v401: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 27328/203643 objects degraded (13.420%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:00.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:01.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/684819352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:01.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/684819352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:01.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/684819352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:02] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.43.0" 2023-10-09T06:15:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:02 smithi150 ceph-mon[194178]: pgmap v402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 27054/201408 objects degraded (13.432%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 27054/201408 objects degraded (13.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[212750]: pgmap v402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 27054/201408 objects degraded (13.432%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:02.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:02.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 27054/201408 objects degraded (13.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:02.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[220921]: pgmap v402: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 27054/201408 objects degraded (13.432%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:15:02.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:02.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:02.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 27054/201408 objects degraded (13.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:03 smithi107 ceph-mon[212750]: pgmap v403: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26538/196194 objects degraded (13.526%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:15:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:03 smithi107 ceph-mon[220921]: pgmap v403: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26538/196194 objects degraded (13.526%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:15:03.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:03 smithi150 ceph-mon[194178]: pgmap v403: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 26538/196194 objects degraded (13.526%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:15:04.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/785886108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:04.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:04.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/785886108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/785886108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:04.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:05 smithi150 ceph-mon[194178]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 25238/189573 objects degraded (13.313%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:05.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:05 smithi107 ceph-mon[212750]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 25238/189573 objects degraded (13.313%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:05.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:05 smithi107 ceph-mon[220921]: pgmap v404: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 25238/189573 objects degraded (13.313%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1555621812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1555621812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1555621812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:07.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T06:15:07.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 24726/186561 objects degraded (13.254%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 25238/189573 objects degraded (13.313%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:07.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 24726/186561 objects degraded (13.254%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:07.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 25238/189573 objects degraded (13.313%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: osdmap e125: 8 total, 7 up, 8 in 2023-10-09T06:15:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: failed to encode map e125 with expected crc 2023-10-09T06:15:07.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: pgmap v406: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 63 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 24726/186561 objects degraded (13.254%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:07.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 25238/189573 objects degraded (13.313%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:15:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[212750]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T06:15:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[212750]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:08 smithi150 ceph-mon[194178]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:08 smithi150 ceph-mon[194178]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:08 smithi150 ceph-mon[194178]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T06:15:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:08 smithi150 ceph-mon[194178]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[220921]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[220921]: failed to encode map e126 with expected crc 2023-10-09T06:15:08.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[220921]: osdmap e126: 8 total, 7 up, 8 in 2023-10-09T06:15:08.909 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:08 smithi107 ceph-mon[220921]: failed to encode map e126 with expected crc 2023-10-09T06:15:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2851762454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:09 smithi107 ceph-mon[212750]: pgmap v408: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 23743/181011 objects degraded (13.117%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:15:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2851762454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:09 smithi107 ceph-mon[220921]: pgmap v408: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 23743/181011 objects degraded (13.117%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:15:09.903 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:15:09 smithi107 conmon[264385]: 2023-10-09T06:15:09.747+0000 7f0aefe75640 -1 osd.2 122 log_to_monitors true 2023-10-09T06:15:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2851762454' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:09 smithi150 ceph-mon[194178]: pgmap v408: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 23743/181011 objects degraded (13.117%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:15:10.902 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:15:10 smithi107 conmon[264385]: 2023-10-09T06:15:10.712+0000 7f0ae0ae0700 -1 osd.2 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:15:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:10 smithi107 ceph-mon[212750]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:10 smithi107 ceph-mon[212750]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:10.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:10 smithi150 ceph-mon[194178]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:10 smithi150 ceph-mon[194178]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:10.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:10 smithi107 ceph-mon[220921]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:10.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:10 smithi107 ceph-mon[220921]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: osdmap e127: 8 total, 7 up, 8 in 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: pgmap v410: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 23556/179271 objects degraded (13.140%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/29940881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: osdmap e127: 8 total, 7 up, 8 in 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: pgmap v410: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 23556/179271 objects degraded (13.140%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:15:12.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/29940881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T06:15:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: osdmap e127: 8 total, 7 up, 8 in 2023-10-09T06:15:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: from='osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: failed to encode map e127 with expected crc 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: pgmap v410: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 64 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 23556/179271 objects degraded (13.140%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:15:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/29940881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:12] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.43.0" 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714] boot 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 23556/179271 objects degraded (13.140%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714] boot 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:15:13.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 23556/179271 objects degraded (13.140%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:12 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:15:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: osd.2 [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714] boot 2023-10-09T06:15:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T06:15:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T06:15:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e128 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 23556/179271 objects degraded (13.140%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:12 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: pgmap v413: 89 pgs: 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 70 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 23148/175221 objects degraded (13.211%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e130 with expected crc 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: failed to encode map e130 with expected crc 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1628557668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:15:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e129 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: pgmap v413: 89 pgs: 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 70 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 23148/175221 objects degraded (13.211%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T06:15:13.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1628557668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:13.905 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:13 smithi107 conmon[200038]: 2023-10-09T06:15:13.770+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e129 with expected crc 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: pgmap v413: 89 pgs: 7 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 70 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 23148/175221 objects degraded (13.211%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T06:15:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e130 with expected crc 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: failed to encode map e130 with expected crc 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1628557668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:15:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T06:15:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:14 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T06:15:15.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: failed to encode map e130 with expected crc 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:15.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:14 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-09T06:15:16.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:15 smithi107 ceph-mon[212750]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11504/170961 objects degraded (6.729%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:16.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:15 smithi107 ceph-mon[220921]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11504/170961 objects degraded (6.729%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:15 smithi150 ceph-mon[194178]: pgmap v415: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 11504/170961 objects degraded (6.729%); 7.8 MiB/s, 125 objects/s recovering 2023-10-09T06:15:17.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3509942038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:17.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:17.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:17.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3509942038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:17.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:17.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:17.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3509942038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:17.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:15:17.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:17 smithi107 ceph-mon[212750]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 6186/169161 objects degraded (3.657%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:15:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11504/170961 objects degraded (6.729%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:17 smithi107 ceph-mon[220921]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 6186/169161 objects degraded (3.657%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:15:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11504/170961 objects degraded (6.729%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:17 smithi150 ceph-mon[194178]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 6186/169161 objects degraded (3.657%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:15:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11504/170961 objects degraded (6.729%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:19.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4077516981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4077516981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4077516981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:19 smithi107 ceph-mon[212750]: pgmap v417: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5859/163854 objects degraded (3.576%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:15:20.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:19 smithi107 ceph-mon[220921]: pgmap v417: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5859/163854 objects degraded (3.576%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:15:20.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:19 smithi150 ceph-mon[194178]: pgmap v417: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5859/163854 objects degraded (3.576%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:15:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[220921]: pgmap v418: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 5859/161577 objects degraded (3.626%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:22.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/89008218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5859/161577 objects degraded (3.626%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:22.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:22] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.43.0" 2023-10-09T06:15:22.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[212750]: pgmap v418: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 5859/161577 objects degraded (3.626%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:22.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/89008218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:22.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:21 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5859/161577 objects degraded (3.626%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:22.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:21 smithi150 ceph-mon[194178]: pgmap v418: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 5859/161577 objects degraded (3.626%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T06:15:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/89008218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:21 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5859/161577 objects degraded (3.626%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:24.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:23 smithi107 ceph-mon[212750]: pgmap v419: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5859/156504 objects degraded (3.744%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T06:15:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/964292979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:23 smithi107 ceph-mon[220921]: pgmap v419: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5859/156504 objects degraded (3.744%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T06:15:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/964292979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:23 smithi150 ceph-mon[194178]: pgmap v419: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5859/156504 objects degraded (3.744%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T06:15:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/964292979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[212750]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:24 smithi107 ceph-mon[220921]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: osdmap e131: 8 total, 8 up, 8 in 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:24 smithi150 ceph-mon[194178]: failed to encode map e131 with expected crc 2023-10-09T06:15:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[212750]: pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4616/150150 objects degraded (3.074%); 3.2 MiB/s, 83 objects/s recovering 2023-10-09T06:15:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T06:15:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[212750]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:15:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[212750]: failed to encode map e132 with expected crc 2023-10-09T06:15:26.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[220921]: pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4616/150150 objects degraded (3.074%); 3.2 MiB/s, 83 objects/s recovering 2023-10-09T06:15:26.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[220921]: failed to encode map e132 with expected crc 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[220921]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:25 smithi107 ceph-mon[220921]: failed to encode map e132 with expected crc 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:25 smithi150 ceph-mon[194178]: pgmap v421: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 4616/150150 objects degraded (3.074%); 3.2 MiB/s, 83 objects/s recovering 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:25 smithi150 ceph-mon[194178]: failed to encode map e132 with expected crc 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:25 smithi150 ceph-mon[194178]: osdmap e132: 8 total, 8 up, 8 in 2023-10-09T06:15:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:25 smithi150 ceph-mon[194178]: failed to encode map e132 with expected crc 2023-10-09T06:15:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1289964200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:26 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4616/150150 objects degraded (3.074%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1289964200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:27.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:26 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4616/150150 objects degraded (3.074%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1289964200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:26 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4616/150150 objects degraded (3.074%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:15:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:27 smithi107 ceph-mon[220921]: pgmap v423: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4616/147564 objects degraded (3.128%); 1.5 MiB/s, 63 objects/s recovering 2023-10-09T06:15:28.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:27 smithi107 ceph-mon[212750]: pgmap v423: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4616/147564 objects degraded (3.128%); 1.5 MiB/s, 63 objects/s recovering 2023-10-09T06:15:28.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:27 smithi150 ceph-mon[194178]: pgmap v423: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4616/147564 objects degraded (3.128%); 1.5 MiB/s, 63 objects/s recovering 2023-10-09T06:15:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4252969843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4252969843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:29.154 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:28 smithi107 conmon[200038]: 2023-10-09T06:15:28.785+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:15:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4252969843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:15:30.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[220921]: pgmap v424: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 4596/142014 objects degraded (3.236%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T06:15:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:15:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:30 smithi107 ceph-mon[212750]: pgmap v424: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 4596/142014 objects degraded (3.236%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T06:15:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:30 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:30 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T06:15:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:30 smithi150 ceph-mon[194178]: pgmap v424: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 4596/142014 objects degraded (3.236%); 1.5 MiB/s, 65 objects/s recovering 2023-10-09T06:15:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[220921]: pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 4596/140091 objects degraded (3.281%); 1.5 MiB/s, 120 objects/s recovering 2023-10-09T06:15:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/638702253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4596/140091 objects degraded (3.281%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:32.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:32] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-10-09T06:15:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:32 smithi150 ceph-mon[194178]: pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 4596/140091 objects degraded (3.281%); 1.5 MiB/s, 120 objects/s recovering 2023-10-09T06:15:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/638702253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4596/140091 objects degraded (3.281%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:32.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[212750]: pgmap v425: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 4596/140091 objects degraded (3.281%); 1.5 MiB/s, 120 objects/s recovering 2023-10-09T06:15:32.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/638702253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:32.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:32.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4596/140091 objects degraded (3.281%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:34.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:34 smithi107 ceph-mon[212750]: pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4596/134943 objects degraded (3.406%); 0 B/s, 69 objects/s recovering 2023-10-09T06:15:34.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3499718406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:34.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:34 smithi107 ceph-mon[220921]: pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4596/134943 objects degraded (3.406%); 0 B/s, 69 objects/s recovering 2023-10-09T06:15:34.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3499718406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:34 smithi150 ceph-mon[194178]: pgmap v426: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4596/134943 objects degraded (3.406%); 0 B/s, 69 objects/s recovering 2023-10-09T06:15:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3499718406' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[220921]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:15:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[220921]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[220921]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[220921]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[220921]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:35 smithi150 ceph-mon[194178]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:35 smithi150 ceph-mon[194178]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:35 smithi150 ceph-mon[194178]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:35 smithi150 ceph-mon[194178]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:35 smithi150 ceph-mon[194178]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[212750]: osdmap e133: 8 total, 8 up, 8 in 2023-10-09T06:15:35.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T06:15:35.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:35 smithi107 ceph-mon[212750]: failed to encode map e133 with expected crc 2023-10-09T06:15:36.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: pgmap v428: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4596/128472 objects degraded (3.577%); 0 B/s, 76 objects/s recovering 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[212750]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: pgmap v428: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4596/128472 objects degraded (3.577%); 0 B/s, 76 objects/s recovering 2023-10-09T06:15:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:15:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:36 smithi107 ceph-mon[220921]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: pgmap v428: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4596/128472 objects degraded (3.577%); 0 B/s, 76 objects/s recovering 2023-10-09T06:15:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: osdmap e134: 8 total, 8 up, 8 in 2023-10-09T06:15:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: failed to encode map e134 with expected crc 2023-10-09T06:15:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:36 smithi150 ceph-mon[194178]: failed to encode map e134 with expected crc 2023-10-09T06:15:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1117144564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4596/128472 objects degraded (3.577%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:37.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1117144564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:37.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4596/128472 objects degraded (3.577%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1117144564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4596/128472 objects degraded (3.577%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:38 smithi107 ceph-mon[220921]: pgmap v430: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4596/125883 objects degraded (3.651%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:38.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:38 smithi107 ceph-mon[212750]: pgmap v430: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4596/125883 objects degraded (3.651%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:38 smithi150 ceph-mon[194178]: pgmap v430: 89 pgs: 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4596/125883 objects degraded (3.651%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1545992854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1545992854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:39.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1545992854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:40.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:40 smithi107 ceph-mon[220921]: pgmap v431: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 3762/120051 objects degraded (3.134%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:40.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:40 smithi107 ceph-mon[212750]: pgmap v431: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 3762/120051 objects degraded (3.134%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:40.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:40 smithi150 ceph-mon[194178]: pgmap v431: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 3762/120051 objects degraded (3.134%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:42.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[212750]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 3762/117936 objects degraded (3.190%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3900938503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:42.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3762/117936 objects degraded (3.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:42] "GET /metrics HTTP/1.1" 200 36293 "" "Prometheus/2.43.0" 2023-10-09T06:15:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[220921]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 3762/117936 objects degraded (3.190%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3900938503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3762/117936 objects degraded (3.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:42.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:42 smithi150 ceph-mon[194178]: pgmap v432: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 3762/117936 objects degraded (3.190%); 0 B/s, 92 objects/s recovering 2023-10-09T06:15:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3900938503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3762/117936 objects degraded (3.190%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:44.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:43 smithi107 conmon[200038]: 2023-10-09T06:15:43.809+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:15:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[212750]: pgmap v433: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3762/112803 objects degraded (3.335%); 0 B/s, 84 objects/s recovering 2023-10-09T06:15:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3065640695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[220921]: pgmap v433: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3762/112803 objects degraded (3.335%); 0 B/s, 84 objects/s recovering 2023-10-09T06:15:44.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3065640695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:44 smithi150 ceph-mon[194178]: pgmap v433: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3762/112803 objects degraded (3.335%); 0 B/s, 84 objects/s recovering 2023-10-09T06:15:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3065640695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:45.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:45.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:15:45.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:15:45.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:45 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:45 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: failed to encode map e135 with expected crc 2023-10-09T06:15:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3762/106296 objects degraded (3.539%); 0 B/s, 80 objects/s recovering 2023-10-09T06:15:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: failed to encode map e135 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3762/106296 objects degraded (3.539%); 0 B/s, 80 objects/s recovering 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:15:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:46 smithi107 ceph-mon[220921]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: failed to encode map e135 with expected crc 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: pgmap v435: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3762/106296 objects degraded (3.539%); 0 B/s, 80 objects/s recovering 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: failed to encode map e136 with expected crc 2023-10-09T06:15:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:46 smithi150 ceph-mon[194178]: failed to encode map e136 with expected crc 2023-10-09T06:15:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[212750]: failed to encode map e136 with expected crc 2023-10-09T06:15:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1001025618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:47.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3762/106296 objects degraded (3.539%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:47.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[220921]: failed to encode map e136 with expected crc 2023-10-09T06:15:47.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:47.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1001025618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:47.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3762/106296 objects degraded (3.539%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:47 smithi150 ceph-mon[194178]: failed to encode map e136 with expected crc 2023-10-09T06:15:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:15:47.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1001025618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:47.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3762/106296 objects degraded (3.539%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:48.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:48 smithi107 ceph-mon[212750]: pgmap v437: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2800/103827 objects degraded (2.697%); 0 B/s, 94 objects/s recovering 2023-10-09T06:15:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:48 smithi107 ceph-mon[220921]: pgmap v437: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2800/103827 objects degraded (2.697%); 0 B/s, 94 objects/s recovering 2023-10-09T06:15:48.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:48 smithi150 ceph-mon[194178]: pgmap v437: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 2800/103827 objects degraded (2.697%); 0 B/s, 94 objects/s recovering 2023-10-09T06:15:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/149499192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:49.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/149499192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:49.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/149499192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:50.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:50 smithi107 ceph-mon[212750]: pgmap v438: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2800/98505 objects degraded (2.842%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:50.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:50 smithi107 ceph-mon[220921]: pgmap v438: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2800/98505 objects degraded (2.842%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:50 smithi150 ceph-mon[194178]: pgmap v438: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2800/98505 objects degraded (2.842%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:52.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:15:52] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-10-09T06:15:52.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[212750]: pgmap v439: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2800/96525 objects degraded (2.901%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:52.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/225416808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:52.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2800/96525 objects degraded (2.901%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[220921]: pgmap v439: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2800/96525 objects degraded (2.901%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/225416808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:52.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2800/96525 objects degraded (2.901%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:52 smithi150 ceph-mon[194178]: pgmap v439: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2800/96525 objects degraded (2.901%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/225416808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2800/96525 objects degraded (2.901%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2800/91281 objects degraded (3.067%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2313594612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2800/91281 objects degraded (3.067%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2313594612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: pgmap v440: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 2800/91281 objects degraded (3.067%); 0 B/s, 100 objects/s recovering 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: failed to encode map e137 with expected crc 2023-10-09T06:15:54.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2313594612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:55.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[220921]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[220921]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[220921]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[220921]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[212750]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:55 smithi107 ceph-mon[212750]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:55 smithi150 ceph-mon[194178]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:55 smithi150 ceph-mon[194178]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T06:15:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:55 smithi150 ceph-mon[194178]: failed to encode map e138 with expected crc 2023-10-09T06:15:55.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:55 smithi150 ceph-mon[194178]: failed to encode map e138 with expected crc 2023-10-09T06:15:56.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:56 smithi107 ceph-mon[220921]: pgmap v443: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1850/85035 objects degraded (2.176%); 0 B/s, 67 objects/s recovering 2023-10-09T06:15:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:56 smithi150 ceph-mon[194178]: pgmap v443: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1850/85035 objects degraded (2.176%); 0 B/s, 67 objects/s recovering 2023-10-09T06:15:56.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:56 smithi107 ceph-mon[212750]: pgmap v443: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 1850/85035 objects degraded (2.176%); 0 B/s, 67 objects/s recovering 2023-10-09T06:15:57.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2852636188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:57.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1850/85035 objects degraded (2.176%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:57.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2852636188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:57.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1850/85035 objects degraded (2.176%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:57.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2852636188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:57.662 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1850/85035 objects degraded (2.176%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:15:58.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:58 smithi107 ceph-mon[220921]: pgmap v444: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 1850/82455 objects degraded (2.244%); 0 B/s, 78 objects/s recovering 2023-10-09T06:15:58.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:58 smithi107 ceph-mon[212750]: pgmap v444: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 1850/82455 objects degraded (2.244%); 0 B/s, 78 objects/s recovering 2023-10-09T06:15:58.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:58 smithi150 ceph-mon[194178]: pgmap v444: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 1850/82455 objects degraded (2.244%); 0 B/s, 78 objects/s recovering 2023-10-09T06:15:59.154 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:15:58 smithi107 conmon[200038]: 2023-10-09T06:15:58.836+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:15:59.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:15:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/990626313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:15:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/990626313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:15:59.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:15:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:15:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/990626313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:00.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:00.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[220921]: pgmap v445: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 1850/76977 objects degraded (2.403%); 0 B/s, 148 objects/s recovering 2023-10-09T06:16:00.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:16:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:00 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:00 smithi150 ceph-mon[194178]: pgmap v445: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 1850/76977 objects degraded (2.403%); 0 B/s, 148 objects/s recovering 2023-10-09T06:16:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:00 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:16:00.666 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:00.667 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[212750]: pgmap v445: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 1850/76977 objects degraded (2.403%); 0 B/s, 148 objects/s recovering 2023-10-09T06:16:00.667 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:00 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T06:16:02.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[212750]: pgmap v446: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1850/74658 objects degraded (2.478%); 0 B/s, 96 objects/s recovering 2023-10-09T06:16:02.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:02.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3859283498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:02.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1850/74658 objects degraded (2.478%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:02.367 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:02] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.43.0" 2023-10-09T06:16:02.367 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[220921]: pgmap v446: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1850/74658 objects degraded (2.478%); 0 B/s, 96 objects/s recovering 2023-10-09T06:16:02.367 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:02.367 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3859283498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:02.367 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1850/74658 objects degraded (2.478%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:02.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:02 smithi150 ceph-mon[194178]: pgmap v446: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 1850/74658 objects degraded (2.478%); 0 B/s, 96 objects/s recovering 2023-10-09T06:16:02.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:02.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3859283498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:02.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1850/74658 objects degraded (2.478%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1850/69705 objects degraded (2.654%); 0 B/s, 80 objects/s recovering 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[212750]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1850/69705 objects degraded (2.654%); 0 B/s, 80 objects/s recovering 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:03 smithi107 ceph-mon[220921]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: pgmap v447: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1850/69705 objects degraded (2.654%); 0 B/s, 80 objects/s recovering 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: osdmap e139: 8 total, 8 up, 8 in 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: failed to encode map e139 with expected crc 2023-10-09T06:16:04.412 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:03 smithi150 ceph-mon[194178]: failed to encode map e139 with expected crc 2023-10-09T06:16:05.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:16:05.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1388396891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1388396891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: osdmap e140: 8 total, 8 up, 8 in 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: failed to encode map e140 with expected crc 2023-10-09T06:16:05.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1388396891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:05 smithi107 ceph-mon[220921]: pgmap v450: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 911/63603 objects degraded (1.432%); 0 B/s, 113 objects/s recovering 2023-10-09T06:16:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:05 smithi107 ceph-mon[212750]: pgmap v450: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 911/63603 objects degraded (1.432%); 0 B/s, 113 objects/s recovering 2023-10-09T06:16:06.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:05 smithi150 ceph-mon[194178]: pgmap v450: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 911/63603 objects degraded (1.432%); 0 B/s, 113 objects/s recovering 2023-10-09T06:16:07.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2738663820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:07.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 911/63603 objects degraded (1.432%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:07.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2738663820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:07.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 911/63603 objects degraded (1.432%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:07.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2738663820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:07.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 911/63603 objects degraded (1.432%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:07 smithi107 ceph-mon[212750]: pgmap v451: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 911/61152 objects degraded (1.490%); 0 B/s, 44 objects/s recovering 2023-10-09T06:16:08.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:07 smithi150 ceph-mon[194178]: pgmap v451: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 911/61152 objects degraded (1.490%); 0 B/s, 44 objects/s recovering 2023-10-09T06:16:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:07 smithi107 ceph-mon[220921]: pgmap v451: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 911/61152 objects degraded (1.490%); 0 B/s, 44 objects/s recovering 2023-10-09T06:16:10.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:09 smithi107 ceph-mon[212750]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 911/55428 objects degraded (1.644%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:10.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2848670957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:09 smithi107 ceph-mon[220921]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 911/55428 objects degraded (1.644%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2848670957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:10.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:09 smithi150 ceph-mon[194178]: pgmap v452: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 911/55428 objects degraded (1.644%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:10.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2848670957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:12.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:12] "GET /metrics HTTP/1.1" 200 36276 "" "Prometheus/2.43.0" 2023-10-09T06:16:12.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[212750]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 911/53178 objects degraded (1.713%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:12.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2595479657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:12.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 911/53178 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:12.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[220921]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 911/53178 objects degraded (1.713%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:12.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2595479657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:12.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:11 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 911/53178 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:11 smithi150 ceph-mon[194178]: pgmap v453: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 911/53178 objects degraded (1.713%); 0 B/s, 106 objects/s recovering 2023-10-09T06:16:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2595479657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:11 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 911/53178 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:14.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:13 smithi107 ceph-mon[212750]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 911/47916 objects degraded (1.901%); 0 B/s, 85 objects/s recovering 2023-10-09T06:16:14.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:13 smithi107 ceph-mon[220921]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 911/47916 objects degraded (1.901%); 0 B/s, 85 objects/s recovering 2023-10-09T06:16:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:14.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:13 smithi150 ceph-mon[194178]: pgmap v454: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 997 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 911/47916 objects degraded (1.901%); 0 B/s, 85 objects/s recovering 2023-10-09T06:16:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3801111754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3801111754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:15.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: osdmap e141: 8 total, 8 up, 8 in 2023-10-09T06:16:15.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: failed to encode map e141 with expected crc 2023-10-09T06:16:15.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3801111754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:16.280 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 864 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 911/41529 objects degraded (2.194%); 0 B/s, 84 objects/s recovering 2023-10-09T06:16:16.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:16:16.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:15 smithi150 ceph-mon[194178]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 864 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 911/41529 objects degraded (2.194%); 0 B/s, 84 objects/s recovering 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[212750]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: pgmap v456: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 864 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 911/41529 objects degraded (2.194%); 0 B/s, 84 objects/s recovering 2023-10-09T06:16:16.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T06:16:16.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: failed to encode map e142 with expected crc 2023-10-09T06:16:16.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:15 smithi107 ceph-mon[220921]: failed to encode map e142 with expected crc 2023-10-09T06:16:17.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 911/41529 objects degraded (2.194%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:16:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2378158585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 911/41529 objects degraded (2.194%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:16:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2378158585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:17.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 911/41529 objects degraded (2.194%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:16:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2378158585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:18 smithi107 ceph-mon[220921]: pgmap v458: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 817 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:18.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:18 smithi107 ceph-mon[212750]: pgmap v458: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 817 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:18.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:18 smithi150 ceph-mon[194178]: pgmap v458: 89 pgs: 2 active+recovering+undersized+remapped, 87 active+clean; 817 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:19.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:19 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:19.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:19 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:19.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:19 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:19.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:19 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:19.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:19 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:19.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:19 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[220921]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 709 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:20.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1756006646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:20.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[212750]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 709 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1756006646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:20 smithi150 ceph-mon[194178]: pgmap v459: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 709 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:20.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1756006646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:20.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:20.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:21.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:21.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:21.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:21.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:21.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:21.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:22.278 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:22] "GET /metrics HTTP/1.1" 200 36278 "" "Prometheus/2.43.0" 2023-10-09T06:16:22.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[220921]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:22.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:22.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[220921]: Upgrade: osd.3 is safe to restart 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2114376585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:22 smithi150 ceph-mon[194178]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:22 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:22 smithi150 ceph-mon[194178]: Upgrade: osd.3 is safe to restart 2023-10-09T06:16:22.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2114376585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[212750]: pgmap v460: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 670 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 0 B/s, 100 objects/s recovering 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[212750]: Upgrade: osd.3 is safe to restart 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:16:22.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2114376585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:24.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 569 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[220921]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 569 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:24 smithi150 ceph-mon[194178]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: pgmap v461: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 569 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 0 B/s, 93 objects/s recovering 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T06:16:24.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:24 smithi107 ceph-mon[212750]: failed to encode map e143 with expected crc 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3896378941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: Upgrade: Updating osd.3 2023-10-09T06:16:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[220921]: Deploying daemon osd.3 on smithi107 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3896378941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: Upgrade: Updating osd.3 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:25.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:25 smithi107 ceph-mon[212750]: Deploying daemon osd.3 on smithi107 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3896378941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: failed to encode map e144 with expected crc 2023-10-09T06:16:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: Upgrade: Updating osd.3 2023-10-09T06:16:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T06:16:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:25 smithi150 ceph-mon[194178]: Deploying daemon osd.3 on smithi107 2023-10-09T06:16:26.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:26 smithi107 ceph-mon[212750]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 0 B/s, 53 objects/s recovering 2023-10-09T06:16:26.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:26 smithi107 ceph-mon[220921]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 0 B/s, 53 objects/s recovering 2023-10-09T06:16:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:26 smithi150 ceph-mon[194178]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 449 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 0 B/s, 53 objects/s recovering 2023-10-09T06:16:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2813266732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:27.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2813266732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2813266732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:28.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[212750]: pgmap v465: 89 pgs: 89 active+clean; 399 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:28.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:16:28.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[220921]: pgmap v465: 89 pgs: 89 active+clean; 399 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:28.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:16:28.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:16:28.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:28 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:16:28.905 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:28 smithi107 systemd[1]: Stopping Ceph osd.3 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:16:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:28 smithi150 ceph-mon[194178]: pgmap v465: 89 pgs: 89 active+clean; 399 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:28 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:16:28.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:28 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:16:29.404 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:29 smithi107 conmon[152901]: 2023-10-09T06:16:29.047+0000 7f0fc9172700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:16:29.404 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:29 smithi107 conmon[152901]: 2023-10-09T06:16:29.047+0000 7f0fc9172700 -1 osd.3 144 *** Got signal Terminated *** 2023-10-09T06:16:29.404 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:29 smithi107 conmon[152901]: 2023-10-09T06:16:29.048+0000 7f0fc9172700 -1 osd.3 144 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:16:29.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:29 smithi107 ceph-mon[212750]: osd.3 marked itself down and dead 2023-10-09T06:16:29.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/285208135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:29.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:29 smithi107 ceph-mon[220921]: osd.3 marked itself down and dead 2023-10-09T06:16:29.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/285208135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:29 smithi150 ceph-mon[194178]: osd.3 marked itself down and dead 2023-10-09T06:16:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/285208135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: pgmap v466: 89 pgs: 89 active+clean; 291 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: osdmap e145: 8 total, 7 up, 8 in 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[220921]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: pgmap v466: 89 pgs: 89 active+clean; 291 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:16:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: osdmap e145: 8 total, 7 up, 8 in 2023-10-09T06:16:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:30 smithi150 ceph-mon[194178]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: pgmap v466: 89 pgs: 89 active+clean; 291 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 0 B/s, 63 objects/s recovering 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: osdmap e145: 8 total, 7 up, 8 in 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T06:16:30.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:30 smithi107 ceph-mon[212750]: failed to encode map e145 with expected crc 2023-10-09T06:16:31.401 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:30 smithi107 bash[273019]: e1f7d3e8c22d3d083c7f2058cf3e3a5a944326718d50460905f077396530748a 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: osdmap e146: 8 total, 7 up, 8 in 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:31.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: osdmap e146: 8 total, 7 up, 8 in 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: osdmap e146: 8 total, 7 up, 8 in 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: failed to encode map e146 with expected crc 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:31.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:32.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:32] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.43.0" 2023-10-09T06:16:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[212750]: pgmap v469: 89 pgs: 1 active+undersized+degraded, 8 peering, 12 stale+active+clean, 3 active+undersized, 65 active+clean; 245 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 166/11790 objects degraded (1.408%); 0 B/s, 19 objects/s recovering 2023-10-09T06:16:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:16:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 166/11790 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1043633183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[220921]: pgmap v469: 89 pgs: 1 active+undersized+degraded, 8 peering, 12 stale+active+clean, 3 active+undersized, 65 active+clean; 245 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 166/11790 objects degraded (1.408%); 0 B/s, 19 objects/s recovering 2023-10-09T06:16:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:16:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 166/11790 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1043633183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:32 smithi150 ceph-mon[194178]: pgmap v469: 89 pgs: 1 active+undersized+degraded, 8 peering, 12 stale+active+clean, 3 active+undersized, 65 active+clean; 245 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 166/11790 objects degraded (1.408%); 0 B/s, 19 objects/s recovering 2023-10-09T06:16:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:32 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:16:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:32 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 166/11790 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:16:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1043633183' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:33.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 116911 objects 2023-10-09T06:16:33.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :163.664 2023-10-09T06:16:33.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:33 smithi107 ceph-mon[212750]: pgmap v470: 89 pgs: 5 active+undersized+degraded, 8 peering, 2 stale+active+clean, 18 active+undersized, 56 active+clean; 150 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 367/7254 objects degraded (5.059%); 0 B/s, 16 objects/s recovering 2023-10-09T06:16:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:33 smithi107 ceph-mon[220921]: pgmap v470: 89 pgs: 5 active+undersized+degraded, 8 peering, 2 stale+active+clean, 18 active+undersized, 56 active+clean; 150 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 367/7254 objects degraded (5.059%); 0 B/s, 16 objects/s recovering 2023-10-09T06:16:33.901 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:33 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T06:16:33.901 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:33 smithi107 systemd[1]: Stopped Ceph osd.3 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:16:33.901 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:33 smithi107 systemd[1]: Starting Ceph osd.3 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:16:33.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:33 smithi150 ceph-mon[194178]: pgmap v470: 89 pgs: 5 active+undersized+degraded, 8 peering, 2 stale+active+clean, 18 active+undersized, 56 active+clean; 150 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 367/7254 objects degraded (5.059%); 0 B/s, 16 objects/s recovering 2023-10-09T06:16:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3220562627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3220562627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:34.902 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T06:16:34.902 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T06:16:34.902 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T06:16:34.903 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T06:16:34.903 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T06:16:34.903 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T06:16:34.903 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T06:16:34.903 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:34 smithi107 bash[273682]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T06:16:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3220562627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:35.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:35 smithi107 ceph-mon[212750]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 69 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 447/3339 objects degraded (13.387%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:35.896 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:35 smithi107 ceph-mon[220921]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 69 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 447/3339 objects degraded (13.387%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:35.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:35 smithi150 ceph-mon[194178]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 69 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 447/3339 objects degraded (13.387%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:36.377 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T06:16:36.377 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T06:16:36.401 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:35 smithi107 ceph-osd[274203]: -- 172.21.15.107:0/3155419790 <== mon.0 v2:172.21.15.107:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556ab7f4a4e0 con 0x556ab7f1d400 2023-10-09T06:16:36.401 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:35 smithi107 bash[274138]: 069ecca357f7c74ca1a492ed3b0cb931ef40fade26905d11a12565a32983bea2 2023-10-09T06:16:36.402 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:35 smithi107 systemd[1]: Started Ceph osd.3 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:16:36.901 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:36 smithi107 conmon[274179]: 2023-10-09T06:16:36.494+0000 7fbb056b0640 -1 Falling back to public interface 2023-10-09T06:16:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:16:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:36 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:16:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:37.034 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1419534284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[212750]: pgmap v472: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 22 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 145/1107 objects degraded (13.098%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 447/3339 objects degraded (13.387%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1419534284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[220921]: pgmap v472: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 22 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 145/1107 objects degraded (13.098%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 447/3339 objects degraded (13.387%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1419534284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:37 smithi150 ceph-mon[194178]: pgmap v472: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 22 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 145/1107 objects degraded (13.098%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 447/3339 objects degraded (13.387%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:40.012 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_3' removed 2023-10-09T06:16:40.179 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: pgmap v473: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 4/72 objects degraded (5.556%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:40.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1662628357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2847439941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:40.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: pgmap v473: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 4/72 objects degraded (5.556%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:40.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1662628357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2847439941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:40.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.182 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: pgmap v473: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 4/72 objects degraded (5.556%); 0 B/s, 0 objects/s recovering 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1662628357' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2847439941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:40.733 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:16:40.733 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T06:16:40.734 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:41.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: osdmap e147: 8 total, 7 up, 8 in 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: osdmap e147: 8 total, 7 up, 8 in 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.082 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.082 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:41.082 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: osdmap e147: 8 total, 7 up, 8 in 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1412822351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1412822351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:41 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:42.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:42] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.43.0" 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: failed to encode map e147 with expected crc 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1412822351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:41 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:16:42.967 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:42 smithi107 ceph-mon[212750]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:42.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:42 smithi107 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:42.968 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:42 smithi107 ceph-mon[220921]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:42.968 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:42 smithi107 ceph-mon[220921]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:42.968 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:42 smithi107 conmon[274179]: 2023-10-09T06:16:42.701+0000 7fbb056b0640 -1 osd.3 144 log_to_monitors true 2023-10-09T06:16:42.968 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:16:42 smithi107 conmon[274179]: 2023-10-09T06:16:42.966+0000 7fbaf4b18700 -1 osd.3 144 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T06:16:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:42 smithi150 ceph-mon[194178]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:42 smithi150 ceph-mon[194178]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T06:16:44.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:44 smithi107 conmon[200038]: 2023-10-09T06:16:44.148+0000 7f5488b69700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T06:16:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:16:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: osdmap e148: 8 total, 7 up, 8 in 2023-10-09T06:16:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1955781452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:44.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:44.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:16:44.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: osdmap e148: 8 total, 7 up, 8 in 2023-10-09T06:16:44.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1955781452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:44.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:44.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T06:16:44.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: osdmap e148: 8 total, 7 up, 8 in 2023-10-09T06:16:44.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: from='osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi107", "root=default"]}]: dispatch 2023-10-09T06:16:44.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: failed to encode map e148 with expected crc 2023-10-09T06:16:44.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1955781452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:16:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119] boot 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[212750]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119] boot 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:16:45.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:44 smithi107 ceph-mon[220921]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: failed to encode map e149 with expected crc 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: osd.3 [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119] boot 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: osdmap e149: 8 total, 8 up, 8 in 2023-10-09T06:16:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:45.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T06:16:45.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T06:16:45.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:44 smithi150 ceph-mon[194178]: osdmap e150: 8 total, 8 up, 8 in 2023-10-09T06:16:46.002 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T06:16:46.008 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:16:45 smithi150 conmon[174357]: logger=sqlstore.transactions t=2023-10-09T06:16:45.663889838Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: pgmap v479: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4037356593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: pgmap v479: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4037356593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: pgmap v479: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: failed to encode map e150 with expected crc 2023-10-09T06:16:46.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4037356593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T06:16:46.658 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T06:16:47.039 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:47.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:47.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:16:48.563 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:48 smithi107 ceph-mon[220921]: pgmap v481: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:48.701 INFO:teuthology.orchestra.run.smithi107.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:16:48.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:48 smithi107 ceph-mon[212750]: pgmap v481: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:48 smithi150 ceph-mon[194178]: pgmap v481: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T06:16:49.276 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:16:49.620 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[212750]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[212750]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[212750]: pgmap v482: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[220921]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[220921]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:16:50.033 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:50.034 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:49 smithi107 ceph-mon[220921]: pgmap v482: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:49 smithi150 ceph-mon[194178]: from='client.35136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:49 smithi150 ceph-mon[194178]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T06:16:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:49 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:50.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:49 smithi150 ceph-mon[194178]: pgmap v482: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:50.870 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:50 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T06:16:50.870 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:50 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:16:50.870 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:50 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T06:16:50.870 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:50 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:16:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:50 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-10-09T06:16:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:50 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:16:51.269 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (15m) - 19m 29.6M - 0.25.0 c8568f914cd2 0715bcb46fa1 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (15m) - 19m 92.7M - 9.4.7 2c41d148cca3 96db62cb38d0 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (18m) - 18m 25.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (18m) - 18m 17.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443,9283,8765 running (14m) - 23m 455M - 18.2.0-598-gb4fa663f 3060897fdf97 7326a92d4fbc 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:8443,9283,8765 running (14m) - 25m 525M - 18.2.0-598-gb4fa663f 3060897fdf97 19d12e0e339c 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (13m) - 25m 121M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7f66f5126e5d 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (10m) - 24m 75.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 de70d3643920 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (12m) - 24m 80.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 6c90cc5e902c 2023-10-09T06:16:51.270 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (15m) - 19m 20.9M - 1.5.0 0da6a335fe13 32eddd377f02 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (15m) - 19m 17.0M - 1.5.0 0da6a335fe13 6b4c6b51266f 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (8m) - 23m 3870M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 f47405cf3d52 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (3m) - 23m 2529M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 a3526ce24032 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (112s) - 22m 1091M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 eea84a85a04b 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (15s) - 22m 13.9M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 069ecca357f7 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (21m) - 21m 5737M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T06:16:51.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (21m) - 21m 7878M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T06:16:51.272 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (21m) - 21m 5984M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T06:16:51.272 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (20m) - 20m 6559M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T06:16:51.272 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (14m) - 20m 72.0M - 2.43.0 a07b618ecd1d 3a1d67d52b64 2023-10-09T06:16:51.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T06:16:52.013 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:51 smithi107 ceph-mon[212750]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:52.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:51 smithi107 ceph-mon[212750]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:52.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:51 smithi107 ceph-mon[220921]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:52.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:51 smithi107 ceph-mon[220921]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:51 smithi150 ceph-mon[194178]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:51 smithi150 ceph-mon[194178]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:52.195 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:52.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:16:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:16:52] "GET /metrics HTTP/1.1" 200 33774 "" "Prometheus/2.43.0" 2023-10-09T06:16:53.789 INFO:teuthology.orchestra.run.smithi107.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T06:16:53.789 INFO:teuthology.orchestra.run.smithi107.stdout:alertmanager.a smithi107 *:9093,9094 running (15m) - 19m 29.6M - 0.25.0 c8568f914cd2 0715bcb46fa1 2023-10-09T06:16:53.789 INFO:teuthology.orchestra.run.smithi107.stdout:grafana.a smithi150 *:3000 running (15m) - 19m 92.7M - 9.4.7 2c41d148cca3 96db62cb38d0 2023-10-09T06:16:53.789 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi107.dqtsqc smithi107 running (18m) - 18m 25.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 0f353a6e44e5 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mds.foo.smithi150.znbnco smithi150 running (18m) - 18m 17.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 951c62831591 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.x smithi150 *:8443,9283,8765 running (14m) - 23m 455M - 18.2.0-598-gb4fa663f 3060897fdf97 7326a92d4fbc 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mgr.y smithi107 *:8443,9283,8765 running (14m) - 25m 525M - 18.2.0-598-gb4fa663f 3060897fdf97 19d12e0e339c 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a smithi107 running (13m) - 25m 121M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7f66f5126e5d 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b smithi150 running (10m) - 24m 75.3M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 de70d3643920 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c smithi107 running (12m) - 24m 80.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 6c90cc5e902c 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.a smithi107 *:9100 running (15m) - 19m 20.9M - 1.5.0 0da6a335fe13 32eddd377f02 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:node-exporter.b smithi150 *:9100 running (15m) - 19m 17.0M - 1.5.0 0da6a335fe13 6b4c6b51266f 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 smithi107 running (8m) - 23m 3870M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 f47405cf3d52 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 smithi107 running (4m) - 23m 2529M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 a3526ce24032 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 smithi107 running (114s) - 22m 1091M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 eea84a85a04b 2023-10-09T06:16:53.790 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 smithi107 running (18s) - 22m 13.9M 2500M 18.2.0-598-gb4fa663f 3060897fdf97 069ecca357f7 2023-10-09T06:16:53.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 smithi150 running (21m) - 21m 5737M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 08e858ffb7ce 2023-10-09T06:16:53.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 smithi150 running (21m) - 21m 7878M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 7c00b8a830f4 2023-10-09T06:16:53.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 smithi150 running (21m) - 21m 5984M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 fbed7d32aa54 2023-10-09T06:16:53.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 smithi150 running (20m) - 20m 6559M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 5e4873826583 2023-10-09T06:16:53.791 INFO:teuthology.orchestra.run.smithi107.stdout:prometheus.a smithi150 *:9095 running (14m) - 20m 72.0M - 2.43.0 a07b618ecd1d 3a1d67d52b64 2023-10-09T06:16:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:53 smithi107 ceph-mon[212750]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:53 smithi107 ceph-mon[220921]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:53 smithi150 ceph-mon[194178]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:54.409 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T06:16:54.760 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:54 smithi107 ceph-mon[212750]: from='client.35148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:55.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:54 smithi107 ceph-mon[220921]: from='client.35148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:54 smithi150 ceph-mon[194178]: from='client.35148 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:16:55.984 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:55 smithi107 ceph-mon[212750]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:55.984 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:55 smithi107 ceph-mon[220921]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:55 smithi150 ceph-mon[194178]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:56.438 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:16:56.438 INFO:teuthology.orchestra.run.smithi107.stdout: "mon": { 2023-10-09T06:16:56.438 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr": { 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: "osd": { 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T06:16:56.439 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: "mds": { 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: "overall": { 2023-10-09T06:16:56.440 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T06:16:56.441 INFO:teuthology.orchestra.run.smithi107.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T06:16:56.441 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:16:56.441 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:16:57.042 DEBUG:teuthology.parallel:result is None 2023-10-09T06:16:57.042 INFO:teuthology.run_tasks:Running task print... 2023-10-09T06:16:57.070 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T06:16:57.070 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T06:16:57.086 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T06:16:57.086 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T06:16:57.086 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T06:16:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3117400709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:56 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3117400709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:56 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3117400709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T06:16:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:56 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:16:57.436 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:16:58.366 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:57 smithi107 ceph-mon[220921]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:58.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:57 smithi107 ceph-mon[212750]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:57 smithi150 ceph-mon[194178]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:16:59.097 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:16:59.722 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":35,"active_gid":34098,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6800","nonce":676022303},{"type":"v1","addr":"172.21.15.107:6801","nonce":676022303}]},"active_addr":"172.21.15.107:6801/676022303","active_change":"2023-10-09T06:03:16.414260+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25063,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":77,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2996714037}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":2604857881}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":1783871956}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.107:0","nonce":3149380736}]}]} 2023-10-09T06:16:59.727 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T06:16:59.727 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:16:59.728 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:17:00.071 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/745445409' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/745445409' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:16:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/745445409' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:17:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:17:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:17:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:16:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:01.674 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:01.674 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":150,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T06:16:44.938136+0000","last_up_change":"2023-10-09T06:16:43.936496+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":149,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6803","nonce":2817416168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6805","nonce":2817416168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6809","nonce":2817416168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6807","nonce":2817416168}]},"public_addr":"172.21.15.107:6803/2817416168","cluster_addr":"172.21.15.107:6805/2817416168","heartbeat_back_addr":"172.21.15.107:6809/2817416168","heartbeat_front_addr":"172.21.15.107:6807/2817416168","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":149,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6811","nonce":3281752293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6813","nonce":3281752293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6817","nonce":3281752293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6815","nonce":3281752293}]},"public_addr":"172.21.15.107:6811/3281752293","cluster_addr":"172.21.15.107:6813/3281752293","heartbeat_back_addr":"172.21.15.107:6817/3281752293","heartbeat_front_addr":"172.21.15.107:6815/3281752293","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":128,"up_thru":149,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6819","nonce":2613148714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6821","nonce":2613148714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6825","nonce":2613148714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6823","nonce":2613148714}]},"public_addr":"172.21.15.107:6819/2613148714","cluster_addr":"172.21.15.107:6821/2613148714","heartbeat_back_addr":"172.21.15.107:6825/2613148714","heartbeat_front_addr":"172.21.15.107:6823/2613148714","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":149,"up_thru":149,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6827","nonce":3856205119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6829","nonce":3856205119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6833","nonce":3856205119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6831","nonce":3856205119}]},"public_addr":"172.21.15.107:6827/3856205119","cluster_addr":"172.21.15.107:6829/3856205119","heartbeat_back_addr":"172.21.15.107:6833/3856205119","heartbeat_front_addr":"172.21.15.107:6831/3856205119","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/443194359":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3067456200":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2371556112":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1657740329":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/700020267":"2023-10-10T06:02:08.994675+0000","172.21.15.150:0/3437069098":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6835/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6834/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/2216216350":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/3684818042":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.150:0/3156803825":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/3366444430":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6801/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/2888496999":"2023-10-10T06:00:53.065448+0000","172.21.15.150:0/1037268415":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/643879886":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1608610913":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/2930305484":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1943625242":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/2681598821":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2681598821":"2023-10-10T06:03:16.414020+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:17:02.190 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:17:02.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:17:02.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:02] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-10-09T06:17:02.538 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:02.878 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[212750]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:02.878 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3870678442' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[220921]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:02.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3870678442' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:02 smithi150 ceph-mon[194178]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:17:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3870678442' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:04.015 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:04.016 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":150,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T06:16:44.938136+0000","last_up_change":"2023-10-09T06:16:43.936496+0000","last_in_change":"2023-10-09T05:55:55.284453+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":149,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6803","nonce":2817416168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6805","nonce":2817416168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6809","nonce":2817416168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6807","nonce":2817416168}]},"public_addr":"172.21.15.107:6803/2817416168","cluster_addr":"172.21.15.107:6805/2817416168","heartbeat_back_addr":"172.21.15.107:6809/2817416168","heartbeat_front_addr":"172.21.15.107:6807/2817416168","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":149,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6811","nonce":3281752293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6813","nonce":3281752293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6817","nonce":3281752293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6815","nonce":3281752293}]},"public_addr":"172.21.15.107:6811/3281752293","cluster_addr":"172.21.15.107:6813/3281752293","heartbeat_back_addr":"172.21.15.107:6817/3281752293","heartbeat_front_addr":"172.21.15.107:6815/3281752293","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":128,"up_thru":149,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6819","nonce":2613148714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6821","nonce":2613148714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6825","nonce":2613148714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6823","nonce":2613148714}]},"public_addr":"172.21.15.107:6819/2613148714","cluster_addr":"172.21.15.107:6821/2613148714","heartbeat_back_addr":"172.21.15.107:6825/2613148714","heartbeat_front_addr":"172.21.15.107:6823/2613148714","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":149,"up_thru":149,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6827","nonce":3856205119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6829","nonce":3856205119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6833","nonce":3856205119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6831","nonce":3856205119}]},"public_addr":"172.21.15.107:6827/3856205119","cluster_addr":"172.21.15.107:6829/3856205119","heartbeat_back_addr":"172.21.15.107:6833/3856205119","heartbeat_front_addr":"172.21.15.107:6831/3856205119","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":149,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/443194359":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3067456200":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2371556112":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1657740329":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/700020267":"2023-10-10T06:02:08.994675+0000","172.21.15.150:0/3437069098":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6835/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6834/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/2216216350":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/3684818042":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.150:0/3156803825":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/3366444430":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6801/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/2888496999":"2023-10-10T06:00:53.065448+0000","172.21.15.150:0/1037268415":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/643879886":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1608610913":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/2930305484":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1943625242":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/2681598821":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2681598821":"2023-10-10T06:03:16.414020+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:17:04.639 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:17:04.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:17:04.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:17:04.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:17:04.641 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:17:04.641 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:17:04.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:17:04.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:17:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:04 smithi107 ceph-mon[220921]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3392425157' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:04 smithi150 ceph-mon[194178]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3392425157' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:04.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:04 smithi107 ceph-mon[212750]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:04.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3392425157' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:17:05.813 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:05.922 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:06.091 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:06.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:06 smithi107 ceph-mon[220921]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:06 smithi107 ceph-mon[212750]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:06 smithi150 ceph-mon[194178]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:07.038 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:07.073 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:07.295 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:07.506 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:07.813 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:08.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:08 smithi107 ceph-mon[220921]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:08.832 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:08 smithi107 ceph-mon[212750]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:08 smithi150 ceph-mon[194178]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:10.223 INFO:teuthology.orchestra.run.smithi107.stdout:639950127111 2023-10-09T06:17:10.223 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:17:10.868 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:10 smithi107 ceph-mon[212750]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:10.869 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:10 smithi107 ceph-mon[220921]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:10 smithi150 ceph-mon[194178]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:12.321 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:12] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-10-09T06:17:12.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:12 smithi107 ceph-mon[212750]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:12.857 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:12 smithi107 ceph-mon[220921]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:12 smithi150 ceph-mon[194178]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:13.820 INFO:teuthology.orchestra.run.smithi107.stdout:128849019147 2023-10-09T06:17:13.821 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:17:14.116 INFO:teuthology.orchestra.run.smithi107.stdout:472446402612 2023-10-09T06:17:14.117 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:17:14.723 INFO:teuthology.orchestra.run.smithi107.stdout:549755813914 2023-10-09T06:17:14.723 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:17:14.771 INFO:teuthology.orchestra.run.smithi107.stdout:163208757512 2023-10-09T06:17:14.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:17:14.893 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:14 smithi107 ceph-mon[220921]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:14 smithi107 ceph-mon[212750]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:14 smithi150 ceph-mon[194178]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:15.224 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:16.034 INFO:teuthology.orchestra.run.smithi107.stdout:197568495876 2023-10-09T06:17:16.035 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:17:16.043 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:16.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:16 smithi107 ceph-mon[212750]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:16.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:16 smithi107 ceph-mon[220921]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:16 smithi150 ceph-mon[194178]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:17.829 INFO:teuthology.orchestra.run.smithi107.stdout:231928234239 2023-10-09T06:17:17.830 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:17:18.233 INFO:teuthology.orchestra.run.smithi107.stdout:390842024046 2023-10-09T06:17:18.233 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:17:18.430 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:18.464 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:18.588 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:18.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:18 smithi107 ceph-mon[212750]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:18.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:18 smithi107 ceph-mon[220921]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:18.814 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:18 smithi150 ceph-mon[194178]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:18.924 INFO:teuthology.orchestra.run.smithi107.stdout:128849019148 2023-10-09T06:17:18.967 INFO:teuthology.orchestra.run.smithi107.stdout:639950127112 2023-10-09T06:17:18.981 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:19.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1597753417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:17:19.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1289288021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:17:19.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1597753417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:17:19.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1289288021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:17:19.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1597753417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:17:19.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1289288021' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:17:20.282 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019147 got 128849019148 for osd.4 2023-10-09T06:17:20.283 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:20.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:20 smithi107 ceph-mon[212750]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:20 smithi107 ceph-mon[220921]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:20 smithi150 ceph-mon[194178]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:21.985 INFO:tasks.cephadm.ceph_manager.ceph:need seq 639950127111 got 639950127112 for osd.3 2023-10-09T06:17:21.985 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:22.056 INFO:teuthology.orchestra.run.smithi107.stdout:472446402613 2023-10-09T06:17:22.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:22] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.43.0" 2023-10-09T06:17:22.596 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:22 smithi107 ceph-mon[212750]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:22.596 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3238752646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:17:22.596 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:22 smithi107 ceph-mon[220921]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:22.596 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3238752646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:17:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:22 smithi150 ceph-mon[194178]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3238752646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:17:23.358 INFO:teuthology.orchestra.run.smithi107.stdout:231928234240 2023-10-09T06:17:23.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/875423498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:17:23.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/875423498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:17:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/875423498' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:17:24.386 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234239 got 231928234240 for osd.7 2023-10-09T06:17:24.386 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:24.534 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402612 got 472446402613 for osd.1 2023-10-09T06:17:24.534 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:24.547 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:24.851 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:24 smithi107 ceph-mon[212750]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:24.851 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:24 smithi107 ceph-mon[220921]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:24.889 INFO:teuthology.orchestra.run.smithi107.stdout:197568495878 2023-10-09T06:17:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:24 smithi150 ceph-mon[194178]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:25.208 INFO:teuthology.orchestra.run.smithi107.stdout:549755813916 2023-10-09T06:17:25.456 INFO:teuthology.orchestra.run.smithi107.stdout:163208757514 2023-10-09T06:17:25.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1576193424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:17:25.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/244740376' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:17:25.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2614761295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:17:25.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1576193424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:17:25.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/244740376' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:17:25.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2614761295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:17:25.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1576193424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:17:25.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/244740376' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:17:25.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2614761295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:17:26.334 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495876 got 197568495878 for osd.6 2023-10-09T06:17:26.334 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:26 smithi107 ceph-mon[212750]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:26.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:26 smithi107 ceph-mon[220921]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:26 smithi150 ceph-mon[194178]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:27.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757512 got 163208757514 for osd.5 2023-10-09T06:17:27.143 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:27.179 INFO:tasks.cephadm.ceph_manager.ceph:need seq 549755813914 got 549755813916 for osd.2 2023-10-09T06:17:27.179 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:27.711 INFO:teuthology.orchestra.run.smithi107.stdout:390842024048 2023-10-09T06:17:28.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024046 got 390842024048 for osd.0 2023-10-09T06:17:28.382 DEBUG:teuthology.parallel:result is None 2023-10-09T06:17:28.383 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:17:28.383 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:28 smithi107 ceph-mon[212750]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3795397166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:17:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:28 smithi107 ceph-mon[220921]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:28.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3795397166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:17:28.730 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:28 smithi150 ceph-mon[194178]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3795397166' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:17:30.432 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:30.435 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:17:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:30 smithi107 ceph-mon[212750]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:30 smithi107 ceph-mon[220921]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:30 smithi150 ceph-mon[194178]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:31.007 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":502,"stamp":"2023-10-09T06:17:28.882941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2793024,"kb_used_data":14016,"kb_used_omap":20,"kb_used_meta":2778923,"kb_avail":747168192,"statfs":{"total":767960285184,"available":765100228608,"internally_reserved":0,"allocated":14352384,"data_stored":9550035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2845617806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002510"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943512+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:16:43.943512+0000","last_peered":"2023-10-09T06:16:43.943512+0000","last_clean":"2023-10-09T06:16:43.943512+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:16:43.943512+0000","last_undegraded":"2023-10-09T06:16:43.943512+0000","last_fullsized":"2023-10-09T06:16:43.943512+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958058+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:16:44.958058+0000","last_peered":"2023-10-09T06:16:44.958058+0000","last_clean":"2023-10-09T06:16:44.958058+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:16:44.958058+0000","last_undegraded":"2023-10-09T06:16:44.958058+0000","last_fullsized":"2023-10-09T06:16:44.958058+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.955580+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:16:44.955580+0000","last_peered":"2023-10-09T06:16:44.955580+0000","last_clean":"2023-10-09T06:16:44.955580+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:16:44.955580+0000","last_undegraded":"2023-10-09T06:16:44.955580+0000","last_fullsized":"2023-10-09T06:16:44.955580+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957836+0000","last_change":"2023-10-09T06:16:44.957836+0000","last_active":"2023-10-09T06:16:44.957836+0000","last_peered":"2023-10-09T06:16:44.957836+0000","last_clean":"2023-10-09T06:16:44.957836+0000","last_became_active":"2023-10-09T06:16:44.957657+0000","last_became_peered":"2023-10-09T06:16:44.957657+0000","last_unstale":"2023-10-09T06:16:44.957836+0000","last_undegraded":"2023-10-09T06:16:44.957836+0000","last_fullsized":"2023-10-09T06:16:44.957836+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:15.752807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959728+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:16:44.959728+0000","last_peered":"2023-10-09T06:16:44.959728+0000","last_clean":"2023-10-09T06:16:44.959728+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:16:44.959728+0000","last_undegraded":"2023-10-09T06:16:44.959728+0000","last_fullsized":"2023-10-09T06:16:44.959728+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947271+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:16:43.947271+0000","last_peered":"2023-10-09T06:16:43.947271+0000","last_clean":"2023-10-09T06:16:43.947271+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:16:43.947271+0000","last_undegraded":"2023-10-09T06:16:43.947271+0000","last_fullsized":"2023-10-09T06:16:43.947271+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958636+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:16:44.958636+0000","last_peered":"2023-10-09T06:16:44.958636+0000","last_clean":"2023-10-09T06:16:44.958636+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:16:44.958636+0000","last_undegraded":"2023-10-09T06:16:44.958636+0000","last_fullsized":"2023-10-09T06:16:44.958636+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963275+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:16:44.963275+0000","last_peered":"2023-10-09T06:16:44.963275+0000","last_clean":"2023-10-09T06:16:44.963275+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:16:44.963275+0000","last_undegraded":"2023-10-09T06:16:44.963275+0000","last_fullsized":"2023-10-09T06:16:44.963275+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951640+0000","last_change":"2023-10-09T06:13:02.835903+0000","last_active":"2023-10-09T06:16:43.951640+0000","last_peered":"2023-10-09T06:16:43.951640+0000","last_clean":"2023-10-09T06:16:43.951640+0000","last_became_active":"2023-10-09T06:13:02.835576+0000","last_became_peered":"2023-10-09T06:13:02.835576+0000","last_unstale":"2023-10-09T06:16:43.951640+0000","last_undegraded":"2023-10-09T06:16:43.951640+0000","last_fullsized":"2023-10-09T06:16:43.951640+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:31.729253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962991+0000","last_change":"2023-10-09T06:16:44.962991+0000","last_active":"2023-10-09T06:16:44.962991+0000","last_peered":"2023-10-09T06:16:44.962991+0000","last_clean":"2023-10-09T06:16:44.962991+0000","last_became_active":"2023-10-09T06:16:44.962656+0000","last_became_peered":"2023-10-09T06:16:44.962656+0000","last_unstale":"2023-10-09T06:16:44.962991+0000","last_undegraded":"2023-10-09T06:16:44.962991+0000","last_fullsized":"2023-10-09T06:16:44.962991+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:23.878640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956659+0000","last_change":"2023-10-09T06:16:44.956659+0000","last_active":"2023-10-09T06:16:44.956659+0000","last_peered":"2023-10-09T06:16:44.956659+0000","last_clean":"2023-10-09T06:16:44.956659+0000","last_became_active":"2023-10-09T06:16:44.956291+0000","last_became_peered":"2023-10-09T06:16:44.956291+0000","last_unstale":"2023-10-09T06:16:44.956659+0000","last_undegraded":"2023-10-09T06:16:44.956659+0000","last_fullsized":"2023-10-09T06:16:44.956659+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:40.513734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.940903+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:16:43.940903+0000","last_peered":"2023-10-09T06:16:43.940903+0000","last_clean":"2023-10-09T06:16:43.940903+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:16:43.940903+0000","last_undegraded":"2023-10-09T06:16:43.940903+0000","last_fullsized":"2023-10-09T06:16:43.940903+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943668+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T06:16:43.943668+0000","last_peered":"2023-10-09T06:16:43.943668+0000","last_clean":"2023-10-09T06:16:43.943668+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T06:16:43.943668+0000","last_undegraded":"2023-10-09T06:16:43.943668+0000","last_fullsized":"2023-10-09T06:16:43.943668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945933+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:16:43.945933+0000","last_peered":"2023-10-09T06:16:43.945933+0000","last_clean":"2023-10-09T06:16:43.945933+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:16:43.945933+0000","last_undegraded":"2023-10-09T06:16:43.945933+0000","last_fullsized":"2023-10-09T06:16:43.945933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949904+0000","last_change":"2023-10-09T06:15:11.874151+0000","last_active":"2023-10-09T06:16:43.949904+0000","last_peered":"2023-10-09T06:16:43.949904+0000","last_clean":"2023-10-09T06:16:43.949904+0000","last_became_active":"2023-10-09T06:15:11.873817+0000","last_became_peered":"2023-10-09T06:15:11.873817+0000","last_unstale":"2023-10-09T06:16:43.949904+0000","last_undegraded":"2023-10-09T06:16:43.949904+0000","last_fullsized":"2023-10-09T06:16:43.949904+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:18.675188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957656+0000","last_change":"2023-10-09T06:16:44.957656+0000","last_active":"2023-10-09T06:16:44.957656+0000","last_peered":"2023-10-09T06:16:44.957656+0000","last_clean":"2023-10-09T06:16:44.957656+0000","last_became_active":"2023-10-09T06:16:44.956987+0000","last_became_peered":"2023-10-09T06:16:44.956987+0000","last_unstale":"2023-10-09T06:16:44.957656+0000","last_undegraded":"2023-10-09T06:16:44.957656+0000","last_fullsized":"2023-10-09T06:16:44.957656+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:19.686112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.958834+0000","last_became_peered":"2023-10-09T06:16:44.958834+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:45.653750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.959015+0000","last_became_peered":"2023-10-09T06:16:44.959015+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:52.310157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946698+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T06:16:43.946698+0000","last_peered":"2023-10-09T06:16:43.946698+0000","last_clean":"2023-10-09T06:16:43.946698+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T06:16:43.946698+0000","last_undegraded":"2023-10-09T06:16:43.946698+0000","last_fullsized":"2023-10-09T06:16:43.946698+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956650+0000","last_change":"2023-10-09T06:16:44.956650+0000","last_active":"2023-10-09T06:16:44.956650+0000","last_peered":"2023-10-09T06:16:44.956650+0000","last_clean":"2023-10-09T06:16:44.956650+0000","last_became_active":"2023-10-09T06:16:44.956333+0000","last_became_peered":"2023-10-09T06:16:44.956333+0000","last_unstale":"2023-10-09T06:16:44.956650+0000","last_undegraded":"2023-10-09T06:16:44.956650+0000","last_fullsized":"2023-10-09T06:16:44.956650+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:45.636029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946241+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:16:43.946241+0000","last_peered":"2023-10-09T06:16:43.946241+0000","last_clean":"2023-10-09T06:16:43.946241+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:16:43.946241+0000","last_undegraded":"2023-10-09T06:16:43.946241+0000","last_fullsized":"2023-10-09T06:16:43.946241+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956576+0000","last_change":"2023-10-09T06:16:44.956576+0000","last_active":"2023-10-09T06:16:44.956576+0000","last_peered":"2023-10-09T06:16:44.956576+0000","last_clean":"2023-10-09T06:16:44.956576+0000","last_became_active":"2023-10-09T06:16:44.956199+0000","last_became_peered":"2023-10-09T06:16:44.956199+0000","last_unstale":"2023-10-09T06:16:44.956576+0000","last_undegraded":"2023-10-09T06:16:44.956576+0000","last_fullsized":"2023-10-09T06:16:44.956576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.170218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.950085+0000","last_change":"2023-10-09T06:08:16.097726+0000","last_active":"2023-10-09T06:16:43.950085+0000","last_peered":"2023-10-09T06:16:43.950085+0000","last_clean":"2023-10-09T06:16:43.950085+0000","last_became_active":"2023-10-09T06:08:16.097256+0000","last_became_peered":"2023-10-09T06:08:16.097256+0000","last_unstale":"2023-10-09T06:16:43.950085+0000","last_undegraded":"2023-10-09T06:16:43.950085+0000","last_fullsized":"2023-10-09T06:16:43.950085+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:20.223173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951497+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T06:16:43.951497+0000","last_peered":"2023-10-09T06:16:43.951497+0000","last_clean":"2023-10-09T06:16:43.951497+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T06:16:43.951497+0000","last_undegraded":"2023-10-09T06:16:43.951497+0000","last_fullsized":"2023-10-09T06:16:43.951497+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948746+0000","last_change":"2023-10-09T06:13:02.837546+0000","last_active":"2023-10-09T06:16:43.948746+0000","last_peered":"2023-10-09T06:16:43.948746+0000","last_clean":"2023-10-09T06:16:43.948746+0000","last_became_active":"2023-10-09T06:13:02.837188+0000","last_became_peered":"2023-10-09T06:13:02.837188+0000","last_unstale":"2023-10-09T06:16:43.948746+0000","last_undegraded":"2023-10-09T06:16:43.948746+0000","last_fullsized":"2023-10-09T06:16:43.948746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:38.707344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957272+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:16:44.957272+0000","last_peered":"2023-10-09T06:16:44.957272+0000","last_clean":"2023-10-09T06:16:44.957272+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:16:44.957272+0000","last_undegraded":"2023-10-09T06:16:44.957272+0000","last_fullsized":"2023-10-09T06:16:44.957272+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948427+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T06:16:43.948427+0000","last_peered":"2023-10-09T06:16:43.948427+0000","last_clean":"2023-10-09T06:16:43.948427+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T06:16:43.948427+0000","last_undegraded":"2023-10-09T06:16:43.948427+0000","last_fullsized":"2023-10-09T06:16:43.948427+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946500+0000","last_change":"2023-10-09T06:13:02.837565+0000","last_active":"2023-10-09T06:16:43.946500+0000","last_peered":"2023-10-09T06:16:43.946500+0000","last_clean":"2023-10-09T06:16:43.946500+0000","last_became_active":"2023-10-09T06:13:02.837258+0000","last_became_peered":"2023-10-09T06:13:02.837258+0000","last_unstale":"2023-10-09T06:16:43.946500+0000","last_undegraded":"2023-10-09T06:16:43.946500+0000","last_fullsized":"2023-10-09T06:16:43.946500+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:11.304315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963142+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:16:44.963142+0000","last_peered":"2023-10-09T06:16:44.963142+0000","last_clean":"2023-10-09T06:16:44.963142+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:16:44.963142+0000","last_undegraded":"2023-10-09T06:16:44.963142+0000","last_fullsized":"2023-10-09T06:16:44.963142+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942316+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:16:44.942316+0000","last_peered":"2023-10-09T06:16:44.942316+0000","last_clean":"2023-10-09T06:16:44.942316+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:16:44.942316+0000","last_undegraded":"2023-10-09T06:16:44.942316+0000","last_fullsized":"2023-10-09T06:16:44.942316+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960277+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:16:44.960277+0000","last_peered":"2023-10-09T06:16:44.960277+0000","last_clean":"2023-10-09T06:16:44.960277+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:16:44.960277+0000","last_undegraded":"2023-10-09T06:16:44.960277+0000","last_fullsized":"2023-10-09T06:16:44.960277+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960879+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:16:44.960879+0000","last_peered":"2023-10-09T06:16:44.960879+0000","last_clean":"2023-10-09T06:16:44.960879+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:16:44.960879+0000","last_undegraded":"2023-10-09T06:16:44.960879+0000","last_fullsized":"2023-10-09T06:16:44.960879+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946957+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:16:43.946957+0000","last_peered":"2023-10-09T06:16:43.946957+0000","last_clean":"2023-10-09T06:16:43.946957+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:16:43.946957+0000","last_undegraded":"2023-10-09T06:16:43.946957+0000","last_fullsized":"2023-10-09T06:16:43.946957+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943713+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:16:43.943713+0000","last_peered":"2023-10-09T06:16:43.943713+0000","last_clean":"2023-10-09T06:16:43.943713+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:16:43.943713+0000","last_undegraded":"2023-10-09T06:16:43.943713+0000","last_fullsized":"2023-10-09T06:16:43.943713+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945160+0000","last_change":"2023-10-09T06:15:11.869407+0000","last_active":"2023-10-09T06:16:43.945160+0000","last_peered":"2023-10-09T06:16:43.945160+0000","last_clean":"2023-10-09T06:16:43.945160+0000","last_became_active":"2023-10-09T06:15:11.869101+0000","last_became_peered":"2023-10-09T06:15:11.869101+0000","last_unstale":"2023-10-09T06:16:43.945160+0000","last_undegraded":"2023-10-09T06:16:43.945160+0000","last_fullsized":"2023-10-09T06:16:43.945160+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:34.742466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943981+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:16:43.943981+0000","last_peered":"2023-10-09T06:16:43.943981+0000","last_clean":"2023-10-09T06:16:43.943981+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:16:43.943981+0000","last_undegraded":"2023-10-09T06:16:43.943981+0000","last_fullsized":"2023-10-09T06:16:43.943981+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956207+0000","last_change":"2023-10-09T06:16:44.956207+0000","last_active":"2023-10-09T06:16:44.956207+0000","last_peered":"2023-10-09T06:16:44.956207+0000","last_clean":"2023-10-09T06:16:44.956207+0000","last_became_active":"2023-10-09T06:16:44.955884+0000","last_became_peered":"2023-10-09T06:16:44.955884+0000","last_unstale":"2023-10-09T06:16:44.956207+0000","last_undegraded":"2023-10-09T06:16:44.956207+0000","last_fullsized":"2023-10-09T06:16:44.956207+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:57.704149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":77,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960353+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:16:44.960353+0000","last_peered":"2023-10-09T06:16:44.960353+0000","last_clean":"2023-10-09T06:16:44.960353+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:16:44.960353+0000","last_undegraded":"2023-10-09T06:16:44.960353+0000","last_fullsized":"2023-10-09T06:16:44.960353+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949624+0000","last_change":"2023-10-09T06:13:02.834962+0000","last_active":"2023-10-09T06:16:44.949624+0000","last_peered":"2023-10-09T06:16:44.949624+0000","last_clean":"2023-10-09T06:16:44.949624+0000","last_became_active":"2023-10-09T06:13:02.834677+0000","last_became_peered":"2023-10-09T06:13:02.834677+0000","last_unstale":"2023-10-09T06:16:44.949624+0000","last_undegraded":"2023-10-09T06:16:44.949624+0000","last_fullsized":"2023-10-09T06:16:44.949624+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:35.835505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962615+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:16:44.962615+0000","last_peered":"2023-10-09T06:16:44.962615+0000","last_clean":"2023-10-09T06:16:44.962615+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:16:44.962615+0000","last_undegraded":"2023-10-09T06:16:44.962615+0000","last_fullsized":"2023-10-09T06:16:44.962615+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948689+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:16:44.948689+0000","last_peered":"2023-10-09T06:16:44.948689+0000","last_clean":"2023-10-09T06:16:44.948689+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:16:44.948689+0000","last_undegraded":"2023-10-09T06:16:44.948689+0000","last_fullsized":"2023-10-09T06:16:44.948689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.952582+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:16:43.952582+0000","last_peered":"2023-10-09T06:16:43.952582+0000","last_clean":"2023-10-09T06:16:43.952582+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:16:43.952582+0000","last_undegraded":"2023-10-09T06:16:43.952582+0000","last_fullsized":"2023-10-09T06:16:43.952582+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.944356+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:16:43.944356+0000","last_peered":"2023-10-09T06:16:43.944356+0000","last_clean":"2023-10-09T06:16:43.944356+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:16:43.944356+0000","last_undegraded":"2023-10-09T06:16:43.944356+0000","last_fullsized":"2023-10-09T06:16:43.944356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959456+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:16:44.959456+0000","last_peered":"2023-10-09T06:16:44.959456+0000","last_clean":"2023-10-09T06:16:44.959456+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:16:44.959456+0000","last_undegraded":"2023-10-09T06:16:44.959456+0000","last_fullsized":"2023-10-09T06:16:44.959456+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953059+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:16:44.953059+0000","last_peered":"2023-10-09T06:16:44.953059+0000","last_clean":"2023-10-09T06:16:44.953059+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:16:44.953059+0000","last_undegraded":"2023-10-09T06:16:44.953059+0000","last_fullsized":"2023-10-09T06:16:44.953059+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.950033+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:16:44.950033+0000","last_peered":"2023-10-09T06:16:44.950033+0000","last_clean":"2023-10-09T06:16:44.950033+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:16:44.950033+0000","last_undegraded":"2023-10-09T06:16:44.950033+0000","last_fullsized":"2023-10-09T06:16:44.950033+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948431+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:16:44.948431+0000","last_peered":"2023-10-09T06:16:44.948431+0000","last_clean":"2023-10-09T06:16:44.948431+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:16:44.948431+0000","last_undegraded":"2023-10-09T06:16:44.948431+0000","last_fullsized":"2023-10-09T06:16:44.948431+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951947+0000","last_change":"2023-10-09T06:15:11.886011+0000","last_active":"2023-10-09T06:16:43.951947+0000","last_peered":"2023-10-09T06:16:43.951947+0000","last_clean":"2023-10-09T06:16:43.951947+0000","last_became_active":"2023-10-09T06:15:11.885754+0000","last_became_peered":"2023-10-09T06:15:11.885754+0000","last_unstale":"2023-10-09T06:16:43.951947+0000","last_undegraded":"2023-10-09T06:16:43.951947+0000","last_fullsized":"2023-10-09T06:16:43.951947+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:33.874944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":490,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947610+0000","last_change":"2023-10-09T06:08:16.098085+0000","last_active":"2023-10-09T06:16:43.947610+0000","last_peered":"2023-10-09T06:16:43.947610+0000","last_clean":"2023-10-09T06:16:43.947610+0000","last_became_active":"2023-10-09T06:08:16.097675+0000","last_became_peered":"2023-10-09T06:08:16.097675+0000","last_unstale":"2023-10-09T06:16:43.947610+0000","last_undegraded":"2023-10-09T06:16:43.947610+0000","last_fullsized":"2023-10-09T06:16:43.947610+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:36.003332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961643+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:16:44.961643+0000","last_peered":"2023-10-09T06:16:44.961643+0000","last_clean":"2023-10-09T06:16:44.961643+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:16:44.961643+0000","last_undegraded":"2023-10-09T06:16:44.961643+0000","last_fullsized":"2023-10-09T06:16:44.961643+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947632+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T06:16:43.947632+0000","last_peered":"2023-10-09T06:16:43.947632+0000","last_clean":"2023-10-09T06:16:43.947632+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T06:16:43.947632+0000","last_undegraded":"2023-10-09T06:16:43.947632+0000","last_fullsized":"2023-10-09T06:16:43.947632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947513+0000","last_change":"2023-10-09T06:13:02.846478+0000","last_active":"2023-10-09T06:16:43.947513+0000","last_peered":"2023-10-09T06:16:43.947513+0000","last_clean":"2023-10-09T06:16:43.947513+0000","last_became_active":"2023-10-09T06:13:02.846169+0000","last_became_peered":"2023-10-09T06:13:02.846169+0000","last_unstale":"2023-10-09T06:16:43.947513+0000","last_undegraded":"2023-10-09T06:16:43.947513+0000","last_fullsized":"2023-10-09T06:16:43.947513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:18.710495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960166+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:16:44.960166+0000","last_peered":"2023-10-09T06:16:44.960166+0000","last_clean":"2023-10-09T06:16:44.960166+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:16:44.960166+0000","last_undegraded":"2023-10-09T06:16:44.960166+0000","last_fullsized":"2023-10-09T06:16:44.960166+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942690+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:16:44.942690+0000","last_peered":"2023-10-09T06:16:44.942690+0000","last_clean":"2023-10-09T06:16:44.942690+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:16:44.942690+0000","last_undegraded":"2023-10-09T06:16:44.942690+0000","last_fullsized":"2023-10-09T06:16:44.942690+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948529+0000","last_change":"2023-10-09T06:08:16.097472+0000","last_active":"2023-10-09T06:16:43.948529+0000","last_peered":"2023-10-09T06:16:43.948529+0000","last_clean":"2023-10-09T06:16:43.948529+0000","last_became_active":"2023-10-09T06:08:16.097159+0000","last_became_peered":"2023-10-09T06:08:16.097159+0000","last_unstale":"2023-10-09T06:16:43.948529+0000","last_undegraded":"2023-10-09T06:16:43.948529+0000","last_fullsized":"2023-10-09T06:16:43.948529+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:09.440710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949726+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:16:44.949726+0000","last_peered":"2023-10-09T06:16:44.949726+0000","last_clean":"2023-10-09T06:16:44.949726+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:16:44.949726+0000","last_undegraded":"2023-10-09T06:16:44.949726+0000","last_fullsized":"2023-10-09T06:16:44.949726+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948458+0000","last_change":"2023-10-09T06:08:16.100157+0000","last_active":"2023-10-09T06:16:43.948458+0000","last_peered":"2023-10-09T06:16:43.948458+0000","last_clean":"2023-10-09T06:16:43.948458+0000","last_became_active":"2023-10-09T06:08:16.099543+0000","last_became_peered":"2023-10-09T06:08:16.099543+0000","last_unstale":"2023-10-09T06:16:43.948458+0000","last_undegraded":"2023-10-09T06:16:43.948458+0000","last_fullsized":"2023-10-09T06:16:43.948458+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:08.543959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963419+0000","last_change":"2023-10-09T06:16:44.963419+0000","last_active":"2023-10-09T06:16:44.963419+0000","last_peered":"2023-10-09T06:16:44.963419+0000","last_clean":"2023-10-09T06:16:44.963419+0000","last_became_active":"2023-10-09T06:16:44.963126+0000","last_became_peered":"2023-10-09T06:16:44.963126+0000","last_unstale":"2023-10-09T06:16:44.963419+0000","last_undegraded":"2023-10-09T06:16:44.963419+0000","last_fullsized":"2023-10-09T06:16:44.963419+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:21.614499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948574+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:16:44.948574+0000","last_peered":"2023-10-09T06:16:44.948574+0000","last_clean":"2023-10-09T06:16:44.948574+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:16:44.948574+0000","last_undegraded":"2023-10-09T06:16:44.948574+0000","last_fullsized":"2023-10-09T06:16:44.948574+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.954350+0000","last_change":"2023-10-09T06:16:44.954350+0000","last_active":"2023-10-09T06:16:44.954350+0000","last_peered":"2023-10-09T06:16:44.954350+0000","last_clean":"2023-10-09T06:16:44.954350+0000","last_became_active":"2023-10-09T06:16:44.954027+0000","last_became_peered":"2023-10-09T06:16:44.954027+0000","last_unstale":"2023-10-09T06:16:44.954350+0000","last_undegraded":"2023-10-09T06:16:44.954350+0000","last_fullsized":"2023-10-09T06:16:44.954350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:18.938865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947159+0000","last_change":"2023-10-09T06:15:11.869497+0000","last_active":"2023-10-09T06:16:43.947159+0000","last_peered":"2023-10-09T06:16:43.947159+0000","last_clean":"2023-10-09T06:16:43.947159+0000","last_became_active":"2023-10-09T06:15:11.868971+0000","last_became_peered":"2023-10-09T06:15:11.868971+0000","last_unstale":"2023-10-09T06:16:43.947159+0000","last_undegraded":"2023-10-09T06:16:43.947159+0000","last_fullsized":"2023-10-09T06:16:43.947159+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:20.786443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942174+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:16:44.942174+0000","last_peered":"2023-10-09T06:16:44.942174+0000","last_clean":"2023-10-09T06:16:44.942174+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:16:44.942174+0000","last_undegraded":"2023-10-09T06:16:44.942174+0000","last_fullsized":"2023-10-09T06:16:44.942174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945269+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:16:43.945269+0000","last_peered":"2023-10-09T06:16:43.945269+0000","last_clean":"2023-10-09T06:16:43.945269+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:16:43.945269+0000","last_undegraded":"2023-10-09T06:16:43.945269+0000","last_fullsized":"2023-10-09T06:16:43.945269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":87,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948619+0000","last_change":"2023-10-09T06:08:16.100332+0000","last_active":"2023-10-09T06:16:43.948619+0000","last_peered":"2023-10-09T06:16:43.948619+0000","last_clean":"2023-10-09T06:16:43.948619+0000","last_became_active":"2023-10-09T06:08:16.099887+0000","last_became_peered":"2023-10-09T06:08:16.099887+0000","last_unstale":"2023-10-09T06:16:43.948619+0000","last_undegraded":"2023-10-09T06:16:43.948619+0000","last_fullsized":"2023-10-09T06:16:43.948619+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:36.739647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958199+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:16:44.958199+0000","last_peered":"2023-10-09T06:16:44.958199+0000","last_clean":"2023-10-09T06:16:44.958199+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:16:44.958199+0000","last_undegraded":"2023-10-09T06:16:44.958199+0000","last_fullsized":"2023-10-09T06:16:44.958199+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.953404+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:16:43.953404+0000","last_peered":"2023-10-09T06:16:43.953404+0000","last_clean":"2023-10-09T06:16:43.953404+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:16:43.953404+0000","last_undegraded":"2023-10-09T06:16:43.953404+0000","last_fullsized":"2023-10-09T06:16:43.953404+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956492+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:16:44.956492+0000","last_peered":"2023-10-09T06:16:44.956492+0000","last_clean":"2023-10-09T06:16:44.956492+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:16:44.956492+0000","last_undegraded":"2023-10-09T06:16:44.956492+0000","last_fullsized":"2023-10-09T06:16:44.956492+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":95,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.954232+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:16:43.954232+0000","last_peered":"2023-10-09T06:16:43.954232+0000","last_clean":"2023-10-09T06:16:43.954232+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:16:43.954232+0000","last_undegraded":"2023-10-09T06:16:43.954232+0000","last_fullsized":"2023-10-09T06:16:43.954232+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956388+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:16:44.956388+0000","last_peered":"2023-10-09T06:16:44.956388+0000","last_clean":"2023-10-09T06:16:44.956388+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:16:44.956388+0000","last_undegraded":"2023-10-09T06:16:44.956388+0000","last_fullsized":"2023-10-09T06:16:44.956388+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961893+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:16:44.961893+0000","last_peered":"2023-10-09T06:16:44.961893+0000","last_clean":"2023-10-09T06:16:44.961893+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:16:44.961893+0000","last_undegraded":"2023-10-09T06:16:44.961893+0000","last_fullsized":"2023-10-09T06:16:44.961893+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953905+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:16:44.953905+0000","last_peered":"2023-10-09T06:16:44.953905+0000","last_clean":"2023-10-09T06:16:44.953905+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:16:44.953905+0000","last_undegraded":"2023-10-09T06:16:44.953905+0000","last_fullsized":"2023-10-09T06:16:44.953905+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949161+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:16:43.949161+0000","last_peered":"2023-10-09T06:16:43.949161+0000","last_clean":"2023-10-09T06:16:43.949161+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:16:43.949161+0000","last_undegraded":"2023-10-09T06:16:43.949161+0000","last_fullsized":"2023-10-09T06:16:43.949161+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943302+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:16:43.943302+0000","last_peered":"2023-10-09T06:16:43.943302+0000","last_clean":"2023-10-09T06:16:43.943302+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:16:43.943302+0000","last_undegraded":"2023-10-09T06:16:43.943302+0000","last_fullsized":"2023-10-09T06:16:43.943302+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234242,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311644,"kb_used_data":2068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433508,"statfs":{"total":95995035648,"available":95675912192,"internally_reserved":0,"allocated":2117632,"data_stored":1556820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:31 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.5629999999999999,"15min":1.306},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.388,"5min":11.785,"15min":11.785},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.329,"15min":1.4470000000000001},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":10.396000000000001,"5min":10.396000000000001,"15min":10.784000000000001},"last":0.53300000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.2480000000000002,"15min":1.728},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.542,"5min":10.542,"15min":10.542},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.899,"15min":1.5069999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.548,"5min":11.818,"15min":11.818},"last":0.48299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:17:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":2.016,"15min":2.0379999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.0859999999999999,"15min":2.1579999999999999},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.2549999999999999,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.68400000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":17.396000000000001,"5min":4.96,"15min":3.6850000000000001},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":267.75099999999998,"5min":267.75099999999998,"15min":267.75099999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":18.986999999999998,"5min":5.4269999999999996,"15min":3.5819999999999999},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":271.76299999999998,"5min":271.76299999999998,"15min":271.76299999999998},"last":0.70699999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.601,"15min":1.601},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.251},"max":{"1min":9.1140000000000008,"5min":11.853999999999999,"15min":41.527000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.256,"5min":1.0509999999999999,"15min":1.653},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":6.7629999999999999,"15min":41.466000000000001},"last":0.76200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":1.915,"15min":1.865},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":8.4589999999999996,"5min":9.8629999999999995,"15min":49.357999999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.762,"5min":1.3420000000000001,"15min":1.518},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.6769999999999996,"5min":11.837,"15min":49.237000000000002},"last":0.748}]},{"osd":6,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.4390000000000001,"15min":1.51},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":8.2249999999999996,"5min":11.878,"15min":11.878},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.2150000000000001,"15min":1.5960000000000001},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.0380000000000003,"5min":7.9340000000000002,"15min":13.189},"last":0.79800000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311624,"kb_used_data":2048,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433528,"statfs":{"total":95995035648,"available":95675932672,"internally_reserved":0,"allocated":2097152,"data_stored":1556086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.659,"15min":1.6479999999999999},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.214},"max":{"1min":12.808,"5min":12.808,"15min":12.808},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.7789999999999999,"5min":1.984,"15min":1.7070000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":12.847,"5min":12.847,"15min":12.847},"last":0.53100000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.5600000000000001,"15min":2.2879999999999998},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.5690000000000008,"5min":10.646000000000001,"15min":10.646000000000001},"last":0.442},{"interface":"front","average":{"1min":1.6850000000000001,"5min":2.4740000000000002,"15min":2.3719999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":9.6080000000000005,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.3319999999999999,"15min":2.4380000000000002},"min":{"1min":0.41499999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6539999999999999,"5min":7.6539999999999999,"15min":7.6539999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.137,"5min":2.383,"15min":2.5910000000000002},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6549999999999998,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":4.8639999999999999,"15min":3.8780000000000001},"min":{"1min":0.42599999999999999,"5min":0.28899999999999998,"15min":0.26700000000000002},"max":{"1min":9.0419999999999998,"5min":222.922,"15min":222.922},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.5640000000000001,"5min":4.4390000000000001,"15min":3.6099999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.30599999999999999},"max":{"1min":11.374000000000001,"5min":222.911,"15min":222.911},"last":0.64300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.532,"15min":1.3120000000000001},"min":{"1min":0.375,"5min":0.252,"15min":0.245},"max":{"1min":5.3620000000000001,"5min":10.75,"15min":10.75},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.51,"15min":1.5669999999999999},"min":{"1min":0.38400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.96,"5min":8.9239999999999995,"15min":12.567},"last":0.55200000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.474,"15min":1.47},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":4.1349999999999998,"5min":10.773,"15min":13.91},"last":0.625},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6040000000000001,"15min":2.069},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.6779999999999999,"5min":11.798999999999999,"15min":159.06800000000001},"last":0.60599999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.1599999999999999,"15min":1.3149999999999999},"min":{"1min":0.33800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.962,"5min":10.962,"15min":14.837},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.6699999999999999,"15min":1.7929999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":8.9920000000000009,"5min":10.891,"15min":11.281000000000001},"last":0.63300000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402615,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256336,"kb_used_data":1608,"kb_used_omap":4,"kb_used_meta":254715,"kb_avail":93488816,"statfs":{"total":95995035648,"available":95732547584,"internally_reserved":0,"allocated":1646592,"data_stored":981322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":260829070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.1989999999999998,"15min":2.5990000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.0939999999999994,"5min":10.726000000000001,"15min":10.726000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":2.681,"15min":2.9940000000000002},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.44700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.526,"15min":1.6639999999999999},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0500000000000007,"5min":9.2769999999999992,"15min":9.2769999999999992},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.448,"15min":2.7330000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.1970000000000001,"5min":10.573,"15min":10.573},"last":0.41199999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":3.0899999999999999,"15min":3.0659999999999998},"min":{"1min":0.29599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":11.044,"5min":11.044,"15min":11.044},"last":0.433},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.5819999999999999,"15min":2.7839999999999998},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":12.464,"5min":13.487,"15min":13.487},"last":0.48299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.2280000000000002,"15min":2.8460000000000001},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1239999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.5169999999999999,"15min":2.907},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":10.723000000000001,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.4860000000000002,"15min":3.2629999999999999},"min":{"1min":0.39200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":10.92,"15min":10.92},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.7909999999999999,"15min":3.2109999999999999},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.68700000000000006}]},{"osd":6,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":2.7999999999999998,"15min":2.956},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.772,"5min":12.91,"15min":12.91},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.6080000000000001,"5min":2.2839999999999998,"15min":2.5699999999999998},"min":{"1min":0.437,"5min":0.374,"15min":0.374},"max":{"1min":6.3170000000000002,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":2.347,"15min":2.7490000000000001},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.0830000000000002,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.3400000000000001,"5min":2.0070000000000001,"15min":2.129},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":5.5599999999999996,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.56100000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024048,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410940,"kb_used_data":2292,"kb_used_omap":5,"kb_used_meta":408634,"kb_avail":93334212,"statfs":{"total":95995035648,"available":95574233088,"internally_reserved":0,"allocated":2347008,"data_stored":1585989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":418441850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.6930000000000001,"15min":1.8759999999999999},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":11.576000000000001,"5min":11.576000000000001,"15min":11.576000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.3700000000000001,"5min":2.7490000000000001,"15min":2.6619999999999999},"min":{"1min":0.27200000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.603,"5min":11.744999999999999,"15min":11.744999999999999},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.488,"15min":2.6150000000000002},"min":{"1min":0.254,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.186,"5min":11.186,"15min":11.186},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.016,"5min":2.5230000000000001,"15min":2.6070000000000002},"min":{"1min":0.21299999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":21.198,"5min":21.198,"15min":21.198},"last":0.29499999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":3.3149999999999999,"15min":3.8319999999999999},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.414,"5min":24.870000000000001,"15min":27.048999999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2839999999999998,"15min":3.423},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.196999999999999,"5min":32.030000000000001,"15min":32.030000000000001},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.5669999999999999,"15min":1.8029999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":4.5499999999999998,"5min":13.244,"15min":15.835000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2549999999999999,"15min":2.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.25700000000000001},"max":{"1min":8.7769999999999992,"5min":11.566000000000001,"15min":15.023},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.0750000000000002,"15min":1.9990000000000001},"min":{"1min":0.33200000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":11.222,"5min":13.285,"15min":15.858000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":3.7480000000000002,"5min":2.3780000000000001,"15min":2.2080000000000002},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.355,"15min":2.2090000000000001},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.088999999999999,"5min":21.088999999999999,"15min":32.079000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.1859999999999999,"15min":2.085},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.0270000000000001,"5min":13.315,"15min":15.893000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.319,"15min":2.387},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.28899999999999998},"max":{"1min":8.8089999999999993,"5min":11.515000000000001,"15min":15.050000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":4.2309999999999999,"5min":2.4470000000000001,"15min":2.3399999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":21.135999999999999,"5min":21.135999999999999,"15min":32.055999999999997},"last":0.57599999999999996}]}]},{"osd":2,"up_from":128,"seq":549755813917,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196712,"kb_used_data":1568,"kb_used_omap":7,"kb_used_meta":195128,"kb_avail":93548440,"statfs":{"total":95995035648,"available":95793602560,"internally_reserved":0,"allocated":1605632,"data_stored":975398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":199811153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.863,"15min":1.9099999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.156000000000001,"5min":12.156000000000001,"15min":12.156000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.145,"5min":2.4159999999999999,"15min":2.6280000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.113,"5min":12.113,"15min":12.113},"last":0.53700000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":11.619999999999999,"5min":3.8079999999999998,"15min":2.5059999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":222.50299999999999,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":11.102,"5min":4.4299999999999997,"15min":3.3180000000000001},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":222.19399999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.44}]},{"osd":3,"last update":"Mon Oct 9 06:16:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":2.8759999999999999,"15min":2.8759999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.435,"5min":10.435,"15min":10.435},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.0179999999999998,"5min":2.0179999999999998,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6299999999999999,"5min":6.6299999999999999,"15min":6.6299999999999999},"last":0.501}]},{"osd":4,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.879,"15min":2.0859999999999999},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.1560000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.1320000000000001,"15min":2.3799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.6230000000000002,"5min":15.183,"15min":15.183},"last":0.69299999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.591,"15min":1.744},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.1960000000000002,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.3039999999999998,"15min":2.524},"min":{"1min":0.35499999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.085,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.0779999999999998,"15min":2.3149999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.6890000000000001,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.5509999999999999,"15min":1.6599999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.1509999999999998,"5min":7.327,"15min":7.327},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.5980000000000001,"15min":1.6899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.1260000000000003,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.626},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.734,"15min":3.089},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.2229999999999999,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.72499999999999998}]}]},{"osd":3,"up_from":149,"seq":639950127115,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111000,"kb_used_data":1488,"kb_used_omap":2,"kb_used_meta":109501,"kb_avail":93634152,"statfs":{"total":95995035648,"available":95881371648,"internally_reserved":0,"allocated":1523712,"data_stored":966500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":112129077},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35999999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45600000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81799999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]}]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591248,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93153904,"statfs":{"total":95995035648,"available":95389597696,"internally_reserved":0,"allocated":1515520,"data_stored":966117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.6879999999999999,"15min":1.409},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.4190000000000005,"5min":8.5990000000000002,"15min":15.484999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.226,"15min":1.7729999999999999},"min":{"1min":0.28499999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.4380000000000006,"5min":11.234,"15min":15.289},"last":0.49299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.25600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.7050000000000001,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.28,"5min":1.542,"15min":1.619},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1449999999999996,"5min":8.7509999999999994,"15min":8.7509999999999994},"last":0.59499999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.784,"15min":1.865},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9100000000000001,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.35599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.7559999999999998,"5min":7.2830000000000004,"15min":7.2830000000000004},"last":0.54200000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":3.0329999999999999,"15min":4.125},"min":{"1min":0.318,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":11.31,"5min":60.414999999999999,"15min":236.78},"last":0.628},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.6339999999999999,"15min":4.0140000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.29499999999999998},"max":{"1min":8.2010000000000005,"5min":60.375999999999998,"15min":236.76599999999999},"last":0.69999999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.3660000000000001,"15min":1.3169999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":4.4779999999999998,"5min":9.6790000000000003,"15min":19.687999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.5820000000000001,"15min":1.4319999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":2.8100000000000001,"5min":11.282,"15min":11.282},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.1439999999999999,"15min":1.359},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.907,"5min":9.7680000000000007,"15min":20.390999999999998},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.923,"5min":1.7450000000000001,"15min":1.623},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.25},"max":{"1min":14.202999999999999,"5min":14.202999999999999,"15min":26.137},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.21,"15min":1.3049999999999999},"min":{"1min":0.379,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":15.911},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.2010000000000001,"5min":1.4259999999999999,"15min":1.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.28799999999999998},"max":{"1min":12.965999999999999,"5min":12.965999999999999,"15min":15.333},"last":0.747}]}]},{"osd":5,"up_from":38,"seq":163208757516,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603520,"kb_used_data":1464,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141632,"statfs":{"total":95995035648,"available":95377031168,"internally_reserved":0,"allocated":1499136,"data_stored":961803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.9119999999999999,"15min":1.7170000000000001},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":8.5139999999999993,"5min":18.637,"15min":18.637},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.032,"5min":2.1400000000000001,"15min":1.5629999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.9459999999999997,"5min":9.9939999999999998,"15min":9.9939999999999998},"last":0.67200000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.032,"15min":2.3420000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.8420000000000001,"5min":11.292,"15min":11.292},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.218,"15min":2.1920000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6349999999999998,"5min":7.9539999999999997,"15min":7.9539999999999997},"last":0.71599999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.1829999999999998,"15min":2.3860000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.0519999999999996,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0779999999999998,"15min":2.2610000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":5.6790000000000003,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":0.78800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.8039999999999998,"15min":3.2080000000000002},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":12.106,"5min":12.106,"15min":128.035},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.96,"5min":2.327,"15min":3.0289999999999999},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.4020000000000001,"5min":10.324999999999999,"15min":128.07300000000001},"last":0.81699999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.696,"15min":1.5940000000000001},"min":{"1min":0.35799999999999998,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":5.6360000000000001,"5min":8.7850000000000001,"15min":19.847999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.276,"15min":1.353},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":5.1200000000000001,"5min":7.7290000000000001,"15min":20.106000000000002},"last":0.75}]},{"osd":6,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":1.5489999999999999,"15min":1.681},"min":{"1min":0.35099999999999998,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":8.8149999999999995,"5min":8.8149999999999995,"15min":20.027000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5149999999999999,"15min":1.49},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.23899999999999999},"max":{"1min":7.3479999999999999,"5min":7.3479999999999999,"15min":16.152999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.407,"15min":1.6100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.8409999999999993,"5min":8.8409999999999993,"15min":23.398},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.363,"5min":1.323,"15min":1.4970000000000001},"min":{"1min":0.40699999999999997,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":7.0049999999999999,"5min":7.7809999999999997,"15min":20.131},"last":0.86799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:31.013 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:31.356 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:31.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:31 smithi107 ceph-mon[212750]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:31.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:31.893 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:31 smithi107 ceph-mon[220921]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:31.893 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:31 smithi150 ceph-mon[194178]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:32.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:32] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-10-09T06:17:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:32 smithi107 ceph-mon[212750]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:32 smithi107 ceph-mon[220921]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:32 smithi150 ceph-mon[194178]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:32.954 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:32.955 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:17:33.625 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":504,"stamp":"2023-10-09T06:17:32.883751+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2793024,"kb_used_data":14016,"kb_used_omap":20,"kb_used_meta":2778923,"kb_avail":747168192,"statfs":{"total":767960285184,"available":765100228608,"internally_reserved":0,"allocated":14352384,"data_stored":9550035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2845617806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002373"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943512+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:16:43.943512+0000","last_peered":"2023-10-09T06:16:43.943512+0000","last_clean":"2023-10-09T06:16:43.943512+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:16:43.943512+0000","last_undegraded":"2023-10-09T06:16:43.943512+0000","last_fullsized":"2023-10-09T06:16:43.943512+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958058+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:16:44.958058+0000","last_peered":"2023-10-09T06:16:44.958058+0000","last_clean":"2023-10-09T06:16:44.958058+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:16:44.958058+0000","last_undegraded":"2023-10-09T06:16:44.958058+0000","last_fullsized":"2023-10-09T06:16:44.958058+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.955580+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:16:44.955580+0000","last_peered":"2023-10-09T06:16:44.955580+0000","last_clean":"2023-10-09T06:16:44.955580+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:16:44.955580+0000","last_undegraded":"2023-10-09T06:16:44.955580+0000","last_fullsized":"2023-10-09T06:16:44.955580+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957836+0000","last_change":"2023-10-09T06:16:44.957836+0000","last_active":"2023-10-09T06:16:44.957836+0000","last_peered":"2023-10-09T06:16:44.957836+0000","last_clean":"2023-10-09T06:16:44.957836+0000","last_became_active":"2023-10-09T06:16:44.957657+0000","last_became_peered":"2023-10-09T06:16:44.957657+0000","last_unstale":"2023-10-09T06:16:44.957836+0000","last_undegraded":"2023-10-09T06:16:44.957836+0000","last_fullsized":"2023-10-09T06:16:44.957836+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:15.752807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959728+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:16:44.959728+0000","last_peered":"2023-10-09T06:16:44.959728+0000","last_clean":"2023-10-09T06:16:44.959728+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:16:44.959728+0000","last_undegraded":"2023-10-09T06:16:44.959728+0000","last_fullsized":"2023-10-09T06:16:44.959728+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947271+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:16:43.947271+0000","last_peered":"2023-10-09T06:16:43.947271+0000","last_clean":"2023-10-09T06:16:43.947271+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:16:43.947271+0000","last_undegraded":"2023-10-09T06:16:43.947271+0000","last_fullsized":"2023-10-09T06:16:43.947271+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958636+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:16:44.958636+0000","last_peered":"2023-10-09T06:16:44.958636+0000","last_clean":"2023-10-09T06:16:44.958636+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:16:44.958636+0000","last_undegraded":"2023-10-09T06:16:44.958636+0000","last_fullsized":"2023-10-09T06:16:44.958636+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963275+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:16:44.963275+0000","last_peered":"2023-10-09T06:16:44.963275+0000","last_clean":"2023-10-09T06:16:44.963275+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:16:44.963275+0000","last_undegraded":"2023-10-09T06:16:44.963275+0000","last_fullsized":"2023-10-09T06:16:44.963275+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951640+0000","last_change":"2023-10-09T06:13:02.835903+0000","last_active":"2023-10-09T06:16:43.951640+0000","last_peered":"2023-10-09T06:16:43.951640+0000","last_clean":"2023-10-09T06:16:43.951640+0000","last_became_active":"2023-10-09T06:13:02.835576+0000","last_became_peered":"2023-10-09T06:13:02.835576+0000","last_unstale":"2023-10-09T06:16:43.951640+0000","last_undegraded":"2023-10-09T06:16:43.951640+0000","last_fullsized":"2023-10-09T06:16:43.951640+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:31.729253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962991+0000","last_change":"2023-10-09T06:16:44.962991+0000","last_active":"2023-10-09T06:16:44.962991+0000","last_peered":"2023-10-09T06:16:44.962991+0000","last_clean":"2023-10-09T06:16:44.962991+0000","last_became_active":"2023-10-09T06:16:44.962656+0000","last_became_peered":"2023-10-09T06:16:44.962656+0000","last_unstale":"2023-10-09T06:16:44.962991+0000","last_undegraded":"2023-10-09T06:16:44.962991+0000","last_fullsized":"2023-10-09T06:16:44.962991+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:23.878640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956659+0000","last_change":"2023-10-09T06:16:44.956659+0000","last_active":"2023-10-09T06:16:44.956659+0000","last_peered":"2023-10-09T06:16:44.956659+0000","last_clean":"2023-10-09T06:16:44.956659+0000","last_became_active":"2023-10-09T06:16:44.956291+0000","last_became_peered":"2023-10-09T06:16:44.956291+0000","last_unstale":"2023-10-09T06:16:44.956659+0000","last_undegraded":"2023-10-09T06:16:44.956659+0000","last_fullsized":"2023-10-09T06:16:44.956659+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:40.513734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.940903+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:16:43.940903+0000","last_peered":"2023-10-09T06:16:43.940903+0000","last_clean":"2023-10-09T06:16:43.940903+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:16:43.940903+0000","last_undegraded":"2023-10-09T06:16:43.940903+0000","last_fullsized":"2023-10-09T06:16:43.940903+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943668+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T06:16:43.943668+0000","last_peered":"2023-10-09T06:16:43.943668+0000","last_clean":"2023-10-09T06:16:43.943668+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T06:16:43.943668+0000","last_undegraded":"2023-10-09T06:16:43.943668+0000","last_fullsized":"2023-10-09T06:16:43.943668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945933+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:16:43.945933+0000","last_peered":"2023-10-09T06:16:43.945933+0000","last_clean":"2023-10-09T06:16:43.945933+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:16:43.945933+0000","last_undegraded":"2023-10-09T06:16:43.945933+0000","last_fullsized":"2023-10-09T06:16:43.945933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949904+0000","last_change":"2023-10-09T06:15:11.874151+0000","last_active":"2023-10-09T06:16:43.949904+0000","last_peered":"2023-10-09T06:16:43.949904+0000","last_clean":"2023-10-09T06:16:43.949904+0000","last_became_active":"2023-10-09T06:15:11.873817+0000","last_became_peered":"2023-10-09T06:15:11.873817+0000","last_unstale":"2023-10-09T06:16:43.949904+0000","last_undegraded":"2023-10-09T06:16:43.949904+0000","last_fullsized":"2023-10-09T06:16:43.949904+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:18.675188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957656+0000","last_change":"2023-10-09T06:16:44.957656+0000","last_active":"2023-10-09T06:16:44.957656+0000","last_peered":"2023-10-09T06:16:44.957656+0000","last_clean":"2023-10-09T06:16:44.957656+0000","last_became_active":"2023-10-09T06:16:44.956987+0000","last_became_peered":"2023-10-09T06:16:44.956987+0000","last_unstale":"2023-10-09T06:16:44.957656+0000","last_undegraded":"2023-10-09T06:16:44.957656+0000","last_fullsized":"2023-10-09T06:16:44.957656+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:19.686112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.958834+0000","last_became_peered":"2023-10-09T06:16:44.958834+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:45.653750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.959015+0000","last_became_peered":"2023-10-09T06:16:44.959015+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:52.310157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946698+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T06:16:43.946698+0000","last_peered":"2023-10-09T06:16:43.946698+0000","last_clean":"2023-10-09T06:16:43.946698+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T06:16:43.946698+0000","last_undegraded":"2023-10-09T06:16:43.946698+0000","last_fullsized":"2023-10-09T06:16:43.946698+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956650+0000","last_change":"2023-10-09T06:16:44.956650+0000","last_active":"2023-10-09T06:16:44.956650+0000","last_peered":"2023-10-09T06:16:44.956650+0000","last_clean":"2023-10-09T06:16:44.956650+0000","last_became_active":"2023-10-09T06:16:44.956333+0000","last_became_peered":"2023-10-09T06:16:44.956333+0000","last_unstale":"2023-10-09T06:16:44.956650+0000","last_undegraded":"2023-10-09T06:16:44.956650+0000","last_fullsized":"2023-10-09T06:16:44.956650+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:45.636029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946241+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:16:43.946241+0000","last_peered":"2023-10-09T06:16:43.946241+0000","last_clean":"2023-10-09T06:16:43.946241+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:16:43.946241+0000","last_undegraded":"2023-10-09T06:16:43.946241+0000","last_fullsized":"2023-10-09T06:16:43.946241+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956576+0000","last_change":"2023-10-09T06:16:44.956576+0000","last_active":"2023-10-09T06:16:44.956576+0000","last_peered":"2023-10-09T06:16:44.956576+0000","last_clean":"2023-10-09T06:16:44.956576+0000","last_became_active":"2023-10-09T06:16:44.956199+0000","last_became_peered":"2023-10-09T06:16:44.956199+0000","last_unstale":"2023-10-09T06:16:44.956576+0000","last_undegraded":"2023-10-09T06:16:44.956576+0000","last_fullsized":"2023-10-09T06:16:44.956576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.170218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.950085+0000","last_change":"2023-10-09T06:08:16.097726+0000","last_active":"2023-10-09T06:16:43.950085+0000","last_peered":"2023-10-09T06:16:43.950085+0000","last_clean":"2023-10-09T06:16:43.950085+0000","last_became_active":"2023-10-09T06:08:16.097256+0000","last_became_peered":"2023-10-09T06:08:16.097256+0000","last_unstale":"2023-10-09T06:16:43.950085+0000","last_undegraded":"2023-10-09T06:16:43.950085+0000","last_fullsized":"2023-10-09T06:16:43.950085+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:20.223173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951497+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T06:16:43.951497+0000","last_peered":"2023-10-09T06:16:43.951497+0000","last_clean":"2023-10-09T06:16:43.951497+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T06:16:43.951497+0000","last_undegraded":"2023-10-09T06:16:43.951497+0000","last_fullsized":"2023-10-09T06:16:43.951497+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948746+0000","last_change":"2023-10-09T06:13:02.837546+0000","last_active":"2023-10-09T06:16:43.948746+0000","last_peered":"2023-10-09T06:16:43.948746+0000","last_clean":"2023-10-09T06:16:43.948746+0000","last_became_active":"2023-10-09T06:13:02.837188+0000","last_became_peered":"2023-10-09T06:13:02.837188+0000","last_unstale":"2023-10-09T06:16:43.948746+0000","last_undegraded":"2023-10-09T06:16:43.948746+0000","last_fullsized":"2023-10-09T06:16:43.948746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:38.707344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957272+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:16:44.957272+0000","last_peered":"2023-10-09T06:16:44.957272+0000","last_clean":"2023-10-09T06:16:44.957272+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:16:44.957272+0000","last_undegraded":"2023-10-09T06:16:44.957272+0000","last_fullsized":"2023-10-09T06:16:44.957272+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948427+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T06:16:43.948427+0000","last_peered":"2023-10-09T06:16:43.948427+0000","last_clean":"2023-10-09T06:16:43.948427+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T06:16:43.948427+0000","last_undegraded":"2023-10-09T06:16:43.948427+0000","last_fullsized":"2023-10-09T06:16:43.948427+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946500+0000","last_change":"2023-10-09T06:13:02.837565+0000","last_active":"2023-10-09T06:16:43.946500+0000","last_peered":"2023-10-09T06:16:43.946500+0000","last_clean":"2023-10-09T06:16:43.946500+0000","last_became_active":"2023-10-09T06:13:02.837258+0000","last_became_peered":"2023-10-09T06:13:02.837258+0000","last_unstale":"2023-10-09T06:16:43.946500+0000","last_undegraded":"2023-10-09T06:16:43.946500+0000","last_fullsized":"2023-10-09T06:16:43.946500+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:11.304315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963142+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:16:44.963142+0000","last_peered":"2023-10-09T06:16:44.963142+0000","last_clean":"2023-10-09T06:16:44.963142+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:16:44.963142+0000","last_undegraded":"2023-10-09T06:16:44.963142+0000","last_fullsized":"2023-10-09T06:16:44.963142+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942316+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:16:44.942316+0000","last_peered":"2023-10-09T06:16:44.942316+0000","last_clean":"2023-10-09T06:16:44.942316+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:16:44.942316+0000","last_undegraded":"2023-10-09T06:16:44.942316+0000","last_fullsized":"2023-10-09T06:16:44.942316+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960277+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:16:44.960277+0000","last_peered":"2023-10-09T06:16:44.960277+0000","last_clean":"2023-10-09T06:16:44.960277+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:16:44.960277+0000","last_undegraded":"2023-10-09T06:16:44.960277+0000","last_fullsized":"2023-10-09T06:16:44.960277+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960879+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:16:44.960879+0000","last_peered":"2023-10-09T06:16:44.960879+0000","last_clean":"2023-10-09T06:16:44.960879+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:16:44.960879+0000","last_undegraded":"2023-10-09T06:16:44.960879+0000","last_fullsized":"2023-10-09T06:16:44.960879+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946957+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:16:43.946957+0000","last_peered":"2023-10-09T06:16:43.946957+0000","last_clean":"2023-10-09T06:16:43.946957+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:16:43.946957+0000","last_undegraded":"2023-10-09T06:16:43.946957+0000","last_fullsized":"2023-10-09T06:16:43.946957+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943713+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:16:43.943713+0000","last_peered":"2023-10-09T06:16:43.943713+0000","last_clean":"2023-10-09T06:16:43.943713+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:16:43.943713+0000","last_undegraded":"2023-10-09T06:16:43.943713+0000","last_fullsized":"2023-10-09T06:16:43.943713+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945160+0000","last_change":"2023-10-09T06:15:11.869407+0000","last_active":"2023-10-09T06:16:43.945160+0000","last_peered":"2023-10-09T06:16:43.945160+0000","last_clean":"2023-10-09T06:16:43.945160+0000","last_became_active":"2023-10-09T06:15:11.869101+0000","last_became_peered":"2023-10-09T06:15:11.869101+0000","last_unstale":"2023-10-09T06:16:43.945160+0000","last_undegraded":"2023-10-09T06:16:43.945160+0000","last_fullsized":"2023-10-09T06:16:43.945160+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:34.742466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943981+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:16:43.943981+0000","last_peered":"2023-10-09T06:16:43.943981+0000","last_clean":"2023-10-09T06:16:43.943981+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:16:43.943981+0000","last_undegraded":"2023-10-09T06:16:43.943981+0000","last_fullsized":"2023-10-09T06:16:43.943981+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956207+0000","last_change":"2023-10-09T06:16:44.956207+0000","last_active":"2023-10-09T06:16:44.956207+0000","last_peered":"2023-10-09T06:16:44.956207+0000","last_clean":"2023-10-09T06:16:44.956207+0000","last_became_active":"2023-10-09T06:16:44.955884+0000","last_became_peered":"2023-10-09T06:16:44.955884+0000","last_unstale":"2023-10-09T06:16:44.956207+0000","last_undegraded":"2023-10-09T06:16:44.956207+0000","last_fullsized":"2023-10-09T06:16:44.956207+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:57.704149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":78,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963851+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:16:44.963851+0000","last_peered":"2023-10-09T06:16:44.963851+0000","last_clean":"2023-10-09T06:16:44.963851+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:16:44.963851+0000","last_undegraded":"2023-10-09T06:16:44.963851+0000","last_fullsized":"2023-10-09T06:16:44.963851+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949624+0000","last_change":"2023-10-09T06:13:02.834962+0000","last_active":"2023-10-09T06:16:44.949624+0000","last_peered":"2023-10-09T06:16:44.949624+0000","last_clean":"2023-10-09T06:16:44.949624+0000","last_became_active":"2023-10-09T06:13:02.834677+0000","last_became_peered":"2023-10-09T06:13:02.834677+0000","last_unstale":"2023-10-09T06:16:44.949624+0000","last_undegraded":"2023-10-09T06:16:44.949624+0000","last_fullsized":"2023-10-09T06:16:44.949624+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:35.835505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962615+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:16:44.962615+0000","last_peered":"2023-10-09T06:16:44.962615+0000","last_clean":"2023-10-09T06:16:44.962615+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:16:44.962615+0000","last_undegraded":"2023-10-09T06:16:44.962615+0000","last_fullsized":"2023-10-09T06:16:44.962615+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948689+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:16:44.948689+0000","last_peered":"2023-10-09T06:16:44.948689+0000","last_clean":"2023-10-09T06:16:44.948689+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:16:44.948689+0000","last_undegraded":"2023-10-09T06:16:44.948689+0000","last_fullsized":"2023-10-09T06:16:44.948689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.952582+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:16:43.952582+0000","last_peered":"2023-10-09T06:16:43.952582+0000","last_clean":"2023-10-09T06:16:43.952582+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:16:43.952582+0000","last_undegraded":"2023-10-09T06:16:43.952582+0000","last_fullsized":"2023-10-09T06:16:43.952582+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.944356+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:16:43.944356+0000","last_peered":"2023-10-09T06:16:43.944356+0000","last_clean":"2023-10-09T06:16:43.944356+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:16:43.944356+0000","last_undegraded":"2023-10-09T06:16:43.944356+0000","last_fullsized":"2023-10-09T06:16:43.944356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959456+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:16:44.959456+0000","last_peered":"2023-10-09T06:16:44.959456+0000","last_clean":"2023-10-09T06:16:44.959456+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:16:44.959456+0000","last_undegraded":"2023-10-09T06:16:44.959456+0000","last_fullsized":"2023-10-09T06:16:44.959456+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953059+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:16:44.953059+0000","last_peered":"2023-10-09T06:16:44.953059+0000","last_clean":"2023-10-09T06:16:44.953059+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:16:44.953059+0000","last_undegraded":"2023-10-09T06:16:44.953059+0000","last_fullsized":"2023-10-09T06:16:44.953059+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.950033+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:16:44.950033+0000","last_peered":"2023-10-09T06:16:44.950033+0000","last_clean":"2023-10-09T06:16:44.950033+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:16:44.950033+0000","last_undegraded":"2023-10-09T06:16:44.950033+0000","last_fullsized":"2023-10-09T06:16:44.950033+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948431+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:16:44.948431+0000","last_peered":"2023-10-09T06:16:44.948431+0000","last_clean":"2023-10-09T06:16:44.948431+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:16:44.948431+0000","last_undegraded":"2023-10-09T06:16:44.948431+0000","last_fullsized":"2023-10-09T06:16:44.948431+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951947+0000","last_change":"2023-10-09T06:15:11.886011+0000","last_active":"2023-10-09T06:16:43.951947+0000","last_peered":"2023-10-09T06:16:43.951947+0000","last_clean":"2023-10-09T06:16:43.951947+0000","last_became_active":"2023-10-09T06:15:11.885754+0000","last_became_peered":"2023-10-09T06:15:11.885754+0000","last_unstale":"2023-10-09T06:16:43.951947+0000","last_undegraded":"2023-10-09T06:16:43.951947+0000","last_fullsized":"2023-10-09T06:16:43.951947+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:33.874944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":490,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947610+0000","last_change":"2023-10-09T06:08:16.098085+0000","last_active":"2023-10-09T06:16:43.947610+0000","last_peered":"2023-10-09T06:16:43.947610+0000","last_clean":"2023-10-09T06:16:43.947610+0000","last_became_active":"2023-10-09T06:08:16.097675+0000","last_became_peered":"2023-10-09T06:08:16.097675+0000","last_unstale":"2023-10-09T06:16:43.947610+0000","last_undegraded":"2023-10-09T06:16:43.947610+0000","last_fullsized":"2023-10-09T06:16:43.947610+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:36.003332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961643+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:16:44.961643+0000","last_peered":"2023-10-09T06:16:44.961643+0000","last_clean":"2023-10-09T06:16:44.961643+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:16:44.961643+0000","last_undegraded":"2023-10-09T06:16:44.961643+0000","last_fullsized":"2023-10-09T06:16:44.961643+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947632+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T06:16:43.947632+0000","last_peered":"2023-10-09T06:16:43.947632+0000","last_clean":"2023-10-09T06:16:43.947632+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T06:16:43.947632+0000","last_undegraded":"2023-10-09T06:16:43.947632+0000","last_fullsized":"2023-10-09T06:16:43.947632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947513+0000","last_change":"2023-10-09T06:13:02.846478+0000","last_active":"2023-10-09T06:16:43.947513+0000","last_peered":"2023-10-09T06:16:43.947513+0000","last_clean":"2023-10-09T06:16:43.947513+0000","last_became_active":"2023-10-09T06:13:02.846169+0000","last_became_peered":"2023-10-09T06:13:02.846169+0000","last_unstale":"2023-10-09T06:16:43.947513+0000","last_undegraded":"2023-10-09T06:16:43.947513+0000","last_fullsized":"2023-10-09T06:16:43.947513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:18.710495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960166+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:16:44.960166+0000","last_peered":"2023-10-09T06:16:44.960166+0000","last_clean":"2023-10-09T06:16:44.960166+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:16:44.960166+0000","last_undegraded":"2023-10-09T06:16:44.960166+0000","last_fullsized":"2023-10-09T06:16:44.960166+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942690+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:16:44.942690+0000","last_peered":"2023-10-09T06:16:44.942690+0000","last_clean":"2023-10-09T06:16:44.942690+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:16:44.942690+0000","last_undegraded":"2023-10-09T06:16:44.942690+0000","last_fullsized":"2023-10-09T06:16:44.942690+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948529+0000","last_change":"2023-10-09T06:08:16.097472+0000","last_active":"2023-10-09T06:16:43.948529+0000","last_peered":"2023-10-09T06:16:43.948529+0000","last_clean":"2023-10-09T06:16:43.948529+0000","last_became_active":"2023-10-09T06:08:16.097159+0000","last_became_peered":"2023-10-09T06:08:16.097159+0000","last_unstale":"2023-10-09T06:16:43.948529+0000","last_undegraded":"2023-10-09T06:16:43.948529+0000","last_fullsized":"2023-10-09T06:16:43.948529+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:09.440710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949726+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:16:44.949726+0000","last_peered":"2023-10-09T06:16:44.949726+0000","last_clean":"2023-10-09T06:16:44.949726+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:16:44.949726+0000","last_undegraded":"2023-10-09T06:16:44.949726+0000","last_fullsized":"2023-10-09T06:16:44.949726+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948458+0000","last_change":"2023-10-09T06:08:16.100157+0000","last_active":"2023-10-09T06:16:43.948458+0000","last_peered":"2023-10-09T06:16:43.948458+0000","last_clean":"2023-10-09T06:16:43.948458+0000","last_became_active":"2023-10-09T06:08:16.099543+0000","last_became_peered":"2023-10-09T06:08:16.099543+0000","last_unstale":"2023-10-09T06:16:43.948458+0000","last_undegraded":"2023-10-09T06:16:43.948458+0000","last_fullsized":"2023-10-09T06:16:43.948458+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:08.543959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963419+0000","last_change":"2023-10-09T06:16:44.963419+0000","last_active":"2023-10-09T06:16:44.963419+0000","last_peered":"2023-10-09T06:16:44.963419+0000","last_clean":"2023-10-09T06:16:44.963419+0000","last_became_active":"2023-10-09T06:16:44.963126+0000","last_became_peered":"2023-10-09T06:16:44.963126+0000","last_unstale":"2023-10-09T06:16:44.963419+0000","last_undegraded":"2023-10-09T06:16:44.963419+0000","last_fullsized":"2023-10-09T06:16:44.963419+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:21.614499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948574+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:16:44.948574+0000","last_peered":"2023-10-09T06:16:44.948574+0000","last_clean":"2023-10-09T06:16:44.948574+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:16:44.948574+0000","last_undegraded":"2023-10-09T06:16:44.948574+0000","last_fullsized":"2023-10-09T06:16:44.948574+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.954350+0000","last_change":"2023-10-09T06:16:44.954350+0000","last_active":"2023-10-09T06:16:44.954350+0000","last_peered":"2023-10-09T06:16:44.954350+0000","last_clean":"2023-10-09T06:16:44.954350+0000","last_became_active":"2023-10-09T06:16:44.954027+0000","last_became_peered":"2023-10-09T06:16:44.954027+0000","last_unstale":"2023-10-09T06:16:44.954350+0000","last_undegraded":"2023-10-09T06:16:44.954350+0000","last_fullsized":"2023-10-09T06:16:44.954350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:18.938865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947159+0000","last_change":"2023-10-09T06:15:11.869497+0000","last_active":"2023-10-09T06:16:43.947159+0000","last_peered":"2023-10-09T06:16:43.947159+0000","last_clean":"2023-10-09T06:16:43.947159+0000","last_became_active":"2023-10-09T06:15:11.868971+0000","last_became_peered":"2023-10-09T06:15:11.868971+0000","last_unstale":"2023-10-09T06:16:43.947159+0000","last_undegraded":"2023-10-09T06:16:43.947159+0000","last_fullsized":"2023-10-09T06:16:43.947159+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:20.786443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942174+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:16:44.942174+0000","last_peered":"2023-10-09T06:16:44.942174+0000","last_clean":"2023-10-09T06:16:44.942174+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:16:44.942174+0000","last_undegraded":"2023-10-09T06:16:44.942174+0000","last_fullsized":"2023-10-09T06:16:44.942174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945269+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:16:43.945269+0000","last_peered":"2023-10-09T06:16:43.945269+0000","last_clean":"2023-10-09T06:16:43.945269+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:16:43.945269+0000","last_undegraded":"2023-10-09T06:16:43.945269+0000","last_fullsized":"2023-10-09T06:16:43.945269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":87,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948619+0000","last_change":"2023-10-09T06:08:16.100332+0000","last_active":"2023-10-09T06:16:43.948619+0000","last_peered":"2023-10-09T06:16:43.948619+0000","last_clean":"2023-10-09T06:16:43.948619+0000","last_became_active":"2023-10-09T06:08:16.099887+0000","last_became_peered":"2023-10-09T06:08:16.099887+0000","last_unstale":"2023-10-09T06:16:43.948619+0000","last_undegraded":"2023-10-09T06:16:43.948619+0000","last_fullsized":"2023-10-09T06:16:43.948619+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:36.739647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958199+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:16:44.958199+0000","last_peered":"2023-10-09T06:16:44.958199+0000","last_clean":"2023-10-09T06:16:44.958199+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:16:44.958199+0000","last_undegraded":"2023-10-09T06:16:44.958199+0000","last_fullsized":"2023-10-09T06:16:44.958199+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.953404+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:16:43.953404+0000","last_peered":"2023-10-09T06:16:43.953404+0000","last_clean":"2023-10-09T06:16:43.953404+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:16:43.953404+0000","last_undegraded":"2023-10-09T06:16:43.953404+0000","last_fullsized":"2023-10-09T06:16:43.953404+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956492+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:16:44.956492+0000","last_peered":"2023-10-09T06:16:44.956492+0000","last_clean":"2023-10-09T06:16:44.956492+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:16:44.956492+0000","last_undegraded":"2023-10-09T06:16:44.956492+0000","last_fullsized":"2023-10-09T06:16:44.956492+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":95,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.954232+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:16:43.954232+0000","last_peered":"2023-10-09T06:16:43.954232+0000","last_clean":"2023-10-09T06:16:43.954232+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:16:43.954232+0000","last_undegraded":"2023-10-09T06:16:43.954232+0000","last_fullsized":"2023-10-09T06:16:43.954232+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956388+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:16:44.956388+0000","last_peered":"2023-10-09T06:16:44.956388+0000","last_clean":"2023-10-09T06:16:44.956388+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:16:44.956388+0000","last_undegraded":"2023-10-09T06:16:44.956388+0000","last_fullsized":"2023-10-09T06:16:44.956388+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961893+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:16:44.961893+0000","last_peered":"2023-10-09T06:16:44.961893+0000","last_clean":"2023-10-09T06:16:44.961893+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:16:44.961893+0000","last_undegraded":"2023-10-09T06:16:44.961893+0000","last_fullsized":"2023-10-09T06:16:44.961893+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953905+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:16:44.953905+0000","last_peered":"2023-10-09T06:16:44.953905+0000","last_clean":"2023-10-09T06:16:44.953905+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:16:44.953905+0000","last_undegraded":"2023-10-09T06:16:44.953905+0000","last_fullsized":"2023-10-09T06:16:44.953905+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949161+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:16:43.949161+0000","last_peered":"2023-10-09T06:16:43.949161+0000","last_clean":"2023-10-09T06:16:43.949161+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:16:43.949161+0000","last_undegraded":"2023-10-09T06:16:43.949161+0000","last_fullsized":"2023-10-09T06:16:43.949161+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943302+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:16:43.943302+0000","last_peered":"2023-10-09T06:16:43.943302+0000","last_clean":"2023-10-09T06:16:43.943302+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:16:43.943302+0000","last_undegraded":"2023-10-09T06:16:43.943302+0000","last_fullsized":"2023-10-09T06:16:43.943302+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234243,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311644,"kb_used_data":2068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433508,"statfs":{"total":95995035648,"available":95675912192,"internally_reserved":0,"allocated":2117632,"data_stored":1556820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.482,"15min":1.272},"min":{"1min":0.37,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":0.74099999999999999,"5min":11.785,"15min":11.785},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":1.232,"15min":1.3919999999999999},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":0.70799999999999996,"5min":10.396000000000001,"15min":10.784000000000001},"last":0.67700000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.2480000000000002,"15min":1.728},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.542,"5min":10.542,"15min":10.542},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.899,"15min":1.5069999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.548,"5min":11.818,"15min":11.818},"last":0.64200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:17:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":2.016,"15min":2.0379999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.0859999999999999,"15min":2.1579999999999999},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.2549999999999999,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.73099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":17.396000000000001,"5min":4.96,"15min":3.6850000000000001},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":267.75099999999998,"5min":267.75099999999998,"15min":267.75099999999998},"last":0.63},{"interface":"front","average":{"1min":18.986999999999998,"5min":5.4269999999999996,"15min":3.5819999999999999},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":271.76299999999998,"5min":271.76299999999998,"15min":271.76299999999998},"last":0.747}]},{"osd":4,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.601,"15min":1.601},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.251},"max":{"1min":9.1140000000000008,"5min":11.853999999999999,"15min":41.527000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.256,"5min":1.0509999999999999,"15min":1.653},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":6.7629999999999999,"15min":41.466000000000001},"last":0.81799999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":1.915,"15min":1.865},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":8.4589999999999996,"5min":9.8629999999999995,"15min":49.357999999999997},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.762,"5min":1.3420000000000001,"15min":1.518},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.6769999999999996,"5min":11.837,"15min":49.237000000000002},"last":0.77200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.4390000000000001,"15min":1.51},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":8.2249999999999996,"5min":11.878,"15min":11.878},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.2150000000000001,"15min":1.5960000000000001},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.0380000000000003,"5min":7.9340000000000002,"15min":13.189},"last":0.78600000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495880,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311624,"kb_used_data":2048,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433528,"statfs":{"total":95995035648,"available":95675932672,"internally_reserved":0,"allocated":2097152,"data_stored":1556086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.659,"15min":1.6479999999999999},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.214},"max":{"1min":12.808,"5min":12.808,"15min":12.808},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.7789999999999999,"5min":1.984,"15min":1.7070000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":12.847,"5min":12.847,"15min":12.847},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.5600000000000001,"15min":2.2879999999999998},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.5690000000000008,"5min":10.646000000000001,"15min":10.646000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.6850000000000001,"5min":2.4740000000000002,"15min":2.3719999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":9.6080000000000005,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.53700000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.3319999999999999,"15min":2.4380000000000002},"min":{"1min":0.41499999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6539999999999999,"5min":7.6539999999999999,"15min":7.6539999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.137,"5min":2.383,"15min":2.5910000000000002},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6549999999999998,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":0.65700000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":4.8639999999999999,"15min":3.8780000000000001},"min":{"1min":0.42599999999999999,"5min":0.28899999999999998,"15min":0.26700000000000002},"max":{"1min":9.0419999999999998,"5min":222.922,"15min":222.922},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.5640000000000001,"5min":4.4390000000000001,"15min":3.6099999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.30599999999999999},"max":{"1min":11.374000000000001,"5min":222.911,"15min":222.911},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.532,"15min":1.3120000000000001},"min":{"1min":0.375,"5min":0.252,"15min":0.245},"max":{"1min":5.3620000000000001,"5min":10.75,"15min":10.75},"last":0.376},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.51,"15min":1.5669999999999999},"min":{"1min":0.38400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.96,"5min":8.9239999999999995,"15min":12.567},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.474,"15min":1.47},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":4.1349999999999998,"5min":10.773,"15min":13.91},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6040000000000001,"15min":2.069},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.6779999999999999,"5min":11.798999999999999,"15min":159.06800000000001},"last":0.626}]},{"osd":7,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.1599999999999999,"15min":1.3149999999999999},"min":{"1min":0.33800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.962,"5min":10.962,"15min":14.837},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.6699999999999999,"15min":1.7929999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":8.9920000000000009,"5min":10.891,"15min":11.281000000000001},"last":0.57899999999999996}]}]},{"osd":1,"up_from":110,"seq":472446402616,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256336,"kb_used_data":1608,"kb_used_omap":4,"kb_used_meta":254715,"kb_avail":93488816,"statfs":{"total":95995035648,"available":95732547584,"internally_reserved":0,"allocated":1646592,"data_stored":981322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":260829070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.1989999999999998,"15min":2.5990000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.0939999999999994,"5min":10.726000000000001,"15min":10.726000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.76800000000000002,"5min":2.681,"15min":2.9940000000000002},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.51600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.526,"15min":1.6639999999999999},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0500000000000007,"5min":9.2769999999999992,"15min":9.2769999999999992},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.448,"15min":2.7330000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.1970000000000001,"5min":10.573,"15min":10.573},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":3.0899999999999999,"15min":3.0659999999999998},"min":{"1min":0.29599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":11.044,"5min":11.044,"15min":11.044},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.5819999999999999,"15min":2.7839999999999998},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":12.464,"5min":13.487,"15min":13.487},"last":0.504}]},{"osd":4,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.2280000000000002,"15min":2.8460000000000001},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1239999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.5169999999999999,"15min":2.907},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":10.723000000000001,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.70099999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.4860000000000002,"15min":3.2629999999999999},"min":{"1min":0.39200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":10.92,"15min":10.92},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.7909999999999999,"15min":3.2109999999999999},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":2.7999999999999998,"15min":2.956},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.772,"5min":12.91,"15min":12.91},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.6080000000000001,"5min":2.2839999999999998,"15min":2.5699999999999998},"min":{"1min":0.437,"5min":0.374,"15min":0.374},"max":{"1min":6.3170000000000002,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.73899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":2.347,"15min":2.7490000000000001},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.0830000000000002,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.3400000000000001,"5min":2.0070000000000001,"15min":2.129},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":5.5599999999999996,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.80100000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024049,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410940,"kb_used_data":2292,"kb_used_omap":5,"kb_used_meta":408634,"kb_avail":93334212,"statfs":{"total":95995035648,"available":95574233088,"internally_reserved":0,"allocated":2347008,"data_stored":1585989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":418441850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.6930000000000001,"15min":1.8759999999999999},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":11.576000000000001,"5min":11.576000000000001,"15min":11.576000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.3700000000000001,"5min":2.7490000000000001,"15min":2.6619999999999999},"min":{"1min":0.27200000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.603,"5min":11.744999999999999,"15min":11.744999999999999},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.488,"15min":2.6150000000000002},"min":{"1min":0.254,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.186,"5min":11.186,"15min":11.186},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.016,"5min":2.5230000000000001,"15min":2.6070000000000002},"min":{"1min":0.21299999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":21.198,"5min":21.198,"15min":21.198},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":3.3149999999999999,"15min":3.8319999999999999},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.414,"5min":24.870000000000001,"15min":27.048999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2839999999999998,"15min":3.423},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.196999999999999,"5min":32.030000000000001,"15min":32.030000000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.5669999999999999,"15min":1.8029999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":4.5499999999999998,"5min":13.244,"15min":15.835000000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2549999999999999,"15min":2.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.25700000000000001},"max":{"1min":8.7769999999999992,"5min":11.566000000000001,"15min":15.023},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.0750000000000002,"15min":1.9990000000000001},"min":{"1min":0.33200000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":11.222,"5min":13.285,"15min":15.858000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":3.7480000000000002,"5min":2.3780000000000001,"15min":2.2080000000000002},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.69699999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.355,"15min":2.2090000000000001},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.088999999999999,"5min":21.088999999999999,"15min":32.079000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.1859999999999999,"15min":2.085},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.0270000000000001,"5min":13.315,"15min":15.893000000000001},"last":0.71499999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.319,"15min":2.387},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.28899999999999998},"max":{"1min":8.8089999999999993,"5min":11.515000000000001,"15min":15.050000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":4.2309999999999999,"5min":2.4470000000000001,"15min":2.3399999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":21.135999999999999,"5min":21.135999999999999,"15min":32.055999999999997},"last":0.55800000000000005}]}]},{"osd":2,"up_from":128,"seq":549755813918,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196712,"kb_used_data":1568,"kb_used_omap":7,"kb_used_meta":195128,"kb_avail":93548440,"statfs":{"total":95995035648,"available":95793602560,"internally_reserved":0,"allocated":1605632,"data_stored":975398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":199811153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.863,"15min":1.9099999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.156000000000001,"5min":12.156000000000001,"15min":12.156000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.145,"5min":2.4159999999999999,"15min":2.6280000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.113,"5min":12.113,"15min":12.113},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":11.619999999999999,"5min":3.8079999999999998,"15min":2.5059999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":222.50299999999999,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":11.102,"5min":4.4299999999999997,"15min":3.3180000000000001},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":222.19399999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:16:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":2.8759999999999999,"15min":2.8759999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.435,"5min":10.435,"15min":10.435},"last":0.28699999999999998},{"interface":"front","average":{"1min":2.0179999999999998,"5min":2.0179999999999998,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6299999999999999,"5min":6.6299999999999999,"15min":6.6299999999999999},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.879,"15min":2.0859999999999999},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.1560000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.1320000000000001,"15min":2.3799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.6230000000000002,"5min":15.183,"15min":15.183},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.591,"15min":1.744},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.1960000000000002,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.3039999999999998,"15min":2.524},"min":{"1min":0.35499999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.085,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.0779999999999998,"15min":2.3149999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.6890000000000001,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.5509999999999999,"15min":1.6599999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.1509999999999998,"5min":7.327,"15min":7.327},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.5980000000000001,"15min":1.6899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.1260000000000003,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.734,"15min":3.089},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.2229999999999999,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.48099999999999998}]}]},{"osd":3,"up_from":149,"seq":639950127115,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111000,"kb_used_data":1488,"kb_used_omap":2,"kb_used_meta":109501,"kb_avail":93634152,"statfs":{"total":95995035648,"available":95881371648,"internally_reserved":0,"allocated":1523712,"data_stored":966500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":112129077},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35999999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45600000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81799999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]}]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591248,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93153904,"statfs":{"total":95995035648,"available":95389597696,"internally_reserved":0,"allocated":1515520,"data_stored":966117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.6879999999999999,"15min":1.409},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.4190000000000005,"5min":8.5990000000000002,"15min":15.484999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.226,"15min":1.7729999999999999},"min":{"1min":0.28499999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.4380000000000006,"5min":11.234,"15min":15.289},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.25600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.7050000000000001,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.28,"5min":1.542,"15min":1.619},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1449999999999996,"5min":8.7509999999999994,"15min":8.7509999999999994},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.784,"15min":1.865},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9100000000000001,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.35599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.7559999999999998,"5min":7.2830000000000004,"15min":7.2830000000000004},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":3.0329999999999999,"15min":4.125},"min":{"1min":0.318,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":11.31,"5min":60.414999999999999,"15min":236.78},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.6339999999999999,"15min":4.0140000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.29499999999999998},"max":{"1min":8.2010000000000005,"5min":60.375999999999998,"15min":236.76599999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.3660000000000001,"15min":1.3169999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":4.4779999999999998,"5min":9.6790000000000003,"15min":19.687999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.5820000000000001,"15min":1.4319999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":2.8100000000000001,"5min":11.282,"15min":11.282},"last":0.68300000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.1439999999999999,"15min":1.359},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.907,"5min":9.7680000000000007,"15min":20.390999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.923,"5min":1.7450000000000001,"15min":1.623},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.25},"max":{"1min":14.202999999999999,"5min":14.202999999999999,"15min":26.137},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.21,"15min":1.3049999999999999},"min":{"1min":0.379,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":15.911},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.2010000000000001,"5min":1.4259999999999999,"15min":1.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.28799999999999998},"max":{"1min":12.965999999999999,"5min":12.965999999999999,"15min":15.333},"last":0.70099999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757516,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603520,"kb_used_data":1464,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141632,"statfs":{"total":95995035648,"available":95377031168,"internally_reserved":0,"allocated":1499136,"data_stored":961803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.9119999999999999,"15min":1.7170000000000001},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":8.5139999999999993,"5min":18.637,"15min":18.637},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.032,"5min":2.1400000000000001,"15min":1.5629999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.9459999999999997,"5min":9.9939999999999998,"15min":9.9939999999999998},"last":0.67200000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.032,"15min":2.3420000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.8420000000000001,"5min":11.292,"15min":11.292},"last":0.73599999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.218,"15min":2.1920000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6349999999999998,"5min":7.9539999999999997,"15min":7.9539999999999997},"last":0.71599999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.1829999999999998,"15min":2.3860000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.0519999999999996,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0779999999999998,"15min":2.2610000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":5.6790000000000003,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":0.78800000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.8039999999999998,"15min":3.2080000000000002},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":12.106,"5min":12.106,"15min":128.035},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.96,"5min":2.327,"15min":3.0289999999999999},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.4020000000000001,"5min":10.324999999999999,"15min":128.07300000000001},"last":0.81699999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.696,"15min":1.5940000000000001},"min":{"1min":0.35799999999999998,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":5.6360000000000001,"5min":8.7850000000000001,"15min":19.847999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.276,"15min":1.353},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":5.1200000000000001,"5min":7.7290000000000001,"15min":20.106000000000002},"last":0.75}]},{"osd":6,"last update":"Mon Oct 9 06:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":1.5489999999999999,"15min":1.681},"min":{"1min":0.35099999999999998,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":8.8149999999999995,"5min":8.8149999999999995,"15min":20.027000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.8600000000000001,"5min":1.5149999999999999,"15min":1.49},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.23899999999999999},"max":{"1min":7.3479999999999999,"5min":7.3479999999999999,"15min":16.152999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.407,"15min":1.6100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.8409999999999993,"5min":8.8409999999999993,"15min":23.398},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.363,"5min":1.323,"15min":1.4970000000000001},"min":{"1min":0.40699999999999997,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":7.0049999999999999,"5min":7.7809999999999997,"15min":20.131},"last":0.86799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:33.631 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:17:33.631 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T06:17:33.631 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T06:17:33.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T06:17:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:33 smithi107 ceph-mon[212750]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:33 smithi107 ceph-mon[212750]: from='client.45188 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:33 smithi107 ceph-mon[220921]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:33 smithi107 ceph-mon[220921]: from='client.45188 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:33 smithi150 ceph-mon[194178]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:33 smithi150 ceph-mon[194178]: from='client.45188 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:33.978 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:35.565 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:35.566 INFO:teuthology.orchestra.run.smithi107.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T06:17:36.114 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T06:17:36.115 INFO:teuthology.run_tasks:Running task print... 2023-10-09T06:17:36.124 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T06:17:36.124 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T06:17:36.229 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T06:17:36.229 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T06:17:36.229 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T06:17:36.231 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T06:17:36.231 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T06:17:36.231 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T06:17:36.231 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T06:17:36.232 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T06:17:36.232 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:36.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:35 smithi107 ceph-mon[212750]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:36.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2118190808' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:17:36.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:35 smithi107 ceph-mon[220921]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:36.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2118190808' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:17:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:35 smithi150 ceph-mon[194178]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2118190808' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T06:17:36.580 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:38.118 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:38.119 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:17:38.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:37 smithi107 ceph-mon[212750]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:38.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:37 smithi107 ceph-mon[220921]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:37 smithi150 ceph-mon[194178]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:38.722 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":506,"stamp":"2023-10-09T06:17:36.884679+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2793024,"kb_used_data":14016,"kb_used_omap":20,"kb_used_meta":2778923,"kb_avail":747168192,"statfs":{"total":767960285184,"available":765100228608,"internally_reserved":0,"allocated":14352384,"data_stored":9550035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2845617806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002502"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943512+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:16:43.943512+0000","last_peered":"2023-10-09T06:16:43.943512+0000","last_clean":"2023-10-09T06:16:43.943512+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:16:43.943512+0000","last_undegraded":"2023-10-09T06:16:43.943512+0000","last_fullsized":"2023-10-09T06:16:43.943512+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958058+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:16:44.958058+0000","last_peered":"2023-10-09T06:16:44.958058+0000","last_clean":"2023-10-09T06:16:44.958058+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:16:44.958058+0000","last_undegraded":"2023-10-09T06:16:44.958058+0000","last_fullsized":"2023-10-09T06:16:44.958058+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.955580+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:16:44.955580+0000","last_peered":"2023-10-09T06:16:44.955580+0000","last_clean":"2023-10-09T06:16:44.955580+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:16:44.955580+0000","last_undegraded":"2023-10-09T06:16:44.955580+0000","last_fullsized":"2023-10-09T06:16:44.955580+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957836+0000","last_change":"2023-10-09T06:16:44.957836+0000","last_active":"2023-10-09T06:16:44.957836+0000","last_peered":"2023-10-09T06:16:44.957836+0000","last_clean":"2023-10-09T06:16:44.957836+0000","last_became_active":"2023-10-09T06:16:44.957657+0000","last_became_peered":"2023-10-09T06:16:44.957657+0000","last_unstale":"2023-10-09T06:16:44.957836+0000","last_undegraded":"2023-10-09T06:16:44.957836+0000","last_fullsized":"2023-10-09T06:16:44.957836+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:15.752807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959728+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:16:44.959728+0000","last_peered":"2023-10-09T06:16:44.959728+0000","last_clean":"2023-10-09T06:16:44.959728+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:16:44.959728+0000","last_undegraded":"2023-10-09T06:16:44.959728+0000","last_fullsized":"2023-10-09T06:16:44.959728+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947271+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:16:43.947271+0000","last_peered":"2023-10-09T06:16:43.947271+0000","last_clean":"2023-10-09T06:16:43.947271+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:16:43.947271+0000","last_undegraded":"2023-10-09T06:16:43.947271+0000","last_fullsized":"2023-10-09T06:16:43.947271+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958636+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:16:44.958636+0000","last_peered":"2023-10-09T06:16:44.958636+0000","last_clean":"2023-10-09T06:16:44.958636+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:16:44.958636+0000","last_undegraded":"2023-10-09T06:16:44.958636+0000","last_fullsized":"2023-10-09T06:16:44.958636+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963275+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:16:44.963275+0000","last_peered":"2023-10-09T06:16:44.963275+0000","last_clean":"2023-10-09T06:16:44.963275+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:16:44.963275+0000","last_undegraded":"2023-10-09T06:16:44.963275+0000","last_fullsized":"2023-10-09T06:16:44.963275+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951640+0000","last_change":"2023-10-09T06:13:02.835903+0000","last_active":"2023-10-09T06:16:43.951640+0000","last_peered":"2023-10-09T06:16:43.951640+0000","last_clean":"2023-10-09T06:16:43.951640+0000","last_became_active":"2023-10-09T06:13:02.835576+0000","last_became_peered":"2023-10-09T06:13:02.835576+0000","last_unstale":"2023-10-09T06:16:43.951640+0000","last_undegraded":"2023-10-09T06:16:43.951640+0000","last_fullsized":"2023-10-09T06:16:43.951640+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:31.729253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962991+0000","last_change":"2023-10-09T06:16:44.962991+0000","last_active":"2023-10-09T06:16:44.962991+0000","last_peered":"2023-10-09T06:16:44.962991+0000","last_clean":"2023-10-09T06:16:44.962991+0000","last_became_active":"2023-10-09T06:16:44.962656+0000","last_became_peered":"2023-10-09T06:16:44.962656+0000","last_unstale":"2023-10-09T06:16:44.962991+0000","last_undegraded":"2023-10-09T06:16:44.962991+0000","last_fullsized":"2023-10-09T06:16:44.962991+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:23.878640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956659+0000","last_change":"2023-10-09T06:16:44.956659+0000","last_active":"2023-10-09T06:16:44.956659+0000","last_peered":"2023-10-09T06:16:44.956659+0000","last_clean":"2023-10-09T06:16:44.956659+0000","last_became_active":"2023-10-09T06:16:44.956291+0000","last_became_peered":"2023-10-09T06:16:44.956291+0000","last_unstale":"2023-10-09T06:16:44.956659+0000","last_undegraded":"2023-10-09T06:16:44.956659+0000","last_fullsized":"2023-10-09T06:16:44.956659+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:40.513734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.940903+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:16:43.940903+0000","last_peered":"2023-10-09T06:16:43.940903+0000","last_clean":"2023-10-09T06:16:43.940903+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:16:43.940903+0000","last_undegraded":"2023-10-09T06:16:43.940903+0000","last_fullsized":"2023-10-09T06:16:43.940903+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943668+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T06:16:43.943668+0000","last_peered":"2023-10-09T06:16:43.943668+0000","last_clean":"2023-10-09T06:16:43.943668+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T06:16:43.943668+0000","last_undegraded":"2023-10-09T06:16:43.943668+0000","last_fullsized":"2023-10-09T06:16:43.943668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945933+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:16:43.945933+0000","last_peered":"2023-10-09T06:16:43.945933+0000","last_clean":"2023-10-09T06:16:43.945933+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:16:43.945933+0000","last_undegraded":"2023-10-09T06:16:43.945933+0000","last_fullsized":"2023-10-09T06:16:43.945933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949904+0000","last_change":"2023-10-09T06:15:11.874151+0000","last_active":"2023-10-09T06:16:43.949904+0000","last_peered":"2023-10-09T06:16:43.949904+0000","last_clean":"2023-10-09T06:16:43.949904+0000","last_became_active":"2023-10-09T06:15:11.873817+0000","last_became_peered":"2023-10-09T06:15:11.873817+0000","last_unstale":"2023-10-09T06:16:43.949904+0000","last_undegraded":"2023-10-09T06:16:43.949904+0000","last_fullsized":"2023-10-09T06:16:43.949904+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:18.675188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957656+0000","last_change":"2023-10-09T06:16:44.957656+0000","last_active":"2023-10-09T06:16:44.957656+0000","last_peered":"2023-10-09T06:16:44.957656+0000","last_clean":"2023-10-09T06:16:44.957656+0000","last_became_active":"2023-10-09T06:16:44.956987+0000","last_became_peered":"2023-10-09T06:16:44.956987+0000","last_unstale":"2023-10-09T06:16:44.957656+0000","last_undegraded":"2023-10-09T06:16:44.957656+0000","last_fullsized":"2023-10-09T06:16:44.957656+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:19.686112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961649+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.961649+0000","last_peered":"2023-10-09T06:16:44.961649+0000","last_clean":"2023-10-09T06:16:44.961649+0000","last_became_active":"2023-10-09T06:16:44.958834+0000","last_became_peered":"2023-10-09T06:16:44.958834+0000","last_unstale":"2023-10-09T06:16:44.961649+0000","last_undegraded":"2023-10-09T06:16:44.961649+0000","last_fullsized":"2023-10-09T06:16:44.961649+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:45.653750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.959015+0000","last_became_peered":"2023-10-09T06:16:44.959015+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:52.310157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946698+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T06:16:43.946698+0000","last_peered":"2023-10-09T06:16:43.946698+0000","last_clean":"2023-10-09T06:16:43.946698+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T06:16:43.946698+0000","last_undegraded":"2023-10-09T06:16:43.946698+0000","last_fullsized":"2023-10-09T06:16:43.946698+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956650+0000","last_change":"2023-10-09T06:16:44.956650+0000","last_active":"2023-10-09T06:16:44.956650+0000","last_peered":"2023-10-09T06:16:44.956650+0000","last_clean":"2023-10-09T06:16:44.956650+0000","last_became_active":"2023-10-09T06:16:44.956333+0000","last_became_peered":"2023-10-09T06:16:44.956333+0000","last_unstale":"2023-10-09T06:16:44.956650+0000","last_undegraded":"2023-10-09T06:16:44.956650+0000","last_fullsized":"2023-10-09T06:16:44.956650+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:45.636029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946241+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:16:43.946241+0000","last_peered":"2023-10-09T06:16:43.946241+0000","last_clean":"2023-10-09T06:16:43.946241+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:16:43.946241+0000","last_undegraded":"2023-10-09T06:16:43.946241+0000","last_fullsized":"2023-10-09T06:16:43.946241+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956576+0000","last_change":"2023-10-09T06:16:44.956576+0000","last_active":"2023-10-09T06:16:44.956576+0000","last_peered":"2023-10-09T06:16:44.956576+0000","last_clean":"2023-10-09T06:16:44.956576+0000","last_became_active":"2023-10-09T06:16:44.956199+0000","last_became_peered":"2023-10-09T06:16:44.956199+0000","last_unstale":"2023-10-09T06:16:44.956576+0000","last_undegraded":"2023-10-09T06:16:44.956576+0000","last_fullsized":"2023-10-09T06:16:44.956576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.170218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.950085+0000","last_change":"2023-10-09T06:08:16.097726+0000","last_active":"2023-10-09T06:16:43.950085+0000","last_peered":"2023-10-09T06:16:43.950085+0000","last_clean":"2023-10-09T06:16:43.950085+0000","last_became_active":"2023-10-09T06:08:16.097256+0000","last_became_peered":"2023-10-09T06:08:16.097256+0000","last_unstale":"2023-10-09T06:16:43.950085+0000","last_undegraded":"2023-10-09T06:16:43.950085+0000","last_fullsized":"2023-10-09T06:16:43.950085+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:20.223173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951497+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T06:16:43.951497+0000","last_peered":"2023-10-09T06:16:43.951497+0000","last_clean":"2023-10-09T06:16:43.951497+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T06:16:43.951497+0000","last_undegraded":"2023-10-09T06:16:43.951497+0000","last_fullsized":"2023-10-09T06:16:43.951497+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948746+0000","last_change":"2023-10-09T06:13:02.837546+0000","last_active":"2023-10-09T06:16:43.948746+0000","last_peered":"2023-10-09T06:16:43.948746+0000","last_clean":"2023-10-09T06:16:43.948746+0000","last_became_active":"2023-10-09T06:13:02.837188+0000","last_became_peered":"2023-10-09T06:13:02.837188+0000","last_unstale":"2023-10-09T06:16:43.948746+0000","last_undegraded":"2023-10-09T06:16:43.948746+0000","last_fullsized":"2023-10-09T06:16:43.948746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:38.707344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957272+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:16:44.957272+0000","last_peered":"2023-10-09T06:16:44.957272+0000","last_clean":"2023-10-09T06:16:44.957272+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:16:44.957272+0000","last_undegraded":"2023-10-09T06:16:44.957272+0000","last_fullsized":"2023-10-09T06:16:44.957272+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948427+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T06:16:43.948427+0000","last_peered":"2023-10-09T06:16:43.948427+0000","last_clean":"2023-10-09T06:16:43.948427+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T06:16:43.948427+0000","last_undegraded":"2023-10-09T06:16:43.948427+0000","last_fullsized":"2023-10-09T06:16:43.948427+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946500+0000","last_change":"2023-10-09T06:13:02.837565+0000","last_active":"2023-10-09T06:16:43.946500+0000","last_peered":"2023-10-09T06:16:43.946500+0000","last_clean":"2023-10-09T06:16:43.946500+0000","last_became_active":"2023-10-09T06:13:02.837258+0000","last_became_peered":"2023-10-09T06:13:02.837258+0000","last_unstale":"2023-10-09T06:16:43.946500+0000","last_undegraded":"2023-10-09T06:16:43.946500+0000","last_fullsized":"2023-10-09T06:16:43.946500+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:11.304315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963142+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:16:44.963142+0000","last_peered":"2023-10-09T06:16:44.963142+0000","last_clean":"2023-10-09T06:16:44.963142+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:16:44.963142+0000","last_undegraded":"2023-10-09T06:16:44.963142+0000","last_fullsized":"2023-10-09T06:16:44.963142+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942316+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:16:44.942316+0000","last_peered":"2023-10-09T06:16:44.942316+0000","last_clean":"2023-10-09T06:16:44.942316+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:16:44.942316+0000","last_undegraded":"2023-10-09T06:16:44.942316+0000","last_fullsized":"2023-10-09T06:16:44.942316+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960277+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:16:44.960277+0000","last_peered":"2023-10-09T06:16:44.960277+0000","last_clean":"2023-10-09T06:16:44.960277+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:16:44.960277+0000","last_undegraded":"2023-10-09T06:16:44.960277+0000","last_fullsized":"2023-10-09T06:16:44.960277+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960879+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:16:44.960879+0000","last_peered":"2023-10-09T06:16:44.960879+0000","last_clean":"2023-10-09T06:16:44.960879+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:16:44.960879+0000","last_undegraded":"2023-10-09T06:16:44.960879+0000","last_fullsized":"2023-10-09T06:16:44.960879+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946957+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:16:43.946957+0000","last_peered":"2023-10-09T06:16:43.946957+0000","last_clean":"2023-10-09T06:16:43.946957+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:16:43.946957+0000","last_undegraded":"2023-10-09T06:16:43.946957+0000","last_fullsized":"2023-10-09T06:16:43.946957+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943713+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:16:43.943713+0000","last_peered":"2023-10-09T06:16:43.943713+0000","last_clean":"2023-10-09T06:16:43.943713+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:16:43.943713+0000","last_undegraded":"2023-10-09T06:16:43.943713+0000","last_fullsized":"2023-10-09T06:16:43.943713+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945160+0000","last_change":"2023-10-09T06:15:11.869407+0000","last_active":"2023-10-09T06:16:43.945160+0000","last_peered":"2023-10-09T06:16:43.945160+0000","last_clean":"2023-10-09T06:16:43.945160+0000","last_became_active":"2023-10-09T06:15:11.869101+0000","last_became_peered":"2023-10-09T06:15:11.869101+0000","last_unstale":"2023-10-09T06:16:43.945160+0000","last_undegraded":"2023-10-09T06:16:43.945160+0000","last_fullsized":"2023-10-09T06:16:43.945160+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:34.742466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943981+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:16:43.943981+0000","last_peered":"2023-10-09T06:16:43.943981+0000","last_clean":"2023-10-09T06:16:43.943981+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:16:43.943981+0000","last_undegraded":"2023-10-09T06:16:43.943981+0000","last_fullsized":"2023-10-09T06:16:43.943981+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956207+0000","last_change":"2023-10-09T06:16:44.956207+0000","last_active":"2023-10-09T06:16:44.956207+0000","last_peered":"2023-10-09T06:16:44.956207+0000","last_clean":"2023-10-09T06:16:44.956207+0000","last_became_active":"2023-10-09T06:16:44.955884+0000","last_became_peered":"2023-10-09T06:16:44.955884+0000","last_unstale":"2023-10-09T06:16:44.956207+0000","last_undegraded":"2023-10-09T06:16:44.956207+0000","last_fullsized":"2023-10-09T06:16:44.956207+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:57.704149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":78,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963851+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:16:44.963851+0000","last_peered":"2023-10-09T06:16:44.963851+0000","last_clean":"2023-10-09T06:16:44.963851+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:16:44.963851+0000","last_undegraded":"2023-10-09T06:16:44.963851+0000","last_fullsized":"2023-10-09T06:16:44.963851+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949624+0000","last_change":"2023-10-09T06:13:02.834962+0000","last_active":"2023-10-09T06:16:44.949624+0000","last_peered":"2023-10-09T06:16:44.949624+0000","last_clean":"2023-10-09T06:16:44.949624+0000","last_became_active":"2023-10-09T06:13:02.834677+0000","last_became_peered":"2023-10-09T06:13:02.834677+0000","last_unstale":"2023-10-09T06:16:44.949624+0000","last_undegraded":"2023-10-09T06:16:44.949624+0000","last_fullsized":"2023-10-09T06:16:44.949624+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:35.835505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962615+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:16:44.962615+0000","last_peered":"2023-10-09T06:16:44.962615+0000","last_clean":"2023-10-09T06:16:44.962615+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:16:44.962615+0000","last_undegraded":"2023-10-09T06:16:44.962615+0000","last_fullsized":"2023-10-09T06:16:44.962615+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948689+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:16:44.948689+0000","last_peered":"2023-10-09T06:16:44.948689+0000","last_clean":"2023-10-09T06:16:44.948689+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:16:44.948689+0000","last_undegraded":"2023-10-09T06:16:44.948689+0000","last_fullsized":"2023-10-09T06:16:44.948689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.952582+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:16:43.952582+0000","last_peered":"2023-10-09T06:16:43.952582+0000","last_clean":"2023-10-09T06:16:43.952582+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:16:43.952582+0000","last_undegraded":"2023-10-09T06:16:43.952582+0000","last_fullsized":"2023-10-09T06:16:43.952582+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.944356+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:16:43.944356+0000","last_peered":"2023-10-09T06:16:43.944356+0000","last_clean":"2023-10-09T06:16:43.944356+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:16:43.944356+0000","last_undegraded":"2023-10-09T06:16:43.944356+0000","last_fullsized":"2023-10-09T06:16:43.944356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959456+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:16:44.959456+0000","last_peered":"2023-10-09T06:16:44.959456+0000","last_clean":"2023-10-09T06:16:44.959456+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:16:44.959456+0000","last_undegraded":"2023-10-09T06:16:44.959456+0000","last_fullsized":"2023-10-09T06:16:44.959456+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953059+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:16:44.953059+0000","last_peered":"2023-10-09T06:16:44.953059+0000","last_clean":"2023-10-09T06:16:44.953059+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:16:44.953059+0000","last_undegraded":"2023-10-09T06:16:44.953059+0000","last_fullsized":"2023-10-09T06:16:44.953059+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.950033+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:16:44.950033+0000","last_peered":"2023-10-09T06:16:44.950033+0000","last_clean":"2023-10-09T06:16:44.950033+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:16:44.950033+0000","last_undegraded":"2023-10-09T06:16:44.950033+0000","last_fullsized":"2023-10-09T06:16:44.950033+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948431+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:16:44.948431+0000","last_peered":"2023-10-09T06:16:44.948431+0000","last_clean":"2023-10-09T06:16:44.948431+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:16:44.948431+0000","last_undegraded":"2023-10-09T06:16:44.948431+0000","last_fullsized":"2023-10-09T06:16:44.948431+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951947+0000","last_change":"2023-10-09T06:15:11.886011+0000","last_active":"2023-10-09T06:16:43.951947+0000","last_peered":"2023-10-09T06:16:43.951947+0000","last_clean":"2023-10-09T06:16:43.951947+0000","last_became_active":"2023-10-09T06:15:11.885754+0000","last_became_peered":"2023-10-09T06:15:11.885754+0000","last_unstale":"2023-10-09T06:16:43.951947+0000","last_undegraded":"2023-10-09T06:16:43.951947+0000","last_fullsized":"2023-10-09T06:16:43.951947+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:33.874944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":490,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947610+0000","last_change":"2023-10-09T06:08:16.098085+0000","last_active":"2023-10-09T06:16:43.947610+0000","last_peered":"2023-10-09T06:16:43.947610+0000","last_clean":"2023-10-09T06:16:43.947610+0000","last_became_active":"2023-10-09T06:08:16.097675+0000","last_became_peered":"2023-10-09T06:08:16.097675+0000","last_unstale":"2023-10-09T06:16:43.947610+0000","last_undegraded":"2023-10-09T06:16:43.947610+0000","last_fullsized":"2023-10-09T06:16:43.947610+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:36.003332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961643+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:16:44.961643+0000","last_peered":"2023-10-09T06:16:44.961643+0000","last_clean":"2023-10-09T06:16:44.961643+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:16:44.961643+0000","last_undegraded":"2023-10-09T06:16:44.961643+0000","last_fullsized":"2023-10-09T06:16:44.961643+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947632+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T06:16:43.947632+0000","last_peered":"2023-10-09T06:16:43.947632+0000","last_clean":"2023-10-09T06:16:43.947632+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T06:16:43.947632+0000","last_undegraded":"2023-10-09T06:16:43.947632+0000","last_fullsized":"2023-10-09T06:16:43.947632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947513+0000","last_change":"2023-10-09T06:13:02.846478+0000","last_active":"2023-10-09T06:16:43.947513+0000","last_peered":"2023-10-09T06:16:43.947513+0000","last_clean":"2023-10-09T06:16:43.947513+0000","last_became_active":"2023-10-09T06:13:02.846169+0000","last_became_peered":"2023-10-09T06:13:02.846169+0000","last_unstale":"2023-10-09T06:16:43.947513+0000","last_undegraded":"2023-10-09T06:16:43.947513+0000","last_fullsized":"2023-10-09T06:16:43.947513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:18.710495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960166+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:16:44.960166+0000","last_peered":"2023-10-09T06:16:44.960166+0000","last_clean":"2023-10-09T06:16:44.960166+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:16:44.960166+0000","last_undegraded":"2023-10-09T06:16:44.960166+0000","last_fullsized":"2023-10-09T06:16:44.960166+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942690+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:16:44.942690+0000","last_peered":"2023-10-09T06:16:44.942690+0000","last_clean":"2023-10-09T06:16:44.942690+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:16:44.942690+0000","last_undegraded":"2023-10-09T06:16:44.942690+0000","last_fullsized":"2023-10-09T06:16:44.942690+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948529+0000","last_change":"2023-10-09T06:08:16.097472+0000","last_active":"2023-10-09T06:16:43.948529+0000","last_peered":"2023-10-09T06:16:43.948529+0000","last_clean":"2023-10-09T06:16:43.948529+0000","last_became_active":"2023-10-09T06:08:16.097159+0000","last_became_peered":"2023-10-09T06:08:16.097159+0000","last_unstale":"2023-10-09T06:16:43.948529+0000","last_undegraded":"2023-10-09T06:16:43.948529+0000","last_fullsized":"2023-10-09T06:16:43.948529+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:09.440710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949726+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:16:44.949726+0000","last_peered":"2023-10-09T06:16:44.949726+0000","last_clean":"2023-10-09T06:16:44.949726+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:16:44.949726+0000","last_undegraded":"2023-10-09T06:16:44.949726+0000","last_fullsized":"2023-10-09T06:16:44.949726+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948458+0000","last_change":"2023-10-09T06:08:16.100157+0000","last_active":"2023-10-09T06:16:43.948458+0000","last_peered":"2023-10-09T06:16:43.948458+0000","last_clean":"2023-10-09T06:16:43.948458+0000","last_became_active":"2023-10-09T06:08:16.099543+0000","last_became_peered":"2023-10-09T06:08:16.099543+0000","last_unstale":"2023-10-09T06:16:43.948458+0000","last_undegraded":"2023-10-09T06:16:43.948458+0000","last_fullsized":"2023-10-09T06:16:43.948458+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:08.543959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963419+0000","last_change":"2023-10-09T06:16:44.963419+0000","last_active":"2023-10-09T06:16:44.963419+0000","last_peered":"2023-10-09T06:16:44.963419+0000","last_clean":"2023-10-09T06:16:44.963419+0000","last_became_active":"2023-10-09T06:16:44.963126+0000","last_became_peered":"2023-10-09T06:16:44.963126+0000","last_unstale":"2023-10-09T06:16:44.963419+0000","last_undegraded":"2023-10-09T06:16:44.963419+0000","last_fullsized":"2023-10-09T06:16:44.963419+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:21.614499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948574+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:16:44.948574+0000","last_peered":"2023-10-09T06:16:44.948574+0000","last_clean":"2023-10-09T06:16:44.948574+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:16:44.948574+0000","last_undegraded":"2023-10-09T06:16:44.948574+0000","last_fullsized":"2023-10-09T06:16:44.948574+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.954350+0000","last_change":"2023-10-09T06:16:44.954350+0000","last_active":"2023-10-09T06:16:44.954350+0000","last_peered":"2023-10-09T06:16:44.954350+0000","last_clean":"2023-10-09T06:16:44.954350+0000","last_became_active":"2023-10-09T06:16:44.954027+0000","last_became_peered":"2023-10-09T06:16:44.954027+0000","last_unstale":"2023-10-09T06:16:44.954350+0000","last_undegraded":"2023-10-09T06:16:44.954350+0000","last_fullsized":"2023-10-09T06:16:44.954350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:18.938865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947159+0000","last_change":"2023-10-09T06:15:11.869497+0000","last_active":"2023-10-09T06:16:43.947159+0000","last_peered":"2023-10-09T06:16:43.947159+0000","last_clean":"2023-10-09T06:16:43.947159+0000","last_became_active":"2023-10-09T06:15:11.868971+0000","last_became_peered":"2023-10-09T06:15:11.868971+0000","last_unstale":"2023-10-09T06:16:43.947159+0000","last_undegraded":"2023-10-09T06:16:43.947159+0000","last_fullsized":"2023-10-09T06:16:43.947159+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:20.786443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942174+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:16:44.942174+0000","last_peered":"2023-10-09T06:16:44.942174+0000","last_clean":"2023-10-09T06:16:44.942174+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:16:44.942174+0000","last_undegraded":"2023-10-09T06:16:44.942174+0000","last_fullsized":"2023-10-09T06:16:44.942174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945269+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:16:43.945269+0000","last_peered":"2023-10-09T06:16:43.945269+0000","last_clean":"2023-10-09T06:16:43.945269+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:16:43.945269+0000","last_undegraded":"2023-10-09T06:16:43.945269+0000","last_fullsized":"2023-10-09T06:16:43.945269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":87,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948619+0000","last_change":"2023-10-09T06:08:16.100332+0000","last_active":"2023-10-09T06:16:43.948619+0000","last_peered":"2023-10-09T06:16:43.948619+0000","last_clean":"2023-10-09T06:16:43.948619+0000","last_became_active":"2023-10-09T06:08:16.099887+0000","last_became_peered":"2023-10-09T06:08:16.099887+0000","last_unstale":"2023-10-09T06:16:43.948619+0000","last_undegraded":"2023-10-09T06:16:43.948619+0000","last_fullsized":"2023-10-09T06:16:43.948619+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:36.739647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958199+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:16:44.958199+0000","last_peered":"2023-10-09T06:16:44.958199+0000","last_clean":"2023-10-09T06:16:44.958199+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:16:44.958199+0000","last_undegraded":"2023-10-09T06:16:44.958199+0000","last_fullsized":"2023-10-09T06:16:44.958199+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.953404+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:16:43.953404+0000","last_peered":"2023-10-09T06:16:43.953404+0000","last_clean":"2023-10-09T06:16:43.953404+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:16:43.953404+0000","last_undegraded":"2023-10-09T06:16:43.953404+0000","last_fullsized":"2023-10-09T06:16:43.953404+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956492+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:16:44.956492+0000","last_peered":"2023-10-09T06:16:44.956492+0000","last_clean":"2023-10-09T06:16:44.956492+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:16:44.956492+0000","last_undegraded":"2023-10-09T06:16:44.956492+0000","last_fullsized":"2023-10-09T06:16:44.956492+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":95,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.954232+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:16:43.954232+0000","last_peered":"2023-10-09T06:16:43.954232+0000","last_clean":"2023-10-09T06:16:43.954232+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:16:43.954232+0000","last_undegraded":"2023-10-09T06:16:43.954232+0000","last_fullsized":"2023-10-09T06:16:43.954232+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956388+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:16:44.956388+0000","last_peered":"2023-10-09T06:16:44.956388+0000","last_clean":"2023-10-09T06:16:44.956388+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:16:44.956388+0000","last_undegraded":"2023-10-09T06:16:44.956388+0000","last_fullsized":"2023-10-09T06:16:44.956388+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961893+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:16:44.961893+0000","last_peered":"2023-10-09T06:16:44.961893+0000","last_clean":"2023-10-09T06:16:44.961893+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:16:44.961893+0000","last_undegraded":"2023-10-09T06:16:44.961893+0000","last_fullsized":"2023-10-09T06:16:44.961893+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953905+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:16:44.953905+0000","last_peered":"2023-10-09T06:16:44.953905+0000","last_clean":"2023-10-09T06:16:44.953905+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:16:44.953905+0000","last_undegraded":"2023-10-09T06:16:44.953905+0000","last_fullsized":"2023-10-09T06:16:44.953905+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949161+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:16:43.949161+0000","last_peered":"2023-10-09T06:16:43.949161+0000","last_clean":"2023-10-09T06:16:43.949161+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:16:43.949161+0000","last_undegraded":"2023-10-09T06:16:43.949161+0000","last_fullsized":"2023-10-09T06:16:43.949161+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943302+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:16:43.943302+0000","last_peered":"2023-10-09T06:16:43.943302+0000","last_clean":"2023-10-09T06:16:43.943302+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:16:43.943302+0000","last_undegraded":"2023-10-09T06:16:43.943302+0000","last_fullsized":"2023-10-09T06:16:43.943302+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234243,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311644,"kb_used_data":2068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433508,"statfs":{"total":95995035648,"available":95675912192,"internally_reserved":0,"allocated":2117632,"data_stored":1556820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.482,"15min":1.272},"min":{"1min":0.37,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":0.74099999999999999,"5min":11.785,"15min":11.785},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":1.232,"15min":1.3919999999999999},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":0.70799999999999996,"5min":10.396000000000001,"15min":10.784000000000001},"last":0.67700000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.2480000000000002,"15min":1.728},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.542,"5min":10.542,"15min":10.542},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.899,"15min":1.5069999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.548,"5min":11.818,"15min":11.818},"last":0.64200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:17:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":2.016,"15min":2.0379999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.0859999999999999,"15min":2.1579999999999999},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.2549999999999999,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.73099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":17.396000000000001,"5min":4.96,"15min":3.6850000000000001},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":267.75099999999998,"5min":267.75099999999998,"15min":267.75099999999998},"last":0.63},{"interface":"front","average":{"1min":18.986999999999998,"5min":5.4269999999999996,"15min":3.5819999999999999},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":271.76299999999998,"5min":271.76299999999998,"15min":271.76299999999998},"last":0.747}]},{"osd":4,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.601,"15min":1.601},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.251},"max":{"1min":9.1140000000000008,"5min":11.853999999999999,"15min":41.527000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.256,"5min":1.0509999999999999,"15min":1.653},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":6.7629999999999999,"15min":41.466000000000001},"last":0.81799999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":1.915,"15min":1.865},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":8.4589999999999996,"5min":9.8629999999999995,"15min":49.357999999999997},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.762,"5min":1.3420000000000001,"15min":1.518},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.6769999999999996,"5min":11.837,"15min":49.237000000000002},"last":0.77200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.4390000000000001,"15min":1.51},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":8.2249999999999996,"5min":11.878,"15min":11.878},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.2150000000000001,"15min":1.5960000000000001},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.0380000000000003,"5min":7.9340000000000002,"15min":13.189},"last":0.78600000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495880,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311624,"kb_used_data":2048,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433528,"statfs":{"total":95995035648,"available":95675932672,"internally_reserved":0,"allocated":2097152,"data_stored":1556086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.659,"15min":1.6479999999999999},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.214},"max":{"1min":12.808,"5min":12.808,"15min":12.808},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.7789999999999999,"5min":1.984,"15min":1.7070000000000001},"min":{"1min":0.28299999999999997,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":12.847,"5min":12.847,"15min":12.847},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.5600000000000001,"15min":2.2879999999999998},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.5690000000000008,"5min":10.646000000000001,"15min":10.646000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.6850000000000001,"5min":2.4740000000000002,"15min":2.3719999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":9.6080000000000005,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.53700000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.3319999999999999,"15min":2.4380000000000002},"min":{"1min":0.41499999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6539999999999999,"5min":7.6539999999999999,"15min":7.6539999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.137,"5min":2.383,"15min":2.5910000000000002},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6549999999999998,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":0.65700000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":4.8639999999999999,"15min":3.8780000000000001},"min":{"1min":0.42599999999999999,"5min":0.28899999999999998,"15min":0.26700000000000002},"max":{"1min":9.0419999999999998,"5min":222.922,"15min":222.922},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.5640000000000001,"5min":4.4390000000000001,"15min":3.6099999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.30599999999999999},"max":{"1min":11.374000000000001,"5min":222.911,"15min":222.911},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.532,"15min":1.3120000000000001},"min":{"1min":0.375,"5min":0.252,"15min":0.245},"max":{"1min":5.3620000000000001,"5min":10.75,"15min":10.75},"last":0.376},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.51,"15min":1.5669999999999999},"min":{"1min":0.38400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.96,"5min":8.9239999999999995,"15min":12.567},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.474,"15min":1.47},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":4.1349999999999998,"5min":10.773,"15min":13.91},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6040000000000001,"15min":2.069},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.6779999999999999,"5min":11.798999999999999,"15min":159.06800000000001},"last":0.626}]},{"osd":7,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.1599999999999999,"15min":1.3149999999999999},"min":{"1min":0.33800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.962,"5min":10.962,"15min":14.837},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.6699999999999999,"15min":1.7929999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":8.9920000000000009,"5min":10.891,"15min":11.281000000000001},"last":0.57899999999999996}]}]},{"osd":1,"up_from":110,"seq":472446402617,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256336,"kb_used_data":1608,"kb_used_omap":4,"kb_used_meta":254715,"kb_avail":93488816,"statfs":{"total":95995035648,"available":95732547584,"internally_reserved":0,"allocated":1646592,"data_stored":981322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":260829070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.1989999999999998,"15min":2.5990000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.0939999999999994,"5min":10.726000000000001,"15min":10.726000000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":2.681,"15min":2.9940000000000002},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.526,"15min":1.6639999999999999},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0500000000000007,"5min":9.2769999999999992,"15min":9.2769999999999992},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.448,"15min":2.7330000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.1970000000000001,"5min":10.573,"15min":10.573},"last":0.315}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":3.0899999999999999,"15min":3.0659999999999998},"min":{"1min":0.29599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":11.044,"5min":11.044,"15min":11.044},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.5819999999999999,"15min":2.7839999999999998},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":12.464,"5min":13.487,"15min":13.487},"last":0.32700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.2280000000000002,"15min":2.8460000000000001},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1239999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.5169999999999999,"15min":2.907},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":10.723000000000001,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.4860000000000002,"15min":3.2629999999999999},"min":{"1min":0.39200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":10.92,"15min":10.92},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.7909999999999999,"15min":3.2109999999999999},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":2.7999999999999998,"15min":2.956},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.772,"5min":12.91,"15min":12.91},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.6080000000000001,"5min":2.2839999999999998,"15min":2.5699999999999998},"min":{"1min":0.437,"5min":0.374,"15min":0.374},"max":{"1min":6.3170000000000002,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":2.347,"15min":2.7490000000000001},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.0830000000000002,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.3400000000000001,"5min":2.0070000000000001,"15min":2.129},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":5.5599999999999996,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.56999999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024050,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410940,"kb_used_data":2292,"kb_used_omap":5,"kb_used_meta":408634,"kb_avail":93334212,"statfs":{"total":95995035648,"available":95574233088,"internally_reserved":0,"allocated":2347008,"data_stored":1585989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":418441850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.6930000000000001,"15min":1.8759999999999999},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":11.576000000000001,"5min":11.576000000000001,"15min":11.576000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.3700000000000001,"5min":2.7490000000000001,"15min":2.6619999999999999},"min":{"1min":0.27200000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.603,"5min":11.744999999999999,"15min":11.744999999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.488,"15min":2.6150000000000002},"min":{"1min":0.254,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.186,"5min":11.186,"15min":11.186},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.016,"5min":2.5230000000000001,"15min":2.6070000000000002},"min":{"1min":0.21299999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":21.198,"5min":21.198,"15min":21.198},"last":0.36299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":3.3149999999999999,"15min":3.8319999999999999},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.414,"5min":24.870000000000001,"15min":27.048999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2839999999999998,"15min":3.423},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.196999999999999,"5min":32.030000000000001,"15min":32.030000000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.5669999999999999,"15min":1.8029999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":4.5499999999999998,"5min":13.244,"15min":15.835000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2549999999999999,"15min":2.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.25700000000000001},"max":{"1min":8.7769999999999992,"5min":11.566000000000001,"15min":15.023},"last":0.441}]},{"osd":5,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.0750000000000002,"15min":1.9990000000000001},"min":{"1min":0.33200000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":11.222,"5min":13.285,"15min":15.858000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.7480000000000002,"5min":2.3780000000000001,"15min":2.2080000000000002},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.61499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.355,"15min":2.2090000000000001},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.088999999999999,"5min":21.088999999999999,"15min":32.079000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.1859999999999999,"15min":2.085},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.0270000000000001,"5min":13.315,"15min":15.893000000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.319,"15min":2.387},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.28899999999999998},"max":{"1min":8.8089999999999993,"5min":11.515000000000001,"15min":15.050000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":4.2309999999999999,"5min":2.4470000000000001,"15min":2.3399999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":21.135999999999999,"5min":21.135999999999999,"15min":32.055999999999997},"last":0.67700000000000005}]}]},{"osd":2,"up_from":128,"seq":549755813919,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196712,"kb_used_data":1568,"kb_used_omap":7,"kb_used_meta":195128,"kb_avail":93548440,"statfs":{"total":95995035648,"available":95793602560,"internally_reserved":0,"allocated":1605632,"data_stored":975398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":199811153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.863,"15min":1.9099999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.156000000000001,"5min":12.156000000000001,"15min":12.156000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.145,"5min":2.4159999999999999,"15min":2.6280000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.113,"5min":12.113,"15min":12.113},"last":0.35499999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":11.619999999999999,"5min":3.8079999999999998,"15min":2.5059999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":222.50299999999999,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":11.102,"5min":4.4299999999999997,"15min":3.3180000000000001},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":222.19399999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 06:16:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":2.8759999999999999,"15min":2.8759999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.435,"5min":10.435,"15min":10.435},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.0179999999999998,"5min":2.0179999999999998,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6299999999999999,"5min":6.6299999999999999,"15min":6.6299999999999999},"last":0.32400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.879,"15min":2.0859999999999999},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.1560000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.1320000000000001,"15min":2.3799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.6230000000000002,"5min":15.183,"15min":15.183},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.591,"15min":1.744},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.1960000000000002,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.3039999999999998,"15min":2.524},"min":{"1min":0.35499999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.085,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.0779999999999998,"15min":2.3149999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.6890000000000001,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.5509999999999999,"15min":1.6599999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.1509999999999998,"5min":7.327,"15min":7.327},"last":0.57999999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.5980000000000001,"15min":1.6899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.1260000000000003,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.734,"15min":3.089},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.2229999999999999,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.69699999999999995}]}]},{"osd":3,"up_from":149,"seq":639950127116,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111000,"kb_used_data":1488,"kb_used_omap":2,"kb_used_meta":109501,"kb_avail":93634152,"statfs":{"total":95995035648,"available":95881371648,"internally_reserved":0,"allocated":1523712,"data_stored":966500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":112129077},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36299999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591248,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93153904,"statfs":{"total":95995035648,"available":95389597696,"internally_reserved":0,"allocated":1515520,"data_stored":966117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.6879999999999999,"15min":1.409},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.4190000000000005,"5min":8.5990000000000002,"15min":15.484999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.3860000000000001,"5min":2.226,"15min":1.7729999999999999},"min":{"1min":0.28499999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.4380000000000006,"5min":11.234,"15min":15.289},"last":0.64200000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.25600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.7050000000000001,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.28,"5min":1.542,"15min":1.619},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1449999999999996,"5min":8.7509999999999994,"15min":8.7509999999999994},"last":0.624}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.784,"15min":1.865},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9100000000000001,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.35599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.7559999999999998,"5min":7.2830000000000004,"15min":7.2830000000000004},"last":0.49299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":3.0329999999999999,"15min":4.125},"min":{"1min":0.318,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":11.31,"5min":60.414999999999999,"15min":236.78},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.6339999999999999,"15min":4.0140000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.29499999999999998},"max":{"1min":8.2010000000000005,"5min":60.375999999999998,"15min":236.76599999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.3660000000000001,"15min":1.3169999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":4.4779999999999998,"5min":9.6790000000000003,"15min":19.687999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.5820000000000001,"15min":1.4319999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":2.8100000000000001,"5min":11.282,"15min":11.282},"last":0.68300000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:16:34 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.1439999999999999,"15min":1.359},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.907,"5min":9.7680000000000007,"15min":20.390999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.923,"5min":1.7450000000000001,"15min":1.623},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.25},"max":{"1min":14.202999999999999,"5min":14.202999999999999,"15min":26.137},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.21,"15min":1.3049999999999999},"min":{"1min":0.379,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":15.911},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.2010000000000001,"5min":1.4259999999999999,"15min":1.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.28799999999999998},"max":{"1min":12.965999999999999,"5min":12.965999999999999,"15min":15.333},"last":0.70099999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757517,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603520,"kb_used_data":1464,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141632,"statfs":{"total":95995035648,"available":95377031168,"internally_reserved":0,"allocated":1499136,"data_stored":961803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.5860000000000001,"15min":1.651},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.215},"max":{"1min":0.81499999999999995,"5min":8.5139999999999993,"15min":18.637},"last":0.505},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.905,"15min":1.5429999999999999},"min":{"1min":0.34000000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.82099999999999995,"5min":9.9939999999999998,"15min":9.9939999999999998},"last":0.32800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.032,"15min":2.3420000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.8420000000000001,"5min":11.292,"15min":11.292},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.218,"15min":2.1920000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6349999999999998,"5min":7.9539999999999997,"15min":7.9539999999999997},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.1829999999999998,"15min":2.3860000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.0519999999999996,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0779999999999998,"15min":2.2610000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":5.6790000000000003,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.8039999999999998,"15min":3.2080000000000002},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":12.106,"5min":12.106,"15min":128.035},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.96,"5min":2.327,"15min":3.0289999999999999},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.4020000000000001,"5min":10.324999999999999,"15min":128.07300000000001},"last":0.55800000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.696,"15min":1.5940000000000001},"min":{"1min":0.35799999999999998,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":5.6360000000000001,"5min":8.7850000000000001,"15min":19.847999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.276,"15min":1.353},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":5.1200000000000001,"5min":7.7290000000000001,"15min":20.106000000000002},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:17:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.5089999999999999,"15min":1.492},"min":{"1min":0.438,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":1.988,"5min":8.8149999999999995,"15min":20.027000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.456,"15min":1.3560000000000001},"min":{"1min":0.46300000000000002,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":0.85899999999999999,"5min":7.3479999999999999,"15min":12.726000000000001},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.407,"15min":1.6100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.8409999999999993,"5min":8.8409999999999993,"15min":23.398},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.363,"5min":1.323,"15min":1.4970000000000001},"min":{"1min":0.40699999999999997,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":7.0049999999999999,"5min":7.7809999999999997,"15min":20.131},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:38.728 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:17:39.077 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:39.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:38 smithi107 ceph-mon[212750]: from='client.45200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:39.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:38 smithi107 ceph-mon[220921]: from='client.45200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:38 smithi150 ceph-mon[194178]: from='client.45200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:40.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:39 smithi107 ceph-mon[212750]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:40.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:39 smithi107 ceph-mon[220921]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:39 smithi150 ceph-mon[194178]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:40.718 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:40.720 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:17:41.297 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":507,"stamp":"2023-10-09T06:17:38.885056+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2793024,"kb_used_data":14016,"kb_used_omap":20,"kb_used_meta":2778923,"kb_avail":747168192,"statfs":{"total":767960285184,"available":765100228608,"internally_reserved":0,"allocated":14352384,"data_stored":9550035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2845617806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002471"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":129,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943512+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:16:43.943512+0000","last_peered":"2023-10-09T06:16:43.943512+0000","last_clean":"2023-10-09T06:16:43.943512+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:16:43.943512+0000","last_undegraded":"2023-10-09T06:16:43.943512+0000","last_fullsized":"2023-10-09T06:16:43.943512+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958058+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:16:44.958058+0000","last_peered":"2023-10-09T06:16:44.958058+0000","last_clean":"2023-10-09T06:16:44.958058+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:16:44.958058+0000","last_undegraded":"2023-10-09T06:16:44.958058+0000","last_fullsized":"2023-10-09T06:16:44.958058+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.955580+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:16:44.955580+0000","last_peered":"2023-10-09T06:16:44.955580+0000","last_clean":"2023-10-09T06:16:44.955580+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:16:44.955580+0000","last_undegraded":"2023-10-09T06:16:44.955580+0000","last_fullsized":"2023-10-09T06:16:44.955580+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957836+0000","last_change":"2023-10-09T06:16:44.957836+0000","last_active":"2023-10-09T06:16:44.957836+0000","last_peered":"2023-10-09T06:16:44.957836+0000","last_clean":"2023-10-09T06:16:44.957836+0000","last_became_active":"2023-10-09T06:16:44.957657+0000","last_became_peered":"2023-10-09T06:16:44.957657+0000","last_unstale":"2023-10-09T06:16:44.957836+0000","last_undegraded":"2023-10-09T06:16:44.957836+0000","last_fullsized":"2023-10-09T06:16:44.957836+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:15.752807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959728+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:16:44.959728+0000","last_peered":"2023-10-09T06:16:44.959728+0000","last_clean":"2023-10-09T06:16:44.959728+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:16:44.959728+0000","last_undegraded":"2023-10-09T06:16:44.959728+0000","last_fullsized":"2023-10-09T06:16:44.959728+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947271+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:16:43.947271+0000","last_peered":"2023-10-09T06:16:43.947271+0000","last_clean":"2023-10-09T06:16:43.947271+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:16:43.947271+0000","last_undegraded":"2023-10-09T06:16:43.947271+0000","last_fullsized":"2023-10-09T06:16:43.947271+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958636+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:16:44.958636+0000","last_peered":"2023-10-09T06:16:44.958636+0000","last_clean":"2023-10-09T06:16:44.958636+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:16:44.958636+0000","last_undegraded":"2023-10-09T06:16:44.958636+0000","last_fullsized":"2023-10-09T06:16:44.958636+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963275+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:16:44.963275+0000","last_peered":"2023-10-09T06:16:44.963275+0000","last_clean":"2023-10-09T06:16:44.963275+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:16:44.963275+0000","last_undegraded":"2023-10-09T06:16:44.963275+0000","last_fullsized":"2023-10-09T06:16:44.963275+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951640+0000","last_change":"2023-10-09T06:13:02.835903+0000","last_active":"2023-10-09T06:16:43.951640+0000","last_peered":"2023-10-09T06:16:43.951640+0000","last_clean":"2023-10-09T06:16:43.951640+0000","last_became_active":"2023-10-09T06:13:02.835576+0000","last_became_peered":"2023-10-09T06:13:02.835576+0000","last_unstale":"2023-10-09T06:16:43.951640+0000","last_undegraded":"2023-10-09T06:16:43.951640+0000","last_fullsized":"2023-10-09T06:16:43.951640+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:31.729253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962991+0000","last_change":"2023-10-09T06:16:44.962991+0000","last_active":"2023-10-09T06:16:44.962991+0000","last_peered":"2023-10-09T06:16:44.962991+0000","last_clean":"2023-10-09T06:16:44.962991+0000","last_became_active":"2023-10-09T06:16:44.962656+0000","last_became_peered":"2023-10-09T06:16:44.962656+0000","last_unstale":"2023-10-09T06:16:44.962991+0000","last_undegraded":"2023-10-09T06:16:44.962991+0000","last_fullsized":"2023-10-09T06:16:44.962991+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:23.878640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956659+0000","last_change":"2023-10-09T06:16:44.956659+0000","last_active":"2023-10-09T06:16:44.956659+0000","last_peered":"2023-10-09T06:16:44.956659+0000","last_clean":"2023-10-09T06:16:44.956659+0000","last_became_active":"2023-10-09T06:16:44.956291+0000","last_became_peered":"2023-10-09T06:16:44.956291+0000","last_unstale":"2023-10-09T06:16:44.956659+0000","last_undegraded":"2023-10-09T06:16:44.956659+0000","last_fullsized":"2023-10-09T06:16:44.956659+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:40.513734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.940903+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:16:43.940903+0000","last_peered":"2023-10-09T06:16:43.940903+0000","last_clean":"2023-10-09T06:16:43.940903+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:16:43.940903+0000","last_undegraded":"2023-10-09T06:16:43.940903+0000","last_fullsized":"2023-10-09T06:16:43.940903+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943668+0000","last_change":"2023-10-09T05:57:53.257493+0000","last_active":"2023-10-09T06:16:43.943668+0000","last_peered":"2023-10-09T06:16:43.943668+0000","last_clean":"2023-10-09T06:16:43.943668+0000","last_became_active":"2023-10-09T05:57:53.257222+0000","last_became_peered":"2023-10-09T05:57:53.257222+0000","last_unstale":"2023-10-09T06:16:43.943668+0000","last_undegraded":"2023-10-09T06:16:43.943668+0000","last_fullsized":"2023-10-09T06:16:43.943668+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:00.623600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945933+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:16:43.945933+0000","last_peered":"2023-10-09T06:16:43.945933+0000","last_clean":"2023-10-09T06:16:43.945933+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:16:43.945933+0000","last_undegraded":"2023-10-09T06:16:43.945933+0000","last_fullsized":"2023-10-09T06:16:43.945933+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949904+0000","last_change":"2023-10-09T06:15:11.874151+0000","last_active":"2023-10-09T06:16:43.949904+0000","last_peered":"2023-10-09T06:16:43.949904+0000","last_clean":"2023-10-09T06:16:43.949904+0000","last_became_active":"2023-10-09T06:15:11.873817+0000","last_became_peered":"2023-10-09T06:15:11.873817+0000","last_unstale":"2023-10-09T06:16:43.949904+0000","last_undegraded":"2023-10-09T06:16:43.949904+0000","last_fullsized":"2023-10-09T06:16:43.949904+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:18.675188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957656+0000","last_change":"2023-10-09T06:16:44.957656+0000","last_active":"2023-10-09T06:16:44.957656+0000","last_peered":"2023-10-09T06:16:44.957656+0000","last_clean":"2023-10-09T06:16:44.957656+0000","last_became_active":"2023-10-09T06:16:44.956987+0000","last_became_peered":"2023-10-09T06:16:44.956987+0000","last_unstale":"2023-10-09T06:16:44.957656+0000","last_undegraded":"2023-10-09T06:16:44.957656+0000","last_fullsized":"2023-10-09T06:16:44.957656+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:19.686112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961649+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.961649+0000","last_peered":"2023-10-09T06:16:44.961649+0000","last_clean":"2023-10-09T06:16:44.961649+0000","last_became_active":"2023-10-09T06:16:44.958834+0000","last_became_peered":"2023-10-09T06:16:44.958834+0000","last_unstale":"2023-10-09T06:16:44.961649+0000","last_undegraded":"2023-10-09T06:16:44.961649+0000","last_fullsized":"2023-10-09T06:16:44.961649+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:45.653750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959372+0000","last_change":"2023-10-09T06:16:44.959372+0000","last_active":"2023-10-09T06:16:44.959372+0000","last_peered":"2023-10-09T06:16:44.959372+0000","last_clean":"2023-10-09T06:16:44.959372+0000","last_became_active":"2023-10-09T06:16:44.959015+0000","last_became_peered":"2023-10-09T06:16:44.959015+0000","last_unstale":"2023-10-09T06:16:44.959372+0000","last_undegraded":"2023-10-09T06:16:44.959372+0000","last_fullsized":"2023-10-09T06:16:44.959372+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:52.310157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946698+0000","last_change":"2023-10-09T05:57:53.260396+0000","last_active":"2023-10-09T06:16:43.946698+0000","last_peered":"2023-10-09T06:16:43.946698+0000","last_clean":"2023-10-09T06:16:43.946698+0000","last_became_active":"2023-10-09T05:57:53.260115+0000","last_became_peered":"2023-10-09T05:57:53.260115+0000","last_unstale":"2023-10-09T06:16:43.946698+0000","last_undegraded":"2023-10-09T06:16:43.946698+0000","last_fullsized":"2023-10-09T06:16:43.946698+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:42:08.123009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956650+0000","last_change":"2023-10-09T06:16:44.956650+0000","last_active":"2023-10-09T06:16:44.956650+0000","last_peered":"2023-10-09T06:16:44.956650+0000","last_clean":"2023-10-09T06:16:44.956650+0000","last_became_active":"2023-10-09T06:16:44.956333+0000","last_became_peered":"2023-10-09T06:16:44.956333+0000","last_unstale":"2023-10-09T06:16:44.956650+0000","last_undegraded":"2023-10-09T06:16:44.956650+0000","last_fullsized":"2023-10-09T06:16:44.956650+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:42:45.636029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":118,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946241+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:16:43.946241+0000","last_peered":"2023-10-09T06:16:43.946241+0000","last_clean":"2023-10-09T06:16:43.946241+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:16:43.946241+0000","last_undegraded":"2023-10-09T06:16:43.946241+0000","last_fullsized":"2023-10-09T06:16:43.946241+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956576+0000","last_change":"2023-10-09T06:16:44.956576+0000","last_active":"2023-10-09T06:16:44.956576+0000","last_peered":"2023-10-09T06:16:44.956576+0000","last_clean":"2023-10-09T06:16:44.956576+0000","last_became_active":"2023-10-09T06:16:44.956199+0000","last_became_peered":"2023-10-09T06:16:44.956199+0000","last_unstale":"2023-10-09T06:16:44.956576+0000","last_undegraded":"2023-10-09T06:16:44.956576+0000","last_fullsized":"2023-10-09T06:16:44.956576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:09.170218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.950085+0000","last_change":"2023-10-09T06:08:16.097726+0000","last_active":"2023-10-09T06:16:43.950085+0000","last_peered":"2023-10-09T06:16:43.950085+0000","last_clean":"2023-10-09T06:16:43.950085+0000","last_became_active":"2023-10-09T06:08:16.097256+0000","last_became_peered":"2023-10-09T06:08:16.097256+0000","last_unstale":"2023-10-09T06:16:43.950085+0000","last_undegraded":"2023-10-09T06:16:43.950085+0000","last_fullsized":"2023-10-09T06:16:43.950085+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:20.223173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951497+0000","last_change":"2023-10-09T05:57:53.536243+0000","last_active":"2023-10-09T06:16:43.951497+0000","last_peered":"2023-10-09T06:16:43.951497+0000","last_clean":"2023-10-09T06:16:43.951497+0000","last_became_active":"2023-10-09T05:57:53.535972+0000","last_became_peered":"2023-10-09T05:57:53.535972+0000","last_unstale":"2023-10-09T06:16:43.951497+0000","last_undegraded":"2023-10-09T06:16:43.951497+0000","last_fullsized":"2023-10-09T06:16:43.951497+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:06:39.358921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948746+0000","last_change":"2023-10-09T06:13:02.837546+0000","last_active":"2023-10-09T06:16:43.948746+0000","last_peered":"2023-10-09T06:16:43.948746+0000","last_clean":"2023-10-09T06:16:43.948746+0000","last_became_active":"2023-10-09T06:13:02.837188+0000","last_became_peered":"2023-10-09T06:13:02.837188+0000","last_unstale":"2023-10-09T06:16:43.948746+0000","last_undegraded":"2023-10-09T06:16:43.948746+0000","last_fullsized":"2023-10-09T06:16:43.948746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:38.707344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.957272+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:16:44.957272+0000","last_peered":"2023-10-09T06:16:44.957272+0000","last_clean":"2023-10-09T06:16:44.957272+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:16:44.957272+0000","last_undegraded":"2023-10-09T06:16:44.957272+0000","last_fullsized":"2023-10-09T06:16:44.957272+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948427+0000","last_change":"2023-10-09T05:57:53.254336+0000","last_active":"2023-10-09T06:16:43.948427+0000","last_peered":"2023-10-09T06:16:43.948427+0000","last_clean":"2023-10-09T06:16:43.948427+0000","last_became_active":"2023-10-09T05:57:53.254066+0000","last_became_peered":"2023-10-09T05:57:53.254066+0000","last_unstale":"2023-10-09T06:16:43.948427+0000","last_undegraded":"2023-10-09T06:16:43.948427+0000","last_fullsized":"2023-10-09T06:16:43.948427+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:45:34.467543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946500+0000","last_change":"2023-10-09T06:13:02.837565+0000","last_active":"2023-10-09T06:16:43.946500+0000","last_peered":"2023-10-09T06:16:43.946500+0000","last_clean":"2023-10-09T06:16:43.946500+0000","last_became_active":"2023-10-09T06:13:02.837258+0000","last_became_peered":"2023-10-09T06:13:02.837258+0000","last_unstale":"2023-10-09T06:16:43.946500+0000","last_undegraded":"2023-10-09T06:16:43.946500+0000","last_fullsized":"2023-10-09T06:16:43.946500+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:11.304315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963142+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:16:44.963142+0000","last_peered":"2023-10-09T06:16:44.963142+0000","last_clean":"2023-10-09T06:16:44.963142+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:16:44.963142+0000","last_undegraded":"2023-10-09T06:16:44.963142+0000","last_fullsized":"2023-10-09T06:16:44.963142+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942316+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:16:44.942316+0000","last_peered":"2023-10-09T06:16:44.942316+0000","last_clean":"2023-10-09T06:16:44.942316+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:16:44.942316+0000","last_undegraded":"2023-10-09T06:16:44.942316+0000","last_fullsized":"2023-10-09T06:16:44.942316+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960277+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:16:44.960277+0000","last_peered":"2023-10-09T06:16:44.960277+0000","last_clean":"2023-10-09T06:16:44.960277+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:16:44.960277+0000","last_undegraded":"2023-10-09T06:16:44.960277+0000","last_fullsized":"2023-10-09T06:16:44.960277+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960879+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:16:44.960879+0000","last_peered":"2023-10-09T06:16:44.960879+0000","last_clean":"2023-10-09T06:16:44.960879+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:16:44.960879+0000","last_undegraded":"2023-10-09T06:16:44.960879+0000","last_fullsized":"2023-10-09T06:16:44.960879+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.946957+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:16:43.946957+0000","last_peered":"2023-10-09T06:16:43.946957+0000","last_clean":"2023-10-09T06:16:43.946957+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:16:43.946957+0000","last_undegraded":"2023-10-09T06:16:43.946957+0000","last_fullsized":"2023-10-09T06:16:43.946957+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943713+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:16:43.943713+0000","last_peered":"2023-10-09T06:16:43.943713+0000","last_clean":"2023-10-09T06:16:43.943713+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:16:43.943713+0000","last_undegraded":"2023-10-09T06:16:43.943713+0000","last_fullsized":"2023-10-09T06:16:43.943713+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945160+0000","last_change":"2023-10-09T06:15:11.869407+0000","last_active":"2023-10-09T06:16:43.945160+0000","last_peered":"2023-10-09T06:16:43.945160+0000","last_clean":"2023-10-09T06:16:43.945160+0000","last_became_active":"2023-10-09T06:15:11.869101+0000","last_became_peered":"2023-10-09T06:15:11.869101+0000","last_unstale":"2023-10-09T06:16:43.945160+0000","last_undegraded":"2023-10-09T06:16:43.945160+0000","last_fullsized":"2023-10-09T06:16:43.945160+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:34.742466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943981+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:16:43.943981+0000","last_peered":"2023-10-09T06:16:43.943981+0000","last_clean":"2023-10-09T06:16:43.943981+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:16:43.943981+0000","last_undegraded":"2023-10-09T06:16:43.943981+0000","last_fullsized":"2023-10-09T06:16:43.943981+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956207+0000","last_change":"2023-10-09T06:16:44.956207+0000","last_active":"2023-10-09T06:16:44.956207+0000","last_peered":"2023-10-09T06:16:44.956207+0000","last_clean":"2023-10-09T06:16:44.956207+0000","last_became_active":"2023-10-09T06:16:44.955884+0000","last_became_peered":"2023-10-09T06:16:44.955884+0000","last_unstale":"2023-10-09T06:16:44.956207+0000","last_undegraded":"2023-10-09T06:16:44.956207+0000","last_fullsized":"2023-10-09T06:16:44.956207+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:34:57.704149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":78,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963851+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:16:44.963851+0000","last_peered":"2023-10-09T06:16:44.963851+0000","last_clean":"2023-10-09T06:16:44.963851+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:16:44.963851+0000","last_undegraded":"2023-10-09T06:16:44.963851+0000","last_fullsized":"2023-10-09T06:16:44.963851+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949624+0000","last_change":"2023-10-09T06:13:02.834962+0000","last_active":"2023-10-09T06:16:44.949624+0000","last_peered":"2023-10-09T06:16:44.949624+0000","last_clean":"2023-10-09T06:16:44.949624+0000","last_became_active":"2023-10-09T06:13:02.834677+0000","last_became_peered":"2023-10-09T06:13:02.834677+0000","last_unstale":"2023-10-09T06:16:44.949624+0000","last_undegraded":"2023-10-09T06:16:44.949624+0000","last_fullsized":"2023-10-09T06:16:44.949624+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:35.835505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.962615+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:16:44.962615+0000","last_peered":"2023-10-09T06:16:44.962615+0000","last_clean":"2023-10-09T06:16:44.962615+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:16:44.962615+0000","last_undegraded":"2023-10-09T06:16:44.962615+0000","last_fullsized":"2023-10-09T06:16:44.962615+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948689+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:16:44.948689+0000","last_peered":"2023-10-09T06:16:44.948689+0000","last_clean":"2023-10-09T06:16:44.948689+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:16:44.948689+0000","last_undegraded":"2023-10-09T06:16:44.948689+0000","last_fullsized":"2023-10-09T06:16:44.948689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.952582+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:16:43.952582+0000","last_peered":"2023-10-09T06:16:43.952582+0000","last_clean":"2023-10-09T06:16:43.952582+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:16:43.952582+0000","last_undegraded":"2023-10-09T06:16:43.952582+0000","last_fullsized":"2023-10-09T06:16:43.952582+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.944356+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:16:43.944356+0000","last_peered":"2023-10-09T06:16:43.944356+0000","last_clean":"2023-10-09T06:16:43.944356+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:16:43.944356+0000","last_undegraded":"2023-10-09T06:16:43.944356+0000","last_fullsized":"2023-10-09T06:16:43.944356+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.959456+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:16:44.959456+0000","last_peered":"2023-10-09T06:16:44.959456+0000","last_clean":"2023-10-09T06:16:44.959456+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:16:44.959456+0000","last_undegraded":"2023-10-09T06:16:44.959456+0000","last_fullsized":"2023-10-09T06:16:44.959456+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953059+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:16:44.953059+0000","last_peered":"2023-10-09T06:16:44.953059+0000","last_clean":"2023-10-09T06:16:44.953059+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:16:44.953059+0000","last_undegraded":"2023-10-09T06:16:44.953059+0000","last_fullsized":"2023-10-09T06:16:44.953059+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.950033+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:16:44.950033+0000","last_peered":"2023-10-09T06:16:44.950033+0000","last_clean":"2023-10-09T06:16:44.950033+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:16:44.950033+0000","last_undegraded":"2023-10-09T06:16:44.950033+0000","last_fullsized":"2023-10-09T06:16:44.950033+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948431+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:16:44.948431+0000","last_peered":"2023-10-09T06:16:44.948431+0000","last_clean":"2023-10-09T06:16:44.948431+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:16:44.948431+0000","last_undegraded":"2023-10-09T06:16:44.948431+0000","last_fullsized":"2023-10-09T06:16:44.948431+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.951947+0000","last_change":"2023-10-09T06:15:11.886011+0000","last_active":"2023-10-09T06:16:43.951947+0000","last_peered":"2023-10-09T06:16:43.951947+0000","last_clean":"2023-10-09T06:16:43.951947+0000","last_became_active":"2023-10-09T06:15:11.885754+0000","last_became_peered":"2023-10-09T06:15:11.885754+0000","last_unstale":"2023-10-09T06:16:43.951947+0000","last_undegraded":"2023-10-09T06:16:43.951947+0000","last_fullsized":"2023-10-09T06:16:43.951947+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:33.874944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":490,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947610+0000","last_change":"2023-10-09T06:08:16.098085+0000","last_active":"2023-10-09T06:16:43.947610+0000","last_peered":"2023-10-09T06:16:43.947610+0000","last_clean":"2023-10-09T06:16:43.947610+0000","last_became_active":"2023-10-09T06:08:16.097675+0000","last_became_peered":"2023-10-09T06:08:16.097675+0000","last_unstale":"2023-10-09T06:16:43.947610+0000","last_undegraded":"2023-10-09T06:16:43.947610+0000","last_fullsized":"2023-10-09T06:16:43.947610+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:36.003332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961643+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:16:44.961643+0000","last_peered":"2023-10-09T06:16:44.961643+0000","last_clean":"2023-10-09T06:16:44.961643+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:16:44.961643+0000","last_undegraded":"2023-10-09T06:16:44.961643+0000","last_fullsized":"2023-10-09T06:16:44.961643+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947632+0000","last_change":"2023-10-09T05:57:54.570524+0000","last_active":"2023-10-09T06:16:43.947632+0000","last_peered":"2023-10-09T06:16:43.947632+0000","last_clean":"2023-10-09T06:16:43.947632+0000","last_became_active":"2023-10-09T05:57:54.570227+0000","last_became_peered":"2023-10-09T05:57:54.570227+0000","last_unstale":"2023-10-09T06:16:43.947632+0000","last_undegraded":"2023-10-09T06:16:43.947632+0000","last_fullsized":"2023-10-09T06:16:43.947632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:00:02.088476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947513+0000","last_change":"2023-10-09T06:13:02.846478+0000","last_active":"2023-10-09T06:16:43.947513+0000","last_peered":"2023-10-09T06:16:43.947513+0000","last_clean":"2023-10-09T06:16:43.947513+0000","last_became_active":"2023-10-09T06:13:02.846169+0000","last_became_peered":"2023-10-09T06:13:02.846169+0000","last_unstale":"2023-10-09T06:16:43.947513+0000","last_undegraded":"2023-10-09T06:16:43.947513+0000","last_fullsized":"2023-10-09T06:16:43.947513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:18.710495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.960166+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:16:44.960166+0000","last_peered":"2023-10-09T06:16:44.960166+0000","last_clean":"2023-10-09T06:16:44.960166+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:16:44.960166+0000","last_undegraded":"2023-10-09T06:16:44.960166+0000","last_fullsized":"2023-10-09T06:16:44.960166+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942690+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:16:44.942690+0000","last_peered":"2023-10-09T06:16:44.942690+0000","last_clean":"2023-10-09T06:16:44.942690+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:16:44.942690+0000","last_undegraded":"2023-10-09T06:16:44.942690+0000","last_fullsized":"2023-10-09T06:16:44.942690+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948529+0000","last_change":"2023-10-09T06:08:16.097472+0000","last_active":"2023-10-09T06:16:43.948529+0000","last_peered":"2023-10-09T06:16:43.948529+0000","last_clean":"2023-10-09T06:16:43.948529+0000","last_became_active":"2023-10-09T06:08:16.097159+0000","last_became_peered":"2023-10-09T06:08:16.097159+0000","last_unstale":"2023-10-09T06:16:43.948529+0000","last_undegraded":"2023-10-09T06:16:43.948529+0000","last_fullsized":"2023-10-09T06:16:43.948529+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:09.440710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.949726+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:16:44.949726+0000","last_peered":"2023-10-09T06:16:44.949726+0000","last_clean":"2023-10-09T06:16:44.949726+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:16:44.949726+0000","last_undegraded":"2023-10-09T06:16:44.949726+0000","last_fullsized":"2023-10-09T06:16:44.949726+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948458+0000","last_change":"2023-10-09T06:08:16.100157+0000","last_active":"2023-10-09T06:16:43.948458+0000","last_peered":"2023-10-09T06:16:43.948458+0000","last_clean":"2023-10-09T06:16:43.948458+0000","last_became_active":"2023-10-09T06:08:16.099543+0000","last_became_peered":"2023-10-09T06:08:16.099543+0000","last_unstale":"2023-10-09T06:16:43.948458+0000","last_undegraded":"2023-10-09T06:16:43.948458+0000","last_fullsized":"2023-10-09T06:16:43.948458+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:08.543959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.963419+0000","last_change":"2023-10-09T06:16:44.963419+0000","last_active":"2023-10-09T06:16:44.963419+0000","last_peered":"2023-10-09T06:16:44.963419+0000","last_clean":"2023-10-09T06:16:44.963419+0000","last_became_active":"2023-10-09T06:16:44.963126+0000","last_became_peered":"2023-10-09T06:16:44.963126+0000","last_unstale":"2023-10-09T06:16:44.963419+0000","last_undegraded":"2023-10-09T06:16:44.963419+0000","last_fullsized":"2023-10-09T06:16:44.963419+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:21.614499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.948574+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:16:44.948574+0000","last_peered":"2023-10-09T06:16:44.948574+0000","last_clean":"2023-10-09T06:16:44.948574+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:16:44.948574+0000","last_undegraded":"2023-10-09T06:16:44.948574+0000","last_fullsized":"2023-10-09T06:16:44.948574+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.954350+0000","last_change":"2023-10-09T06:16:44.954350+0000","last_active":"2023-10-09T06:16:44.954350+0000","last_peered":"2023-10-09T06:16:44.954350+0000","last_clean":"2023-10-09T06:16:44.954350+0000","last_became_active":"2023-10-09T06:16:44.954027+0000","last_became_peered":"2023-10-09T06:16:44.954027+0000","last_unstale":"2023-10-09T06:16:44.954350+0000","last_undegraded":"2023-10-09T06:16:44.954350+0000","last_fullsized":"2023-10-09T06:16:44.954350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:18.938865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.947159+0000","last_change":"2023-10-09T06:15:11.869497+0000","last_active":"2023-10-09T06:16:43.947159+0000","last_peered":"2023-10-09T06:16:43.947159+0000","last_clean":"2023-10-09T06:16:43.947159+0000","last_became_active":"2023-10-09T06:15:11.868971+0000","last_became_peered":"2023-10-09T06:15:11.868971+0000","last_unstale":"2023-10-09T06:16:43.947159+0000","last_undegraded":"2023-10-09T06:16:43.947159+0000","last_fullsized":"2023-10-09T06:16:43.947159+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:20.786443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.942174+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:16:44.942174+0000","last_peered":"2023-10-09T06:16:44.942174+0000","last_clean":"2023-10-09T06:16:44.942174+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:16:44.942174+0000","last_undegraded":"2023-10-09T06:16:44.942174+0000","last_fullsized":"2023-10-09T06:16:44.942174+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.945269+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:16:43.945269+0000","last_peered":"2023-10-09T06:16:43.945269+0000","last_clean":"2023-10-09T06:16:43.945269+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:16:43.945269+0000","last_undegraded":"2023-10-09T06:16:43.945269+0000","last_fullsized":"2023-10-09T06:16:43.945269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":87,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.948619+0000","last_change":"2023-10-09T06:08:16.100332+0000","last_active":"2023-10-09T06:16:43.948619+0000","last_peered":"2023-10-09T06:16:43.948619+0000","last_clean":"2023-10-09T06:16:43.948619+0000","last_became_active":"2023-10-09T06:08:16.099887+0000","last_became_peered":"2023-10-09T06:08:16.099887+0000","last_unstale":"2023-10-09T06:16:43.948619+0000","last_undegraded":"2023-10-09T06:16:43.948619+0000","last_fullsized":"2023-10-09T06:16:43.948619+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:58:36.739647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.958199+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:16:44.958199+0000","last_peered":"2023-10-09T06:16:44.958199+0000","last_clean":"2023-10-09T06:16:44.958199+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:16:44.958199+0000","last_undegraded":"2023-10-09T06:16:44.958199+0000","last_fullsized":"2023-10-09T06:16:44.958199+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.953404+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:16:43.953404+0000","last_peered":"2023-10-09T06:16:43.953404+0000","last_clean":"2023-10-09T06:16:43.953404+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:16:43.953404+0000","last_undegraded":"2023-10-09T06:16:43.953404+0000","last_fullsized":"2023-10-09T06:16:43.953404+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956492+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:16:44.956492+0000","last_peered":"2023-10-09T06:16:44.956492+0000","last_clean":"2023-10-09T06:16:44.956492+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:16:44.956492+0000","last_undegraded":"2023-10-09T06:16:44.956492+0000","last_fullsized":"2023-10-09T06:16:44.956492+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":95,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.954232+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:16:43.954232+0000","last_peered":"2023-10-09T06:16:43.954232+0000","last_clean":"2023-10-09T06:16:43.954232+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:16:43.954232+0000","last_undegraded":"2023-10-09T06:16:43.954232+0000","last_fullsized":"2023-10-09T06:16:43.954232+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.956388+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:16:44.956388+0000","last_peered":"2023-10-09T06:16:44.956388+0000","last_clean":"2023-10-09T06:16:44.956388+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:16:44.956388+0000","last_undegraded":"2023-10-09T06:16:44.956388+0000","last_fullsized":"2023-10-09T06:16:44.956388+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.961893+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:16:44.961893+0000","last_peered":"2023-10-09T06:16:44.961893+0000","last_clean":"2023-10-09T06:16:44.961893+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:16:44.961893+0000","last_undegraded":"2023-10-09T06:16:44.961893+0000","last_fullsized":"2023-10-09T06:16:44.961893+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-09T06:16:44.953905+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:16:44.953905+0000","last_peered":"2023-10-09T06:16:44.953905+0000","last_clean":"2023-10-09T06:16:44.953905+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:16:44.953905+0000","last_undegraded":"2023-10-09T06:16:44.953905+0000","last_fullsized":"2023-10-09T06:16:44.953905+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.949161+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:16:43.949161+0000","last_peered":"2023-10-09T06:16:43.949161+0000","last_clean":"2023-10-09T06:16:43.949161+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:16:43.949161+0000","last_undegraded":"2023-10-09T06:16:43.949161+0000","last_fullsized":"2023-10-09T06:16:43.949161+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-09T06:16:43.943302+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:16:43.943302+0000","last_peered":"2023-10-09T06:16:43.943302+0000","last_clean":"2023-10-09T06:16:43.943302+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:16:43.943302+0000","last_undegraded":"2023-10-09T06:16:43.943302+0000","last_fullsized":"2023-10-09T06:16:43.943302+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234244,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311644,"kb_used_data":2068,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433508,"statfs":{"total":95995035648,"available":95675912192,"internally_reserved":0,"allocated":2117632,"data_stored":1556820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.482,"15min":1.272},"min":{"1min":0.37,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":0.74099999999999999,"5min":11.785,"15min":11.785},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":1.232,"15min":1.3919999999999999},"min":{"1min":0.38300000000000001,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":0.70799999999999996,"5min":10.396000000000001,"15min":10.784000000000001},"last":0.51600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.2480000000000002,"15min":1.728},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.542,"5min":10.542,"15min":10.542},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.899,"15min":1.5069999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.548,"5min":11.818,"15min":11.818},"last":0.73499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:17:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":2.016,"15min":2.0379999999999998},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.442},{"interface":"front","average":{"1min":1.6539999999999999,"5min":2.0859999999999999,"15min":2.1579999999999999},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":6.2549999999999999,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.68000000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:15:40 2023","interfaces":[{"interface":"back","average":{"1min":17.396000000000001,"5min":4.96,"15min":3.6850000000000001},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.26600000000000001},"max":{"1min":267.75099999999998,"5min":267.75099999999998,"15min":267.75099999999998},"last":0.58699999999999997},{"interface":"front","average":{"1min":18.986999999999998,"5min":5.4269999999999996,"15min":3.5819999999999999},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":271.76299999999998,"5min":271.76299999999998,"15min":271.76299999999998},"last":0.70499999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":1.601,"15min":1.601},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.251},"max":{"1min":9.1140000000000008,"5min":11.853999999999999,"15min":41.527000000000001},"last":0.623},{"interface":"front","average":{"1min":1.256,"5min":1.0509999999999999,"15min":1.653},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":6.1500000000000004,"5min":6.7629999999999999,"15min":41.466000000000001},"last":0.752}]},{"osd":5,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0739999999999998,"5min":1.915,"15min":1.865},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":8.4589999999999996,"5min":9.8629999999999995,"15min":49.357999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.762,"5min":1.3420000000000001,"15min":1.518},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.6769999999999996,"5min":11.837,"15min":49.237000000000002},"last":0.79000000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:16:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.4390000000000001,"15min":1.51},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":8.2249999999999996,"5min":11.878,"15min":11.878},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.2150000000000001,"15min":1.5960000000000001},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.0380000000000003,"5min":7.9340000000000002,"15min":13.189},"last":0.64800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495881,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311624,"kb_used_data":2048,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":93433528,"statfs":{"total":95995035648,"available":95675932672,"internally_reserved":0,"allocated":2097152,"data_stored":1556086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.518,"15min":1.581},"min":{"1min":0.33900000000000002,"5min":0.222,"15min":0.214},"max":{"1min":0.749,"5min":12.808,"15min":12.808},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.8540000000000001,"15min":1.651},"min":{"1min":0.42799999999999999,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":0.70299999999999996,"5min":12.847,"15min":12.847},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.5600000000000001,"15min":2.2879999999999998},"min":{"1min":0.35799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.5690000000000008,"5min":10.646000000000001,"15min":10.646000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":2.4740000000000002,"15min":2.3719999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":9.6080000000000005,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6950000000000001,"5min":2.3319999999999999,"15min":2.4380000000000002},"min":{"1min":0.41499999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6539999999999999,"5min":7.6539999999999999,"15min":7.6539999999999999},"last":0.503},{"interface":"front","average":{"1min":1.137,"5min":2.383,"15min":2.5910000000000002},"min":{"1min":0.35199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6549999999999998,"5min":7.4279999999999999,"15min":7.4279999999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:16:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0289999999999999,"5min":4.8639999999999999,"15min":3.8780000000000001},"min":{"1min":0.42599999999999999,"5min":0.28899999999999998,"15min":0.26700000000000002},"max":{"1min":9.0419999999999998,"5min":222.922,"15min":222.922},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.5640000000000001,"5min":4.4390000000000001,"15min":3.6099999999999999},"min":{"1min":0.317,"5min":0.313,"15min":0.30599999999999999},"max":{"1min":11.374000000000001,"5min":222.911,"15min":222.911},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.532,"15min":1.3120000000000001},"min":{"1min":0.375,"5min":0.252,"15min":0.245},"max":{"1min":5.3620000000000001,"5min":10.75,"15min":10.75},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.51,"15min":1.5669999999999999},"min":{"1min":0.38400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.96,"5min":8.9239999999999995,"15min":12.567},"last":0.32100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:17:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.474,"15min":1.47},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":4.1349999999999998,"5min":10.773,"15min":13.91},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6040000000000001,"15min":2.069},"min":{"1min":0.45500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.6779999999999999,"5min":11.798999999999999,"15min":159.06800000000001},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 06:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.1599999999999999,"15min":1.3149999999999999},"min":{"1min":0.33800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.962,"5min":10.962,"15min":14.837},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.6699999999999999,"15min":1.7929999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":8.9920000000000009,"5min":10.891,"15min":11.281000000000001},"last":0.48899999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402617,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256336,"kb_used_data":1608,"kb_used_omap":4,"kb_used_meta":254715,"kb_avail":93488816,"statfs":{"total":95995035648,"available":95732547584,"internally_reserved":0,"allocated":1646592,"data_stored":981322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":260829070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.1989999999999998,"15min":2.5990000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.0939999999999994,"5min":10.726000000000001,"15min":10.726000000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":2.681,"15min":2.9940000000000002},"min":{"1min":0.21199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.526,"15min":1.6639999999999999},"min":{"1min":0.22700000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.0500000000000007,"5min":9.2769999999999992,"15min":9.2769999999999992},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":2.448,"15min":2.7330000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.1970000000000001,"5min":10.573,"15min":10.573},"last":0.315}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.2069999999999999,"5min":3.0899999999999999,"15min":3.0659999999999998},"min":{"1min":0.29599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":11.044,"5min":11.044,"15min":11.044},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.5819999999999999,"15min":2.7839999999999998},"min":{"1min":0.28000000000000003,"5min":0.245,"15min":0.245},"max":{"1min":12.464,"5min":13.487,"15min":13.487},"last":0.32700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.2280000000000002,"15min":2.8460000000000001},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1239999999999997,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.5169999999999999,"15min":2.907},"min":{"1min":0.315,"5min":0.311,"15min":0.311},"max":{"1min":10.723000000000001,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.60299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.4860000000000002,"15min":3.2629999999999999},"min":{"1min":0.39200000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.4500000000000002,"5min":10.92,"15min":10.92},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.7909999999999999,"15min":3.2109999999999999},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.53400000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":2.7999999999999998,"15min":2.956},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.772,"5min":12.91,"15min":12.91},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.6080000000000001,"5min":2.2839999999999998,"15min":2.5699999999999998},"min":{"1min":0.437,"5min":0.374,"15min":0.374},"max":{"1min":6.3170000000000002,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":2.347,"15min":2.7490000000000001},"min":{"1min":0.38100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.0830000000000002,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.3400000000000001,"5min":2.0070000000000001,"15min":2.129},"min":{"1min":0.40899999999999997,"5min":0.378,"15min":0.378},"max":{"1min":5.5599999999999996,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.56999999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024050,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410940,"kb_used_data":2292,"kb_used_omap":5,"kb_used_meta":408634,"kb_avail":93334212,"statfs":{"total":95995035648,"available":95574233088,"internally_reserved":0,"allocated":2347008,"data_stored":1585989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":418441850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:17:08 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.6930000000000001,"15min":1.8759999999999999},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":11.576000000000001,"5min":11.576000000000001,"15min":11.576000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.3700000000000001,"5min":2.7490000000000001,"15min":2.6619999999999999},"min":{"1min":0.27200000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":11.603,"5min":11.744999999999999,"15min":11.744999999999999},"last":0.34399999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.488,"15min":2.6150000000000002},"min":{"1min":0.254,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.186,"5min":11.186,"15min":11.186},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.016,"5min":2.5230000000000001,"15min":2.6070000000000002},"min":{"1min":0.21299999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":21.198,"5min":21.198,"15min":21.198},"last":0.36299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:15:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":3.3149999999999999,"15min":3.8319999999999999},"min":{"1min":0.26300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":14.414,"5min":24.870000000000001,"15min":27.048999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2839999999999998,"15min":3.423},"min":{"1min":0.27900000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":13.196999999999999,"5min":32.030000000000001,"15min":32.030000000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.5669999999999999,"15min":1.8029999999999999},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":4.5499999999999998,"5min":13.244,"15min":15.835000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2549999999999999,"15min":2.2629999999999999},"min":{"1min":0.39800000000000002,"5min":0.30299999999999999,"15min":0.25700000000000001},"max":{"1min":8.7769999999999992,"5min":11.566000000000001,"15min":15.023},"last":0.441}]},{"osd":5,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":2.0750000000000002,"15min":1.9990000000000001},"min":{"1min":0.33200000000000002,"5min":0.316,"15min":0.29299999999999998},"max":{"1min":11.222,"5min":13.285,"15min":15.858000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.7480000000000002,"5min":2.3780000000000001,"15min":2.2080000000000002},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":21.167000000000002,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.61499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":2.355,"15min":2.2090000000000001},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.088999999999999,"5min":21.088999999999999,"15min":32.079000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.1459999999999999,"5min":2.1859999999999999,"15min":2.085},"min":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.0270000000000001,"5min":13.315,"15min":15.893000000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:16:36 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.319,"15min":2.387},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.28899999999999998},"max":{"1min":8.8089999999999993,"5min":11.515000000000001,"15min":15.050000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":4.2309999999999999,"5min":2.4470000000000001,"15min":2.3399999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":21.135999999999999,"5min":21.135999999999999,"15min":32.055999999999997},"last":0.67700000000000005}]}]},{"osd":2,"up_from":128,"seq":549755813919,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196712,"kb_used_data":1568,"kb_used_omap":7,"kb_used_meta":195128,"kb_avail":93548440,"statfs":{"total":95995035648,"available":95793602560,"internally_reserved":0,"allocated":1605632,"data_stored":975398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":199811153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.863,"15min":1.9099999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.156000000000001,"5min":12.156000000000001,"15min":12.156000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.145,"5min":2.4159999999999999,"15min":2.6280000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":12.113,"5min":12.113,"15min":12.113},"last":0.35499999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":11.619999999999999,"5min":3.8079999999999998,"15min":2.5059999999999998},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":222.50299999999999,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":11.102,"5min":4.4299999999999997,"15min":3.3180000000000001},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":222.19399999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 06:16:13 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":2.8759999999999999,"15min":2.8759999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.435,"5min":10.435,"15min":10.435},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.0179999999999998,"5min":2.0179999999999998,"15min":2.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.6299999999999999,"5min":6.6299999999999999,"15min":6.6299999999999999},"last":0.32400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.879,"15min":2.0859999999999999},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.1560000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.1320000000000001,"15min":2.3799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.6230000000000002,"5min":15.183,"15min":15.183},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.591,"15min":1.744},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":2.1960000000000002,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.3039999999999998,"15min":2.524},"min":{"1min":0.35499999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.085,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.0779999999999998,"15min":2.3149999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.6890000000000001,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.5509999999999999,"15min":1.6599999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.1509999999999998,"5min":7.327,"15min":7.327},"last":0.57999999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.5980000000000001,"15min":1.6899999999999999},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.1260000000000003,"5min":6.5529999999999999,"15min":6.5529999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.734,"15min":3.089},"min":{"1min":0.39700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.2229999999999999,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.69699999999999995}]}]},{"osd":3,"up_from":149,"seq":639950127117,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111000,"kb_used_data":1488,"kb_used_omap":2,"kb_used_meta":109501,"kb_avail":93634152,"statfs":{"total":95995035648,"available":95881371648,"internally_reserved":0,"allocated":1523712,"data_stored":966500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":112129077},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36299999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61799999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019153,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591248,"kb_used_data":1480,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93153904,"statfs":{"total":95995035648,"available":95389597696,"internally_reserved":0,"allocated":1515520,"data_stored":966117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.464,"15min":1.3779999999999999},"min":{"1min":0.29099999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.73499999999999999,"5min":8.5990000000000002,"15min":15.484999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.9990000000000001,"15min":1.72},"min":{"1min":0.315,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.76300000000000001,"5min":11.234,"15min":15.289},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 06:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.25600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":4.7050000000000001,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.28,"5min":1.542,"15min":1.619},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1449999999999996,"5min":8.7509999999999994,"15min":8.7509999999999994},"last":0.59399999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:17:15 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.784,"15min":1.865},"min":{"1min":0.30499999999999999,"5min":0.255,"15min":0.255},"max":{"1min":8.9100000000000001,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.35599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.7559999999999998,"5min":7.2830000000000004,"15min":7.2830000000000004},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.835,"5min":3.0329999999999999,"15min":4.125},"min":{"1min":0.318,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":11.31,"5min":60.414999999999999,"15min":236.78},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.6339999999999999,"15min":4.0140000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.29499999999999998},"max":{"1min":8.2010000000000005,"5min":60.375999999999998,"15min":236.76599999999999},"last":0.55900000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.3660000000000001,"15min":1.3169999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":4.4779999999999998,"5min":9.6790000000000003,"15min":19.687999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.5820000000000001,"15min":1.4319999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.23699999999999999},"max":{"1min":2.8100000000000001,"5min":11.282,"15min":11.282},"last":0.39400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:17:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.1160000000000001,"15min":1.1870000000000001},"min":{"1min":0.41999999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.81999999999999995,"5min":9.7680000000000007,"15min":20.390999999999998},"last":0.622},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.7210000000000001,"15min":1.5289999999999999},"min":{"1min":0.37,"5min":0.32800000000000001,"15min":0.25},"max":{"1min":0.753,"5min":14.202999999999999,"15min":26.137},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.21,"15min":1.3049999999999999},"min":{"1min":0.379,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":6.6849999999999996,"5min":6.6849999999999996,"15min":15.911},"last":0.436},{"interface":"front","average":{"1min":2.2010000000000001,"5min":1.4259999999999999,"15min":1.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.28799999999999998},"max":{"1min":12.965999999999999,"5min":12.965999999999999,"15min":15.333},"last":0.45800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757518,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":603520,"kb_used_data":1464,"kb_used_omap":0,"kb_used_meta":602048,"kb_avail":93141632,"statfs":{"total":95995035648,"available":95377031168,"internally_reserved":0,"allocated":1499136,"data_stored":961803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616497152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:17:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":1.5860000000000001,"15min":1.651},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.215},"max":{"1min":0.81499999999999995,"5min":8.5139999999999993,"15min":18.637},"last":0.622},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.905,"15min":1.5429999999999999},"min":{"1min":0.34000000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.82099999999999995,"5min":9.9939999999999998,"15min":9.9939999999999998},"last":0.59399999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.032,"15min":2.3420000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.8420000000000001,"5min":11.292,"15min":11.292},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.218,"15min":2.1920000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6349999999999998,"5min":7.9539999999999997,"15min":7.9539999999999997},"last":0.60499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:17:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":2.1829999999999998,"15min":2.3860000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.0519999999999996,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.0779999999999998,"15min":2.2610000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":5.6790000000000003,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":0.66800000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:16:03 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.8039999999999998,"15min":3.2080000000000002},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.30399999999999999},"max":{"1min":12.106,"5min":12.106,"15min":128.035},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.96,"5min":2.327,"15min":3.0289999999999999},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.28199999999999997},"max":{"1min":7.4020000000000001,"5min":10.324999999999999,"15min":128.07300000000001},"last":0.63700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.696,"15min":1.5940000000000001},"min":{"1min":0.35799999999999998,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":5.6360000000000001,"5min":8.7850000000000001,"15min":19.847999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.276,"15min":1.353},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":5.1200000000000001,"5min":7.7290000000000001,"15min":20.106000000000002},"last":0.48899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:17:28 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.5089999999999999,"15min":1.492},"min":{"1min":0.438,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":1.988,"5min":8.8149999999999995,"15min":20.027000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.456,"15min":1.3560000000000001},"min":{"1min":0.46300000000000002,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":0.85899999999999999,"5min":7.3479999999999999,"15min":12.726000000000001},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:16:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.407,"15min":1.6100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.8409999999999993,"5min":8.8409999999999993,"15min":23.398},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.363,"5min":1.323,"15min":1.4970000000000001},"min":{"1min":0.40699999999999997,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":7.0049999999999999,"5min":7.7809999999999997,"15min":20.131},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:17:41.302 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T06:17:41.303 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T06:17:41.648 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:41.950 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:41 smithi107 ceph-mon[212750]: from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:41.950 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:41 smithi107 ceph-mon[212750]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:41.951 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:41 smithi107 ceph-mon[220921]: from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:42.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:42] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-10-09T06:17:42.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:41 smithi107 ceph-mon[220921]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:41 smithi150 ceph-mon[194178]: from='client.45206 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:17:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:41 smithi150 ceph-mon[194178]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:43.233 INFO:teuthology.orchestra.run.smithi107.stdout:epoch 150 2023-10-09T06:17:43.233 INFO:teuthology.orchestra.run.smithi107.stdout:fsid b7a98308-6667-11ee-8db5-212e2dc638e7 2023-10-09T06:17:43.233 INFO:teuthology.orchestra.run.smithi107.stdout:created 2023-10-09T05:51:17.416913+0000 2023-10-09T06:17:43.233 INFO:teuthology.orchestra.run.smithi107.stdout:modified 2023-10-09T06:16:44.938136+0000 2023-10-09T06:17:43.233 INFO:teuthology.orchestra.run.smithi107.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:crush_version 22 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:full_ratio 0.95 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:backfillfull_ratio 0.9 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:nearfull_ratio 0.85 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:require_min_compat_client luminous 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:min_compat_client luminous 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:require_osd_release quincy 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:stretch_mode_enabled false 2023-10-09T06:17:43.234 INFO:teuthology.orchestra.run.smithi107.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:max_osd 8 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 up in weight 1 up_from 91 up_thru 149 down_at 88 last_clean_interval [8,87) [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168] [v2:172.21.15.107:6804/2817416168,v1:172.21.15.107:6805/2817416168] exists,up 61782add-fd90-440a-b489-da0bd6cdb972 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 up in weight 1 up_from 110 up_thru 149 down_at 107 last_clean_interval [13,106) [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293] [v2:172.21.15.107:6812/3281752293,v1:172.21.15.107:6813/3281752293] exists,up a0c947d5-3ca8-468c-9f6d-2859ed2275b2 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2 up in weight 1 up_from 128 up_thru 149 down_at 123 last_clean_interval [18,122) [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714] [v2:172.21.15.107:6820/2613148714,v1:172.21.15.107:6821/2613148714] exists,up ba0a04dd-49a5-498f-a016-aff57d88ff33 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3 up in weight 1 up_from 149 up_thru 149 down_at 145 last_clean_interval [25,144) [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119] [v2:172.21.15.107:6828/3856205119,v1:172.21.15.107:6829/3856205119] exists,up a3ca913f-7262-4d0a-8f15-07e70fe94239 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 up in weight 1 up_from 30 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864] [v2:172.21.15.150:6802/2438956864,v1:172.21.15.150:6803/2438956864] exists,up c43fad0c-5200-44d6-b725-36b66a11482e 2023-10-09T06:17:43.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 up in weight 1 up_from 38 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682] [v2:172.21.15.150:6810/2897644682,v1:172.21.15.150:6811/2897644682] exists,up 98904b1f-c066-4710-974a-288f9323551d 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6 up in weight 1 up_from 46 up_thru 145 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495] [v2:172.21.15.150:6818/2964927495,v1:172.21.15.150:6819/2964927495] exists,up d40d76a2-9d6d-41d9-91ed-dfa6f8228130 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 up in weight 1 up_from 54 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784] [v2:172.21.15.150:6826/274719784,v1:172.21.15.150:6827/274719784] exists,up 4772509b-9d82-4565-86d2-557b830afc74 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:pg_upmap_items 4.6 [1,5] 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:pg_upmap_items 4.1b [1,2] 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:pg_upmap_items 4.1c [1,5] 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/443194359 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/3156803825 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1608610913 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6801/2437377245 expires 2023-10-10T05:51:45.273080+0000 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/4289301557 expires 2023-10-10T05:51:45.273080+0000 2023-10-09T06:17:43.236 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1657740329 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/700020267 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/3437069098 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/2371556112 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6800/2437377245 expires 2023-10-10T05:51:45.273080+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:6835/3571092793 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:6834/3571092793 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/3008436255 expires 2023-10-10T05:52:10.327166+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1162645774 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6800/2681598821 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1943625242 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/643879886 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.237 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6800/2171634441 expires 2023-10-10T05:52:10.327166+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1557369746 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/2216216350 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/4293783841 expires 2023-10-10T05:52:10.327166+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6801/2681598821 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/3684818042 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/4121084552 expires 2023-10-10T05:52:10.327166+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6800/1545922954 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/3067456200 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6801/2171634441 expires 2023-10-10T05:52:10.327166+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6801/1545922954 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.238 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1999847309 expires 2023-10-10T05:51:45.273080+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/2428731564 expires 2023-10-10T05:51:45.273080+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/3366444430 expires 2023-10-10T06:03:16.414020+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6801/2525813986 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/2888496999 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.150:0/1037268415 expires 2023-10-10T06:02:08.994675+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/283261379 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/2930305484 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:0/1830374370 expires 2023-10-10T05:56:45.306041+0000 2023-10-09T06:17:43.239 INFO:teuthology.orchestra.run.smithi107.stdout:blocklist 172.21.15.107:6800/2525813986 expires 2023-10-10T06:00:53.065448+0000 2023-10-09T06:17:43.923 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 91 up_thru 149 down_at 88 last_clean_interval [8,87) [v2:172.21.15.107:6802/2817416168,v1:172.21.15.107:6803/2817416168] [v2:172.21.15.107:6804/2817416168,v1:172.21.15.107:6805/2817416168] exists,up 61782add-fd90-440a-b489-da0bd6cdb972', 'osd.1 up in weight 1 up_from 110 up_thru 149 down_at 107 last_clean_interval [13,106) [v2:172.21.15.107:6810/3281752293,v1:172.21.15.107:6811/3281752293] [v2:172.21.15.107:6812/3281752293,v1:172.21.15.107:6813/3281752293] exists,up a0c947d5-3ca8-468c-9f6d-2859ed2275b2', 'osd.2 up in weight 1 up_from 128 up_thru 149 down_at 123 last_clean_interval [18,122) [v2:172.21.15.107:6818/2613148714,v1:172.21.15.107:6819/2613148714] [v2:172.21.15.107:6820/2613148714,v1:172.21.15.107:6821/2613148714] exists,up ba0a04dd-49a5-498f-a016-aff57d88ff33', 'osd.3 up in weight 1 up_from 149 up_thru 149 down_at 145 last_clean_interval [25,144) [v2:172.21.15.107:6826/3856205119,v1:172.21.15.107:6827/3856205119] [v2:172.21.15.107:6828/3856205119,v1:172.21.15.107:6829/3856205119] exists,up a3ca913f-7262-4d0a-8f15-07e70fe94239', 'osd.4 up in weight 1 up_from 30 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6800/2438956864,v1:172.21.15.150:6801/2438956864] [v2:172.21.15.150:6802/2438956864,v1:172.21.15.150:6803/2438956864] exists,up c43fad0c-5200-44d6-b725-36b66a11482e', 'osd.5 up in weight 1 up_from 38 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6808/2897644682,v1:172.21.15.150:6809/2897644682] [v2:172.21.15.150:6810/2897644682,v1:172.21.15.150:6811/2897644682] exists,up 98904b1f-c066-4710-974a-288f9323551d', 'osd.6 up in weight 1 up_from 46 up_thru 145 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6816/2964927495,v1:172.21.15.150:6817/2964927495] [v2:172.21.15.150:6818/2964927495,v1:172.21.15.150:6819/2964927495] exists,up d40d76a2-9d6d-41d9-91ed-dfa6f8228130', 'osd.7 up in weight 1 up_from 54 up_thru 149 down_at 0 last_clean_interval [0,0) [v2:172.21.15.150:6824/274719784,v1:172.21.15.150:6825/274719784] [v2:172.21.15.150:6826/274719784,v1:172.21.15.150:6827/274719784] exists,up 4772509b-9d82-4565-86d2-557b830afc74'] 2023-10-09T06:17:43.923 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T06:17:44.275 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:44.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:43 smithi107 ceph-mon[212750]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/337897145' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:43 smithi107 ceph-mon[220921]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/337897145' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:43 smithi150 ceph-mon[194178]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/337897145' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:17:45.645 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:17:45.645 INFO:teuthology.orchestra.run.smithi107.stdout: "name": "mon.a", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "cluster": "ceph", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "admin_socket_mode": "", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_client_required": "cephx, none", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_cluster_required": "cephx", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_debug": "true", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_service_required": "cephx", 2023-10-09T06:17:45.646 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "auth_supported": "", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_aio": "true", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_aio_reap_max": "16", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_async_discard": "false", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_block_size": "4096", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_debug_aio": "false", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T06:17:45.647 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_enable_discard": "false", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_flock_retry": "3", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_inject_crash": "0", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_ioring": "false", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_ioring_hipri": "false", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bdev_type": "", 2023-10-09T06:17:45.648 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_allocator": "hybrid", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_buffered_io": "true", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T06:17:45.649 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_replay_recovery": "false", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluefs_sync_write": "false", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_allocator": "hybrid", 2023-10-09T06:17:45.650 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_create": "true", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_db_create": "false", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_db_path": "", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_db_size": "0", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_path": "", 2023-10-09T06:17:45.651 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_size": "107374182400", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_wal_create": "false", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_wal_path": "", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bluefs": "true", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:45.652 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_autotune": "true", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_size": "0", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T06:17:45.653 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cache_type": "2q", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_clone_cow": "true", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_mode": "none", 2023-10-09T06:17:45.654 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_freelist": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T06:17:45.655 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T06:17:45.656 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fail_eio": "false", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T06:17:45.657 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T06:17:45.658 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_blob_size": "0", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T06:17:45.659 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T06:17:45.660 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_spdk_mem": "512", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T06:17:45.661 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_tracing": "false", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T06:17:45.662 INFO:teuthology.orchestra.run.smithi107.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T06:17:45.663 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_cluster_require_version": "2", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_require_signatures": "false", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_require_version": "2", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_service_require_signatures": "false", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_service_require_version": "2", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "cephx_sign_messages": "true", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "chdir": "", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "client_acl_type": "", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "client_asio_thread_count": "2", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "client_cache_mid": "0.750000", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "client_cache_size": "16384", 2023-10-09T06:17:45.664 INFO:teuthology.orchestra.run.smithi107.stdout: "client_caps_release_delay": "5", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_check_pool_perm": "true", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_debug_force_sync_read": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_debug_getattr_caps": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_die_on_failed_remount": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_dirsize_rbytes": "true", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_force_lazyio": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_fs": "", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T06:17:45.665 INFO:teuthology.orchestra.run.smithi107.stdout: "client_inject_release_failure": "false", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_max_inline_size": "4096", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_mds_namespace": "", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_metadata": "", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_mount_gid": "-1", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_mount_timeout": "300", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_mount_uid": "-1", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_mountpoint": "/", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_notify_timeout": "10", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc": "true", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T06:17:45.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc_max_objects": "1000", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc_size": "209715200", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_permissions": "true", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_quota": "true", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_quota_df": "true", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_readahead_max_bytes": "0", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_readahead_max_periods": "4", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_readahead_min": "131072", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_reconnect_stale": "false", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_shutdown_timeout": "30", 2023-10-09T06:17:45.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_snapdir": ".snap", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "client_tick_interval": "1", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "client_trace": "", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "client_use_faked_inos": "false", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "client_use_random_mds": "false", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_graylog": "false", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_graylog_port": "12201", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_monitors": "default=true", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_syslog": "false", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "clog_to_syslog_level": "info", 2023-10-09T06:17:45.668 INFO:teuthology.orchestra.run.smithi107.stdout: "cluster_addr": "-", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "cluster_network": "", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "cluster_network_interface": "", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "compressor_zlib_isal": "false", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "compressor_zlib_level": "5", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "compressor_zstd_level": "1", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T06:17:45.669 INFO:teuthology.orchestra.run.smithi107.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "crimson_seastar_smp": "1", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "crush_location": "", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "crush_location_hook": "", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "crush_location_hook_timeout": "10", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "daemonize": "false", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "daos_pool": "tank", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_alienstore": "0/5", 2023-10-09T06:17:45.670 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_asok": "1/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_asok_assert_abort": "false", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_auth": "1/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_bdev": "1/3", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_bluefs": "1/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_bluestore": "1/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_buffer": "0/1", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_cephsqlite": "0/5", 2023-10-09T06:17:45.671 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_client": "0/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_compressor": "1/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_context": "0/1", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_crush": "1/1", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_crypto": "1/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_cyanstore": "0/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_dpdk": "1/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_eventtrace": "1/5", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_filer": "0/1", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_filestore": "1/3", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_finisher": "1/1", 2023-10-09T06:17:45.672 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_fuse": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_javaclient": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_journal": "1/3", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_journaler": "0/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_kstore": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_leveldb": "4/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_lockdep": "0/1", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mclock": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds": "1/5", 2023-10-09T06:17:45.673 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds_balancer": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds_locker": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds_log": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mds_migrator": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_memstore": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mgr": "2/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mgrc": "1/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_mon": "20/20", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_monc": "0/10", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_ms": "1/1", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_none": "0/5", 2023-10-09T06:17:45.674 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_objclass": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_objectcacher": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_objecter": "0/1", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_optracker": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_osd": "1/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_paxos": "20/20", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_perfcounter": "1/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_prioritycache": "1/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rados": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rbd": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T06:17:45.675 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rbd_replay": "0/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_refs": "0/0", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_reserver": "1/1", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw_access": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw_flight": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rgw_sync": "1/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_rocksdb": "4/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore": "0/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_backref": "0/5", 2023-10-09T06:17:45.676 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_cache": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_device": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_epm": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_journal": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_lba": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_odata": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_omap": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_onode": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_t": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_seastore_tm": "0/5", 2023-10-09T06:17:45.677 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_striper": "0/1", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_test": "0/5", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_throttle": "1/1", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_timer": "0/1", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "debug_tp": "0/5", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "device_failure_prediction_mode": "none", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "err_to_graylog": "false", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "err_to_journald": "false", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "err_to_stderr": "true", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "err_to_syslog": "false", 2023-10-09T06:17:45.678 INFO:teuthology.orchestra.run.smithi107.stdout: "event_tracing": "false", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_http_port": "9926", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_prio_limit": "5", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_sort_metrics": "true", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "exporter_stats_period": "5", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "fake_statfs_for_testing": "0", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "fatal_signal_handlers": "true", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "filer_max_purge_ops": "10", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "filer_max_truncate_ops": "128", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T06:17:45.679 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_blackhole": "false", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_btrfs_snap": "true", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_caller_concurrency": "10", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_debug_omap_check": "false", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_debug_verify_split": "false", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_dump_file": "", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T06:17:45.680 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fadvise": "true", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fail_eio": "true", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fd_cache_size": "128", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fiemap": "false", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_inject_stall": "0", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_journal_parallel": "false", 2023-10-09T06:17:45.681 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_journal_trailing": "false", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_journal_writeahead": "false", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_kill_at": "0", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T06:17:45.682 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_merge_threshold": "-10", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_odsync_write": "false", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_omap_backend_path": "", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T06:17:45.683 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_op_threads": "2", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_punch_hole": "false", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T06:17:45.684 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_queue_max_ops": "50", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_seek_data_hole": "false", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_sloppy_crc": "false", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_splice": "false", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_split_multiple": "2", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_split_rand_factor": "20", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_update_to": "1000", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T06:17:45.685 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_xfs_extsize": "false", 2023-10-09T06:17:45.686 INFO:teuthology.orchestra.run.smithi107.stdout: "filestore_zfs_snap": "false", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fio_dir": "/tmp/fio", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fsid": "b7a98308-6667-11ee-8db5-212e2dc638e7", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_allow_other": "true", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_big_writes": "true", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_debug": "false", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_default_permissions": "false", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_disable_pagecache": "false", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_max_write": "0", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_multithreaded": "true", 2023-10-09T06:17:45.687 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_require_active_mds": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_set_user_groups": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_splice_move": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_splice_read": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_splice_write": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "gss_target_name": "ceph", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "heartbeat_file": "", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "heartbeat_inject_failure": "0", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "heartbeat_interval": "5", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "host": "smithi107", 2023-10-09T06:17:45.688 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T06:17:45.689 INFO:teuthology.orchestra.run.smithi107.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "inject_early_sigterm": "false", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "jaeger_agent_port": "6799", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "jaeger_tracing_enable": "false", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_aio": "true", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_align_min_size": "65536", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_block_align": "true", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_block_size": "4096", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_dio": "true", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_discard": "false", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_force_aio": "false", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_ignore_corruption": "false", 2023-10-09T06:17:45.690 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_max_write_entries": "100", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_replay_from": "0", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_write_header_frequency": "0", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journal_zero_on_create": "false", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journaler_prefetch_periods": "10", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journaler_prezero_periods": "5", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "journaler_write_head_interval": "15", 2023-10-09T06:17:45.691 INFO:teuthology.orchestra.run.smithi107.stdout: "key": "", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "keyfile": "", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_backend": "rocksdb", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_max_bytes": "67108864", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_max_ops": "512", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_onode_map_size": "1024", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T06:17:45.692 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "kstore_sync_transaction": "false", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_block_size": "65536", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_bloom_size": "0", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_cache_size": "536870912", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_compression": "false", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_log": "", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_max_open_files": "0", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_paranoid": "false", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "librados_thread_count": "2", 2023-10-09T06:17:45.693 INFO:teuthology.orchestra.run.smithi107.stdout: "lockdep": "false", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "lockdep_force_backtrace": "false", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_coarse_timestamps": "true", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_flush_on_exit": "false", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_graylog_port": "12201", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_max_new": "1000", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_max_recent": "10000", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_stderr_prefix": "", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T06:17:45.694 INFO:teuthology.orchestra.run.smithi107.stdout: "log_to_file": "true", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "log_to_graylog": "false", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "log_to_journald": "false", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "log_to_stderr": "false", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "log_to_syslog": "false", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_action_on_write_error": "1", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_alternate_name_max": "8192", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_asio_thread_count": "2", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_export_pin": "true", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T06:17:45.695 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_interval": "10", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_max": "-1", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_max_until": "-1", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_merge_size": "50", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_mode": "0", 2023-10-09T06:17:45.696 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_split_bits": "3", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_split_size": "10000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T06:17:45.697 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_mid": "0.700000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_trim_interval": "1", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T06:17:45.698 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_debug_auth_pins": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_debug_frag": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_debug_scatterstat": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_debug_subtrees": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_default_dir_hash": "2", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_defer_session_stale": "true", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T06:17:45.699 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dir_max_entries": "0", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dir_prefetch": "true", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_early_reply": "true", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_enable_op_tracker": "true", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_enforce_unique_name": "true", 2023-10-09T06:17:45.700 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T06:17:45.701 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_health_dummy": "false", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_join_fs": "", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_journal_format": "1", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_create_at": "0", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_export_at": "0", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_import_at": "0", 2023-10-09T06:17:45.702 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_journal_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_link_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_openc_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_rename_at": "0", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_max_events": "-1", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_max_segments": "128", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_pause": "false", 2023-10-09T06:17:45.703 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_segment_size": "0", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_completed_requests": "100000", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_export_size": "20971520", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_file_recover": "32", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_purge_files": "64", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_purge_ops": "8192", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T06:17:45.704 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_metrics_update_interval": "2", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_min_caps_per_client": "100", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_numa_node": "-1", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_op_history_duration": "600", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_op_history_size": "20", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_op_log_threshold": "5", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_ping_grace": "15", 2023-10-09T06:17:45.705 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_ping_interval": "5", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_max_caps": "30000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_replay_interval": "1.000000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T06:17:45.706 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_root_ino_gid": "0", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_root_ino_uid": "0", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_shutdown_check": "0", 2023-10-09T06:17:45.707 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_skip_ino": "0", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_snap_min_uid": "0", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_snap_rstat": "false", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_symlink_recovery": "true", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_thrash_exports": "0", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_thrash_fragments": "0", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_tick_interval": "5.000000", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_valgrind_exit": "false", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_verify_backtrace": "1", 2023-10-09T06:17:45.708 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_verify_scatter": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mds_wipe_sessions": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mempool_debug": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "memstore_page_set": "false", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "memstore_page_size": "65536", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_client_bytes": "134217728", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_client_messages": "512", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T06:17:45.709 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_disabled_modules": "", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_mds_messages": "128", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_mon_messages": "128", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_osd_messages": "8192", 2023-10-09T06:17:45.710 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_pool": "true", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_standby_modules": "true", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_stats_period": "5", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_stats_threshold": "5", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_tick_period": "2", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_allow_pool_delete": "true", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T06:17:45.711 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_bytes": "104857600", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T06:17:45.712 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T06:17:45.713 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_compact_on_start": "false", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_compact_on_trim": "true", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_cpu_threads": "4", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_data_avail_crit": "5", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_data_avail_warn": "5", 2023-10-09T06:17:45.714 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_dump_json": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_extra_checks": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T06:17:45.715 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_election_default_strategy": "3", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_election_timeout": "5.000000", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_enable_op_tracker": "true", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_fake_pool_delete": "false", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_force_quorum_join": "false", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T06:17:45.716 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_log_update_period": "5", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_max_detail": "50", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_to_clog": "true", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_host": "[v2:172.21.15.107:3300/0,v1:172.21.15.107:6789/0] [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] [v2:172.21.15.107:3301/0,v1:172.21.15.107:6790/0]", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_host_override": "", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_initial_members": "", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T06:17:45.717 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_lease": "5.000000", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_log_full_interval": "50", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_log_max": "10000", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_log_max_summary": "50", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_log_epochs": "500", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_osd": "10000", 2023-10-09T06:17:45.718 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_memory_autotune": "true", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_memory_target": "2147483648", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_digest_period": "5", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T06:17:45.719 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_complaint_time": "30", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_history_duration": "600", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_history_size": "20", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_op_log_threshold": "5", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T06:17:45.720 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_cache_size": "500", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T06:17:45.721 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T06:17:45.722 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_report_timeout": "900", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T06:17:45.723 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_reweight_max_osds": "4", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T06:17:45.724 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_scrub_interval": "86400", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_scrub_max_keys": "100", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_scrub_timeout": "300", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_session_timeout": "300", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_smart_report_timeout": "5", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T06:17:45.725 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_stretch_pool_size": "4", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_debug": "false", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_tick_interval": "5", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T06:17:45.726 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T06:17:45.727 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_older_version": "true", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "monmap": "", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T06:17:45.728 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "motr_tracing_enabled": "false", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_op_threads": "3", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_cm": "false", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_device_name": "", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T06:17:45.729 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_sl": "3", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_rdma_type": "ib", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_async_reap_threshold": "5", 2023-10-09T06:17:45.730 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_before_connect": "false", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_ipv4": "true", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_ipv6": "false", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_msgr1": "true", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_msgr2": "true", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_port_max": "7568", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_port_min": "6800", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_retry_count": "3", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_bind_retry_delay": "5", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_blackhole_client": "false", 2023-10-09T06:17:45.731 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_blackhole_mds": "false", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_blackhole_mgr": "false", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_blackhole_mon": "false", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_blackhole_osd": "false", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_client_mode": "crc secure", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_cluster_type": "", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_compress_secure": "false", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_crc_data": "true", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_crc_header": "true", 2023-10-09T06:17:45.732 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_die_on_bug": "true", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_die_on_old_message": "true", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T06:17:45.733 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_hugepages": "", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_lro": "true", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_pmd": "", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_port_id": "0", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_dump_on_send": "false", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T06:17:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_delay_type": "", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_network_congestion": "0", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_inject_socket_failures": "0", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_max_accept_failures": "4", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_max_backoff": "15.000000", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T06:17:45.735 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_osd_compress_mode": "none", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_pq_min_cost": "65536", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_public_type": "", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_service_mode": "crc secure", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_tcp_nodelay": "true", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "ms_type": "async+posix", 2023-10-09T06:17:45.736 INFO:teuthology.orchestra.run.smithi107.stdout: "no_config_file": "false", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_inflight_ops": "1024", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objecter_timeout": "10.000000", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "objectstore_blackhole": "false", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "openssl_engine_opts": "", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T06:17:45.737 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_max_ops": "4", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_agent_slop": "0.020000", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_asio_thread_count": "2", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T06:17:45.738 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backfill_scan_max": "512", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backfill_scan_min": "64", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backoff_on_peering": "false", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_beacon_report_interval": "300", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_bench_duration": "30", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_blkin_trace_all": "false", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T06:17:45.739 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_class_update_on_start": "true", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_client_message_cap": "256", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_client_op_priority": "63", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_client_watch_timeout": "30", 2023-10-09T06:17:45.740 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_command_max_records": "256", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_compact_on_start": "false", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_crush_update_on_start": "true", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T06:17:45.741 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_op_order": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T06:17:45.742 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_shutdown": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T06:17:45.743 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_default_notify_timeout": "30", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_enable_op_tracker": "true", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T06:17:45.744 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_fast_info": "true", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_fast_shutdown": "true", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_function_tracing": "false", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_grace": "20", 2023-10-09T06:17:45.745 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_interval": "6", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_stale": "600", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T06:17:45.746 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_journal_size": "5120", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_kill_backfill_at": "0", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_map_cache_size": "50", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_map_dedup": "true", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_map_message_max": "40", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_attr_name_len": "100", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_attr_size": "0", 2023-10-09T06:17:45.747 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_backfills": "1", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_markdown_count": "5", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_markdown_period": "600", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_object_name_len": "2048", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_object_size": "134217728", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_pgls": "1024", 2023-10-09T06:17:45.748 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_push_cost": "8388608", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_push_objects": "10", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_scrubs": "1", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_max_write_size": "90", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T06:17:45.749 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_profile": "balanced", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T06:17:45.750 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_base": "805306368", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_target": "4294967296", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_target_autotune": "false", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_min_recovery_priority": "0", 2023-10-09T06:17:45.751 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mon_report_interval": "5", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_num_cache_shards": "32", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_numa_node": "-1", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_objecter_finishers": "1", 2023-10-09T06:17:45.752 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_objectstore": "bluestore", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_objectstore_fuse": "false", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_objectstore_tracing": "false", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_history_duration": "600", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_history_size": "20", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_log_threshold": "5", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_shards": "0", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T06:17:45.753 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_op_thread_timeout": "15", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_open_classes_on_start": "true", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_os_flags": "0", 2023-10-09T06:17:45.754 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_peering_op_priority": "255", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_delete_priority": "5", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T06:17:45.755 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_crimson": "false", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T06:17:45.756 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_flags": "0", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_min_size": "0", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_size": "3", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_default_type": "replicated", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T06:17:45.757 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_cost": "20971520", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_active": "0", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T06:17:45.758 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_op_priority": "3", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_priority": "5", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_repair_during_recovery": "false", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T06:17:45.759 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_cost": "52428800", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_end_hour": "0", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T06:17:45.760 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_priority": "5", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T06:17:45.761 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_skip_data_digest": "false", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_smart_report_timeout": "5", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_priority": "5", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T06:17:45.762 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_target_transaction_size": "30", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T06:17:45.763 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_tracing": "false", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_use_stale_snap": "false", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_kill_at": "0", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_max_join_drift": "10", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_min": "500", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_min_wait": "0.050000", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_service_trim_max": "500", 2023-10-09T06:17:45.764 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_service_trim_min": "250", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_stash_full_interval": "25", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_trim_max": "500", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "paxos_trim_min": "250", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "perf": "true", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "pid_file": "", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "public_addr": "-", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "public_addrv": "", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "public_bind_addr": "-", 2023-10-09T06:17:45.765 INFO:teuthology.orchestra.run.smithi107.stdout: "public_network": "172.21.0.0/20", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "public_network_interface": "", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "qat_compressor_enabled": "false", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rados_mon_op_timeout": "0", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rados_osd_op_timeout": "0", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rados_tracing": "false", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_atime_update_interval": "60", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T06:17:45.766 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache": "true", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_policy": "writearound", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_size": "33554432", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T06:17:45.767 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_compression_hint": "none", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_clone_format": "auto", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_data_pool": "", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_features": "61", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_format": "2", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_map_options": "", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_order": "22", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_pool": "rbd", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_stripe_count": "0", 2023-10-09T06:17:45.768 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_io_scheduler": "simple", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T06:17:45.769 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_order": "24", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_pool": "", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_journal_splay_width": "4", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T06:17:45.770 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T06:17:45.771 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T06:17:45.772 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T06:17:45.772 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T06:17:45.773 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T06:17:45.773 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T06:17:45.773 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T06:17:45.773 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T06:17:45.773 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_op_threads": "1", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T06:17:45.774 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_plugins": "", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T06:17:45.775 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T06:17:45.776 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_tracing": "false", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_validate_names": "true", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "rbd_validate_pool": "true", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "restapi_base_url": "", 2023-10-09T06:17:45.777 INFO:teuthology.orchestra.run.smithi107.stdout: "restapi_log_level": "", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_admin_entry": "admin", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_backend_store": "rados", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_barbican_url": "", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_beast_enable_async": "true", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T06:17:45.778 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_cache_enabled": "true", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_config_store": "rados", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_content_length_compat": "false", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_cross_domain_policy": "", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T06:17:45.779 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T06:17:45.780 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T06:17:45.781 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T06:17:45.782 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T06:17:45.783 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_log_window": "30", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T06:17:45.784 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T06:17:45.785 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dns_name": "", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dns_s3website_name": "", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T06:17:45.786 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_ops_log": "false", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_static_website": "false", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enable_usage_log": "false", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_expose_bucket": "false", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_extended_http_attrs": "", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_filter": "none", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T06:17:45.787 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_objs": "32", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_init_timeout": "300", 2023-10-09T06:17:45.788 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_password": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_project": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_token": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_admin_user": "", 2023-10-09T06:17:45.789 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_api_version": "2", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_url": "", 2023-10-09T06:17:45.790 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_max_objs": "32", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_max_worker": "3", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T06:17:45.791 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_log_http_headers": "", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_attr_size": "0", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T06:17:45.792 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_listing_results": "1000", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_notify_retries": "10", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T06:17:45.793 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T06:17:45.794 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_num_control_oids": "8", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_numa_node": "-1", 2023-10-09T06:17:45.795 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_op_tracing": "false", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_opa_token": "", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_opa_url": "", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T06:17:45.796 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ops_log_rados": "false", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_print_continue": "true", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T06:17:45.797 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_rados_tracing": "false", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_realm": "", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_realm_id": "", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_region": "", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T06:17:45.798 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_request_uri": "", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_resolve_cname": "false", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_run_sync_thread": "true", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T06:17:45.799 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_script_uri": "", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_service_provider_name": "", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_client_id": "", 2023-10-09T06:17:45.800 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_client_secret": "", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_entry": "sts", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_key": "sts", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_auth_url": "", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_custom_header": "", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_need_stats": "true", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_tenant_name": "", 2023-10-09T06:17:45.801 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_url": "", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_lease_period": "120", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T06:17:45.802 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_thread_pool_size": "512", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_comment": "", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_createby": "", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_encoding": "", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_flag": "false", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_origin": "", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T06:17:45.803 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_torrent_tracker": "", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_usage_max_shards": "32", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_use_opa_authz": "false", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T06:17:45.804 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_user_unique_email": "true", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_verify_ssl": "true", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zone": "", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zone_id": "", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zonegroup": "", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zonegroup_id": "", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T06:17:45.805 INFO:teuthology.orchestra.run.smithi107.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_block_size": "4096", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T06:17:45.806 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_partition_filters": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_perf": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T06:17:45.807 INFO:teuthology.orchestra.run.smithi107.stdout: "run_dir": "/var/run/ceph", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_block_create": "true", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_device_size": "53687091200", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_main_device_type": "SSD", 2023-10-09T06:17:45.808 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "seastore_segment_size": "67108864", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "set_keepcaps": "false", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "setgroup": "ceph", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "setuser": "ceph", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "setuser_match_path": "", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "thp": "false", 2023-10-09T06:17:45.809 INFO:teuthology.orchestra.run.smithi107.stdout: "threadpool_default_timeout": "60", 2023-10-09T06:17:45.810 INFO:teuthology.orchestra.run.smithi107.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T06:17:45.810 INFO:teuthology.orchestra.run.smithi107.stdout: "throttler_perf_counter": "true" 2023-10-09T06:17:45.810 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:17:46.214 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:45 smithi107 ceph-mon[212750]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:46.215 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:45 smithi107 ceph-mon[220921]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:46.230 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T06:17:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:45 smithi150 ceph-mon[194178]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:46.576 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:47.097 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:47.098 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:17:47.981 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a: {} 2023-10-09T06:17:47.991 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b: {} 2023-10-09T06:17:48.000 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c: {} 2023-10-09T06:17:48.001 INFO:teuthology.orchestra.run.smithi107.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:48.001 INFO:teuthology.orchestra.run.smithi107.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:48.002 INFO:teuthology.orchestra.run.smithi107.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T06:17:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:47 smithi107 ceph-mon[212750]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:48.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:47 smithi107 ceph-mon[220921]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:47 smithi150 ceph-mon[194178]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:48.570 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T06:17:48.570 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T06:17:48.570 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:17:48.570 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:17:48.571 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T06:17:48.571 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T06:17:48.571 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T06:17:48.571 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:17:48.571 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:17:48.572 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:17:48.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T06:17:48.572 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T06:17:48.572 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:48.573 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T06:17:48.573 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:17:48.573 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T06:17:48.573 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:17:48.574 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T06:17:48.574 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:17:48.623 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.673 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:48.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:48.780 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:48.834 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:48.900 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:48.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:49.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:49.034 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.082 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:49.085 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3588929404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:49.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3588929404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:49.186 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:49.217 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3588929404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:17:49.219 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.267 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:49.372 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:49.400 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.444 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:49.493 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:49.494 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:49.545 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:49.551 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:49.578 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.614 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:49.624 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:49.696 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:49.730 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:49.758 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.801 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:49.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:49.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:49.972 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:49.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:49.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:50.077 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:50.106 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:49 smithi150 ceph-mon[194178]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:50.237 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:49 smithi107 ceph-mon[212750]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:50.238 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:49 smithi107 ceph-mon[220921]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:50.402 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:50.506 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:50.534 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:50.578 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:50.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:50.683 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:50.712 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:51.107 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:51.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:51.210 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:17:51.237 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:51.832 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:17:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:51.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:17:51.936 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:51.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:52.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:51 smithi107 ceph-mon[212750]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:52.330 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:17:52] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.43.0" 2023-10-09T06:17:52.330 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:51 smithi107 ceph-mon[220921]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:51 smithi150 ceph-mon[194178]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:52.761 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:52.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:52.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:52.866 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:52.894 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:53.348 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:53.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:53.452 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:53.483 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:53.591 INFO:teuthology.orchestra.run.smithi107.stdout:moved 22 / 219 (10.0457%) 2023-10-09T06:17:53.591 INFO:teuthology.orchestra.run.smithi107.stdout:avg 27.375 2023-10-09T06:17:53.591 INFO:teuthology.orchestra.run.smithi107.stdout:stddev 4.89738 -> 6.57528 (expected baseline 4.89419) 2023-10-09T06:17:53.592 INFO:teuthology.orchestra.run.smithi107.stdout:min osd.2 with 18 -> 21 pgs (0.657534 -> 0.767123 * mean) 2023-10-09T06:17:53.592 INFO:teuthology.orchestra.run.smithi107.stdout:max osd.4 with 33 -> 26 pgs (1.20548 -> 0.949772 * mean) 2023-10-09T06:17:53.592 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:53.592 INFO:teuthology.orchestra.run.smithi107.stdout:oload 110 2023-10-09T06:17:53.592 INFO:teuthology.orchestra.run.smithi107.stdout:max_change 1 2023-10-09T06:17:53.593 INFO:teuthology.orchestra.run.smithi107.stdout:max_change_osds 4 2023-10-09T06:17:53.593 INFO:teuthology.orchestra.run.smithi107.stdout:average_utilization 0.0037 2023-10-09T06:17:53.593 INFO:teuthology.orchestra.run.smithi107.stdout:overload_utilization 0.0041 2023-10-09T06:17:53.593 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 weight 1.0000 -> 0.5785 2023-10-09T06:17:53.593 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 weight 1.0000 -> 0.5905 2023-10-09T06:17:53.594 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 weight 1.0000 -> 0.8496 2023-10-09T06:17:53.594 INFO:teuthology.orchestra.run.smithi107.stderr:no change 2023-10-09T06:17:53.837 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:53.843 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:17:53 smithi107 conmon[200038]: 2023-10-09T06:17:53.590+0000 7f5488b69700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T06:17:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:53.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:53.945 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:17:53.978 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:53.987 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:17:54.028 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:17:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:54.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:17:54.134 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:54.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[212750]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[212750]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:17:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[220921]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[220921]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:17:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:54.163 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:54.211 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:17:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:17:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:17:54.211 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:17:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:53 smithi150 ceph-mon[194178]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:53 smithi150 ceph-mon[194178]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:17:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:17:54.962 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:17:54.980 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:17:54.999 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:17:55.018 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:17:55.036 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:17:55.055 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:17:55.074 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:17:55.092 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:17:55.114 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:55.114 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.114 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:55.114 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:17:55.114 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2936206689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2936206689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:55.327 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1733139988' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: osdmap e151: 8 total, 8 up, 8 in 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2936206689' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:17:55.515 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:55.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:55.598 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:17:55.598 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-09T06:17:55.619 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:55.652 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:55.701 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:55.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:55.810 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:55.839 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:55.883 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:55.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:55.990 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:56.004 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_4' created 2023-10-09T06:17:56.019 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.061 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:56.166 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:56.192 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.241 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:56.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:56.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:17:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.245 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.246 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.246 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:17:56.345 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:17:56.371 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: failed to encode map e151 with expected crc 2023-10-09T06:17:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T06:17:56.413 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:17:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:17:56.517 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:17:56.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.587 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:17:56.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:17:56.692 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:56.718 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:56.760 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:56.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:56.774 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:56.799 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T06:17:56.825 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:56.868 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:56.973 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[212750]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:57 smithi107 ceph-mon[220921]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.040 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:57.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:57.144 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:17:57.172 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:57.213 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:17:57.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:57.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:17:57.318 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:57.350 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2398772339' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: osdmap e152: 8 total, 8 up, 8 in 2023-10-09T06:17:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:57 smithi150 ceph-mon[194178]: failed to encode map e152 with expected crc 2023-10-09T06:17:57.447 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:17:58.571 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:17:58.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:17:58.573 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:58 smithi150 ceph-mon[194178]: pgmap v518: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:58.573 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:58 smithi150 ceph-mon[194178]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.573 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:58 smithi150 ceph-mon[194178]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.573 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:58 smithi150 ceph-mon[194178]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:17:58.573 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:58 smithi150 ceph-mon[194178]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:58.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:17:58.575 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[212750]: pgmap v518: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:58.575 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.575 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.575 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[212750]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:17:58.576 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[212750]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.576 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[220921]: pgmap v518: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:17:58.625 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[220921]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.625 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[220921]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.626 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[220921]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T06:17:58.626 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:58 smithi107 ceph-mon[220921]: failed to encode map e153 with expected crc 2023-10-09T06:17:58.626 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:17:58.676 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:17:58.778 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:17:58.779 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:17:58.809 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:58.859 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:17:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:58.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:17:58.964 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:17:58.997 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:17:59.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:59 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:17:59.140 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:17:59 smithi107 ceph-mon[220921]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:17:59.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:59 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:17:59.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:17:59 smithi107 ceph-mon[212750]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:17:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:59 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:17:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:17:59 smithi150 ceph-mon[194178]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T06:17:59.155 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:17:59.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:17:59.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:moved 0 / 267 (0%) 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:avg 33.375 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:stddev 6.22369 -> 6.22369 (expected baseline 5.40399) 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:min osd.2 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:oload 1000 2023-10-09T06:17:59.237 INFO:teuthology.orchestra.run.smithi107.stdout:max_change 3 2023-10-09T06:17:59.238 INFO:teuthology.orchestra.run.smithi107.stdout:max_change_osds 4 2023-10-09T06:17:59.238 INFO:teuthology.orchestra.run.smithi107.stdout:average_utilization 0.0037 2023-10-09T06:17:59.238 INFO:teuthology.orchestra.run.smithi107.stdout:overload_utilization 0.0372 2023-10-09T06:17:59.239 INFO:teuthology.orchestra.run.smithi107.stderr:no change 2023-10-09T06:17:59.258 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:17:59.287 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: pgmap v521: 89 pgs: 2 creating+peering, 3 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: pgmap v521: 89 pgs: 2 creating+peering, 3 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:00.235 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:00.340 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:00.369 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:18:00.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: pgmap v521: 89 pgs: 2 creating+peering, 3 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:00.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.372 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.372 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: failed to encode map e154 with expected crc 2023-10-09T06:18:00.372 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:00.372 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:18:00.373 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.419 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:00.525 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:00.552 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.649 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:18:00.650 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T06:18:00.702 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:00.748 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:00.809 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:00.838 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:00.884 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:00.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:00.943 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:00.987 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:00.996 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:01.042 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:01.985 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:01.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:01.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:02.088 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:02.120 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:02.143 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:02] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2023-10-09T06:18:02.213 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:02.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:02.328 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:02.354 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:02.403 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:02.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:02.507 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:02.540 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[212750]: pgmap v522: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[220921]: pgmap v522: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:02.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:02 smithi150 ceph-mon[194178]: pgmap v522: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:03.242 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:03.256 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:03.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:03.362 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:03.391 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.622 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:03.668 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T06:18:03.726 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:03.759 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:03.805 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:03.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:03.894 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[220921]: pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4105433425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.895 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[212750]: pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:03.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4105433425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:03 smithi150 ceph-mon[194178]: pgmap v523: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4105433425' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:18:03.913 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:03.941 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:04.581 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:04.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:04.665 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:04.684 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:04.686 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:04.706 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:04.722 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:04.725 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:04.748 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:04.768 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:04.771 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:04.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:04.786 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:04.805 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.807 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:04.864 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:18:04.864 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:18:04.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:18:04.864 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:83179 7.9 7:983970d6:::benchmark_data_smithi107_237467_object83178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e94)", 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:09:00.242488+0000", 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 544.61916735399996, 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0033781829999999, 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:18:04.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 83179 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242488+0000", 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242488+0000", 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242487+0000", 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:04.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242513+0000", 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6296e-05 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242527+0000", 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4239e-05 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.242537+0000", 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6889999999999999e-06 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.247410+0000", 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0048728110000000003 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:00.247422+0000", 2023-10-09T06:18:04.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2384999999999999e-05 2023-10-09T06:18:04.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:01.245771+0000", 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99834889199999999 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:09:01.245867+0000", 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5589999999999998e-05 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164028 7.6 7:6e57a499:::benchmark_data_smithi107_237467_object61139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:18:04.872 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 424.98433347999998, 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0265854210000001, 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164028 2023-10-09T06:18:04.873 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.874 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877320+0000", 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:04.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877332+0000", 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1945e-05 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877355+0000", 2023-10-09T06:18:04.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2385e-05 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877367+0000", 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2782999999999999e-05 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879335+0000", 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019675890000000001 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879341+0000", 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9309999999999996e-06 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.878 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.903847+0000", 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0245063409999999 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.903908+0000", 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0418000000000003e-05 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.879 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164036 7.6 7:68fb9b1f:::benchmark_data_smithi107_237467_object61147:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 424.98237431699999, 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.070745885, 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164036 2023-10-09T06:18:04.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879280+0000", 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879287+0000", 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5100000000000004e-06 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.882 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879299+0000", 2023-10-09T06:18:04.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2593e-05 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879314+0000", 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4611e-05 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879352+0000", 2023-10-09T06:18:04.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8498000000000003e-05 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879356+0000", 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1970000000000001e-06 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904149+0000", 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0247934160000001 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904218+0000", 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9089000000000002e-05 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.949949+0000", 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.045731115000000003 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.950027+0000", 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8215999999999998e-05 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164055 7.6 7:68939ae2:::benchmark_data_smithi107_237467_object61166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 424.96210086000002, 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.014796101, 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.887 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164055 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899554+0000", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899559+0000", 2023-10-09T06:18:04.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7469999999999996e-06 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899570+0000", 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899577+0000", 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1999999999999997e-06 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.903132+0000", 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003554911 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.903144+0000", 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2489e-05 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904266+0000", 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0011211259999999 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904350+0000", 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4363000000000005e-05 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904437+0000", 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6550000000000003e-05 2023-10-09T06:18:04.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.911505+0000", 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070688929999999997 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914120+0000", 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002614487 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914178+0000", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8471999999999999e-05 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914232+0000", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3752999999999998e-05 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914351+0000", 2023-10-09T06:18:04.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011889 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164064 7.6 7:6bc792c6:::benchmark_data_smithi107_237467_object61175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.922177+0000", 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 424.93947891900001, 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.999634366, 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.895 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164064 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922177+0000", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922177+0000", 2023-10-09T06:18:04.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922175+0000", 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922185+0000", 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0081000000000001e-05 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922204+0000", 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8856999999999999e-05 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.922215+0000", 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1686e-05 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.927335+0000", 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051201720000000001 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.927349+0000", 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3841e-05 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.908203+0000", 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98085335399999996 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.908357+0000", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015410200000000001 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.908500+0000", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014296400000000001 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914444+0000", 2023-10-09T06:18:04.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059443849999999999 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914484+0000", 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0028999999999999e-05 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.921651+0000", 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0071667880000000003 2023-10-09T06:18:04.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.921709+0000", 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7969e-05 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.921811+0000", 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000102292 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.902 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53739 8.1 8:8560477b:::benchmark_data_smithi107_251897_object53738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 319.96603299100002, 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011737895, 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53739 2023-10-09T06:18:04.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895622+0000", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895646+0000", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4346e-05 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895658+0000", 2023-10-09T06:18:04.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1829e-05 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895667+0000", 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0189999999999995e-06 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895725+0000", 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7825999999999998e-05 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895734+0000", 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3729999999999999e-06 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907232+0000", 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0114980069999999 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907361+0000", 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012863999999999999 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53743 8.1 8:8dd73d4f:::benchmark_data_smithi107_251897_object53742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 319.96596909700003, 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0119421850000001, 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.908 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53743 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895686+0000", 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895706+0000", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9834999999999999e-05 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895713+0000", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.691e-06 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895718+0000", 2023-10-09T06:18:04.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6189999999999999e-06 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8346000000000002e-05 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895762+0000", 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8189999999999997e-06 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907554+0000", 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0117913940000001 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907629+0000", 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4975999999999995e-05 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53745 8.1 8:89b04a6c:::benchmark_data_smithi107_251897_object53744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 319.96593228400002, 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011994064, 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53745 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.914 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895741+0000", 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7733000000000001e-05 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895747+0000", 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.596e-06 2023-10-09T06:18:04.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895753+0000", 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2510000000000002e-06 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895784+0000", 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0846999999999998e-05 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895789+0000", 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6219999999999997e-06 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907652+0000", 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0118627149999999 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907718+0000", 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.567e-05 2023-10-09T06:18:04.918 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53746 8.1 8:8f4403ee:::benchmark_data_smithi107_251897_object53745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 319.96589847600001, 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.023758089, 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.919 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53746 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:18:04.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895775+0000", 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7768e-05 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895780+0000", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6509999999999998e-06 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895785+0000", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8169999999999998e-06 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895808+0000", 2023-10-09T06:18:04.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3003e-05 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895813+0000", 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2610000000000001e-06 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907740+0000", 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0119261020000001 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907864+0000", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012438099999999999 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.919426+0000", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011562413000000001 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.919515+0000", 2023-10-09T06:18:04.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9061000000000007e-05 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75980 8.1 8:85ec2575:::benchmark_data_smithi107_251897_object75979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 302.03433893800002, 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0842550950000001, 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.925 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75980 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827315+0000", 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827343+0000", 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7511000000000001e-05 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827360+0000", 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7618999999999999e-05 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827372+0000", 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1697e-05 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827469+0000", 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7077999999999994e-05 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827478+0000", 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829307+0000", 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018292200000000001 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829320+0000", 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2662e-05 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900507+0000", 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0711866210000001 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900688+0000", 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018153099999999999 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900863+0000", 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017435399999999999 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907377+0000", 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065140379999999998 2023-10-09T06:18:04.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907392+0000", 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5113000000000001e-05 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910524+0000", 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031320279999999998 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910577+0000", 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3477999999999999e-05 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911572+0000", 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00099476300000000007 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.933 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75985 8.1 8:844c4be0:::benchmark_data_smithi107_251897_object75984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 302.03426899900001, 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0907094479999999, 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75985 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.934 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827386+0000", 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827408+0000", 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1869999999999999e-05 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827416+0000", 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3399999999999998e-06 2023-10-09T06:18:04.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827423+0000", 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7270000000000001e-06 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829369+0000", 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00194557 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829379+0000", 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.006e-05 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904462+0000", 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0750832909999999 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904613+0000", 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015070900000000001 2023-10-09T06:18:04.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904744+0000", 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013179500000000001 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910436+0000", 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056916629999999996 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910480+0000", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4121e-05 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.917077+0000", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065964780000000002 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.917136+0000", 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9684e-05 2023-10-09T06:18:04.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918096+0000", 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00095980100000000004 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75993 8.1 8:8297030a:::benchmark_data_smithi107_251897_object75992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:18:04.941 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 302.03373883199998, 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0907953589999999, 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75993 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827916+0000", 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827963+0000", 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6594999999999997e-05 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827973+0000", 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0414999999999999e-05 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827982+0000", 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.49e-06 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829408+0000", 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014259850000000001 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829417+0000", 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1819999999999995e-06 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907421+0000", 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.078004368 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907583+0000", 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016161499999999999 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:18:04.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907778+0000", 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000194971 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914689+0000", 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069108240000000003 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914704+0000", 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5427999999999999e-05 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918139+0000", 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034351619999999999 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918191+0000", 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1901000000000003e-05 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918712+0000", 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052121800000000005 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:76002 8.1 8:8589f225:::benchmark_data_smithi107_251897_object76001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 302.03366715200002, 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.091170872, 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.949 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 76002 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:18:04.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827988+0000", 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828043+0000", 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5210999999999997e-05 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828051+0000", 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7829999999999999e-06 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828057+0000", 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7989999999999999e-06 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829442+0000", 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013856490000000001 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829448+0000", 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8200000000000002e-06 2023-10-09T06:18:04.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911665+0000", 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0822164009999999 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911818+0000", 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015297800000000001 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911933+0000", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115646 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914801+0000", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028680709999999998 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.915159+0000", 2023-10-09T06:18:04.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035749499999999999 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918752+0000", 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003592961 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918805+0000", 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3224999999999999e-05 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.919159+0000", 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035442399999999999 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162302 8.e 8:7101b874:::benchmark_data_smithi107_251897_object45388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 193.70130470199999, 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.060445393, 2023-10-09T06:18:04.957 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162302 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160349+0000", 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160359+0000", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4290000000000007e-06 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160380+0000", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1739999999999999e-05 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160390+0000", 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2269999999999999e-06 2023-10-09T06:18:04.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161137+0000", 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00074714099999999999 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161255+0000", 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000118217 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161363+0000", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108325 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.166561+0000", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051975420000000003 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173923+0000", 2023-10-09T06:18:04.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073616810000000001 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188983+0000", 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015060772 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188993+0000", 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.005e-05 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.220709+0000", 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.031715916 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.220796+0000", 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7115999999999997e-05 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162314 8.e 8:779025f7:::benchmark_data_smithi107_251897_object45400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 193.687813417, 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0473069500000001, 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162314 2023-10-09T06:18:04.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173844+0000", 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3350000000000001e-06 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173851+0000", 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4710000000000004e-06 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173857+0000", 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8340000000000004e-06 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173948+0000", 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0878000000000006e-05 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.174058+0000", 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011074299999999999 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.174174+0000", 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115643 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.178772+0000", 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045980989999999996 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189024+0000", 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01025155 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189031+0000", 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1559999999999998e-06 2023-10-09T06:18:04.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221086+0000", 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0320555389999999 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221149+0000", 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3066999999999998e-05 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162323 8.e 8:7b14d2dc:::benchmark_data_smithi107_251897_object45409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 193.67302232099999, 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.047554023, 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162323 2023-10-09T06:18:04.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188632+0000", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188639+0000", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2869999999999999e-06 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188651+0000", 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1702000000000001e-05 2023-10-09T06:18:04.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188660+0000", 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8060000000000001e-06 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189123+0000", 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046345499999999999 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189130+0000", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.579e-06 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221200+0000", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0320701640000001 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221288+0000", 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8208000000000007e-05 2023-10-09T06:18:04.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4", 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221369+0000", 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0900000000000001e-05 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231308+0000", 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099386189999999992 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236045+0000", 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047374510000000002 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236096+0000", 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0469000000000001e-05 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236187+0000", 2023-10-09T06:18:04.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1762999999999995e-05 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176913 8.e 8:751e71d0:::benchmark_data_smithi107_251897_object59999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 173.145510312, 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.154407339, 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.979 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176913 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:18:04.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716148+0000", 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.286e-06 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716155+0000", 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716160+0000", 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9740000000000001e-06 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716285+0000", 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012556700000000001 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716296+0000", 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.098e-05 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.860697+0000", 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.14440092099999999 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.860709+0000", 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1685000000000001e-05 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859539+0000", 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9988303840000001 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859613+0000", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3980000000000007e-05 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859727+0000", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114228 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.866046+0000", 2023-10-09T06:18:04.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0063190549999999996 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.866078+0000", 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1996999999999999e-05 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870327+0000", 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0042490569999999997 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870388+0000", 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1098000000000004e-05 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870553+0000", 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016430500000000001 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:192018 8.e 8:708941a0:::benchmark_data_smithi107_251897_object75104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 150.79323547000001, 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.012711275, 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 192018 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.988 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068423+0000", 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1190000000000001e-06 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068429+0000", 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4470000000000003e-06 2023-10-09T06:18:04.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068435+0000", 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7030000000000003e-06 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.069829+0000", 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001393949 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.069840+0000", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1068e-05 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.075899+0000", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.006058517 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.076004+0000", 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105814 2023-10-09T06:18:04.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.076113+0000", 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109028 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081010+0000", 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0048969770000000003 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081063+0000", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2398999999999998e-05 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081074+0000", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.128e-05 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081109+0000", 2023-10-09T06:18:04.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4637000000000001e-05 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081132+0000", 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2836e-05 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231197 8.c 8:3d5e510f:::benchmark_data_smithi107_251897_object114283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:18:04.995 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 95.273869798000007, 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0702522830000001, 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231197 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:04.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587785+0000", 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:04.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587792+0000", 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0180000000000004e-06 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587807+0000", 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4932e-05 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:04.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587817+0000", 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0679999999999999e-05 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587874+0000", 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6702e-05 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588034+0000", 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016021899999999999 2023-10-09T06:18:04.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588171+0000", 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013668400000000001 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.598003+0000", 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0098321499999999996 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:05.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.605004+0000", 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070009989999999999 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.605013+0000", 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1630000000000002e-06 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.657929+0000", 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0529163100000001 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658038+0000", 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108799 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231205 8.c 8:3765e502:::benchmark_data_smithi107_251897_object114291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 95.249345590999994, 2023-10-09T06:18:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.056454129, 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231205 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612308+0000", 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612315+0000", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1620000000000002e-06 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612332+0000", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6688e-05 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612343+0000", 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0988000000000001e-05 2023-10-09T06:18:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612397+0000", 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3671000000000002e-05 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612410+0000", 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3529e-05 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658355+0000", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0459445469999999 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658467+0000", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000111896 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 7", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658540+0000", 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3749000000000002e-05 2023-10-09T06:18:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.665781+0000", 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072405739999999996 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668654+0000", 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028725000000000001 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668685+0000", 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1189999999999998e-05 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668764+0000", 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9584000000000002e-05 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:05.010 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:18:05.010 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[212750]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:18:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[212750]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2403849777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:05.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[220921]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:05.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[220921]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:18:05.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[220921]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.012 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2403849777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:05.036 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:04 smithi150 ceph-mon[194178]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:18:05.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:04 smithi150 ceph-mon[194178]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T06:18:05.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:04 smithi150 ceph-mon[194178]: failed to encode map e155 with expected crc 2023-10-09T06:18:05.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2403849777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:05.039 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:05.085 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:05.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:05.193 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:05.224 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:05.448 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T06:18:05.451 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:18:05.451 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:18:05.564 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:05.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:05.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:05.668 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:05.695 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:05.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:18:05.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:18:05.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_300016 2023-10-09T06:18:05.822 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:05.823 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:18:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:05 smithi107 ceph-mon[212750]: pgmap v525: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:05 smithi107 ceph-mon[220921]: pgmap v525: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:05.928 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:05.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:05.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:06.035 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:06.086 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:05 smithi150 ceph-mon[194178]: pgmap v525: 89 pgs: 89 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-09T06:18:06.257 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:18:06.432 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:06.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:06.547 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:06.584 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:06.646 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:06.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:06.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:06.752 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:06.786 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:06.823 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2985 2969 185.552 185.562 0.00568671 0.00535972 2023-10-09T06:18:06.847 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:06.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:06.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:06.996 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.058 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:07.166 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:07.211 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.238 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:07.284 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:07.391 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:07.432 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.452 INFO:teuthology.orchestra.run.smithi107.stdout:moved 26 / 267 (9.73783%) 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:avg 33.375 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:stddev 6.22369 -> 8.52845 (expected baseline 5.40399) 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:min osd.2 with 21 -> 25 pgs (0.629214 -> 0.749064 * mean) 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:max osd.1 with 41 -> 46 pgs (1.22846 -> 1.37828 * mean) 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:oload 110 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:max_change 1 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:max_change_osds 4 2023-10-09T06:18:07.453 INFO:teuthology.orchestra.run.smithi107.stdout:average_utilization 0.0037 2023-10-09T06:18:07.454 INFO:teuthology.orchestra.run.smithi107.stdout:overload_utilization 0.0041 2023-10-09T06:18:07.454 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 weight 1.0000 -> 0.5785 2023-10-09T06:18:07.454 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4 weight 1.0000 -> 0.5905 2023-10-09T06:18:07.454 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0 weight 1.0000 -> 0.8495 2023-10-09T06:18:07.458 INFO:teuthology.orchestra.run.smithi107.stderr:no change 2023-10-09T06:18:07.507 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:07.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:07.614 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:07.676 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:07.793 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:07 smithi107 conmon[200038]: 2023-10-09T06:18:07.445+0000 7f5488b69700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T06:18:07.823 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 5923 5907 184.577 183.625 0.00314531 0.00540931 2023-10-09T06:18:07.996 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:18:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[212750]: pgmap v526: 89 pgs: 89 active+clean; 7.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 807 KiB/s wr, 12 op/s 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2137480612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[212750]: from='client.45287 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[220921]: pgmap v526: 89 pgs: 89 active+clean; 7.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 807 KiB/s wr, 12 op/s 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2137480612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:07 smithi107 ceph-mon[220921]: from='client.45287 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:07 smithi150 ceph-mon[194178]: pgmap v526: 89 pgs: 89 active+clean; 7.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 807 KiB/s wr, 12 op/s 2023-10-09T06:18:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2137480612' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:07 smithi150 ceph-mon[194178]: from='client.45287 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:08.300 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:08.824 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 9293 9277 193.154 210.625 0.00382613 0.00516886 2023-10-09T06:18:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:18:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[212750]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:08 smithi107 ceph-mon[220921]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:08 smithi150 ceph-mon[194178]: failed to encode map e156 with expected crc 2023-10-09T06:18:09.825 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 12149 12133 189.432 178.5 0.00738925 0.00527235 2023-10-09T06:18:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:09 smithi107 ceph-mon[212750]: pgmap v528: 89 pgs: 89 active+clean; 281 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:18:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:09 smithi107 ceph-mon[220921]: pgmap v528: 89 pgs: 89 active+clean; 281 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:18:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:09 smithi150 ceph-mon[194178]: pgmap v528: 89 pgs: 89 active+clean; 281 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s wr, 559 op/s 2023-10-09T06:18:10.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 15189 15173 189.538 190 0.00468117 0.00526998 2023-10-09T06:18:11.099 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:11.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:11.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:11.203 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:11.253 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:11.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 17550 17534 182.543 147.562 0.00218565 0.0054279 2023-10-09T06:18:11.874 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:11.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:11.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:11.978 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:12.032 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:12.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:12] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2023-10-09T06:18:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:11 smithi107 ceph-mon[212750]: pgmap v529: 89 pgs: 89 active+clean; 421 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-09T06:18:12.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:11 smithi107 ceph-mon[220921]: pgmap v529: 89 pgs: 89 active+clean; 421 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-09T06:18:12.300 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:18:12.301 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-10-09T06:18:12.358 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:12.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:11 smithi150 ceph-mon[194178]: pgmap v529: 89 pgs: 89 active+clean; 421 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-09T06:18:12.474 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:12.557 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:12.716 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:12.773 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:12.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:12.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 15 19721 19706 175.86 135.75 0.00407626 0.00568091 2023-10-09T06:18:12.881 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:12.974 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:13.224 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:13.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:20612 9.a e156/152 9:5d94a416:::benchmark_data_smithi107_300016_object20611:head v 156'1288, mlcod=156'1286)", 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.311063+0000", 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0055929910000000003, 2023-10-09T06:18:13.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056285889999999998, 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311063+0000", 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311063+0000", 2023-10-09T06:18:13.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311062+0000", 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311135+0000", 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3256000000000005e-05 2023-10-09T06:18:13.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311142+0000", 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.635e-06 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311146+0000", 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.233e-06 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311186+0000", 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9585999999999997e-05 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311206+0000", 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9740000000000001e-05 2023-10-09T06:18:13.324 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:20609 9.e e156/152 9:7e97cea6:::benchmark_data_smithi107_300016_object20608:head v 156'1272, mlcod=156'1270)", 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.311118+0000", 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0055375559999999999, 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056191449999999999, 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:13.325 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311118+0000", 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311118+0000", 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311118+0000", 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311193+0000", 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5042000000000004e-05 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311199+0000", 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4799999999999998e-06 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311204+0000", 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0250000000000002e-06 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311257+0000", 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2901000000000001e-05 2023-10-09T06:18:13.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311276+0000", 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8831999999999999e-05 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312419+0000", 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001142419 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.329 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:20610 9.2 e156/152 9:42faed6b:::benchmark_data_smithi107_300016_object20609:head v 156'1312, mlcod=156'1310)", 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.311229+0000", 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0054272249999999999, 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055650090000000001, 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:18:13.330 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311229+0000", 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311229+0000", 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311228+0000", 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311263+0000", 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4459999999999999e-05 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311270+0000", 2023-10-09T06:18:13.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4039999999999998e-06 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311275+0000", 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9819999999999999e-06 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311308+0000", 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3337000000000003e-05 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.311328+0000", 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9347999999999998e-05 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:20613 9.0 9:05966b0d:::benchmark_data_smithi107_300016_object20612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.312051+0000", 2023-10-09T06:18:13.334 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0046048360000000002, 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047984719999999998, 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 20613 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312051+0000", 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312051+0000", 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312050+0000", 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312087+0000", 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6665999999999998e-05 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312098+0000", 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1619e-05 2023-10-09T06:18:13.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312105+0000", 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3100000000000003e-06 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312145+0000", 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9919999999999997e-05 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.314861+0000", 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027154739999999998 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.315021+0000", 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000160116 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.339 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:20616 9.4 9:25445ad2:::benchmark_data_smithi107_300016_object20615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.312117+0000", 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0045385149999999999, 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0048222100000000004, 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:18:13.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 20616 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312117+0000", 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312117+0000", 2023-10-09T06:18:13.341 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312117+0000", 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312144+0000", 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6849999999999999e-05 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312152+0000", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8390000000000007e-06 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.312158+0000", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9499999999999998e-06 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316316+0000", 2023-10-09T06:18:13.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0041580979999999998 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316454+0000", 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000138398 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316605+0000", 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015097099999999999 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.344 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:20613 9.0 e156/152)", 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:13.316293+0000", 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.000362419, 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00072317799999999999, 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "queued for pg", 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:13.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316293+0000", 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316293+0000", 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316292+0000", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316299+0000", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.956e-06 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316314+0000", 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4134000000000001e-05 2023-10-09T06:18:13.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:13.316322+0000", 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7509999999999997e-06 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 6 2023-10-09T06:18:13.348 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:18:13.349 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:13.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 21897 21881 170.87 135.938 0.00480093 0.00584739 2023-10-09T06:18:14.119 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.255 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:14.363 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:14.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:13 smithi107 ceph-mon[212750]: pgmap v530: 89 pgs: 89 active+clean; 699 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:18:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:13 smithi107 ceph-mon[220921]: pgmap v530: 89 pgs: 89 active+clean; 699 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:18:14.405 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:18:14.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:13 smithi150 ceph-mon[194178]: pgmap v530: 89 pgs: 89 active+clean; 699 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:18:14.755 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:14.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 23925 23909 165.958 126.75 0.00564175 0.00601969 2023-10-09T06:18:14.862 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:14.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:14.969 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:15.074 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.120 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:15.225 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:15.252 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.294 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:15.399 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:15.453 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.732 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:18:15.773 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:15.783 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:15.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 25372 25356 158.408 90.4375 0.00974734 0.00630672 2023-10-09T06:18:15.890 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:15.924 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:15.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:15 smithi107 ceph-mon[212750]: pgmap v531: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:18:15.991 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:15.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:16.099 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:16.175 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.234 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:16.358 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:15 smithi107 ceph-mon[220921]: pgmap v531: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:18:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:15 smithi150 ceph-mon[194178]: pgmap v531: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:18:16.506 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:16.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:16.614 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:16.679 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:16.746 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:16.802 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:16.826 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 26840 26824 152.349 91.75 0.0127218 0.00655777 2023-10-09T06:18:16.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:16.887 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:16.928 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:16.961 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:16.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:16.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:16.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:16.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4180719034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:16.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:16.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:16.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:16.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:16.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:16.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:16.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4180719034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:16.998 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:17.035 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:17.038 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.038 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:17.241 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:17.241 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:17.241 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4180719034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:18:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:17.827 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 28596 28580 148.799 109.75 0.00215083 0.00671036 2023-10-09T06:18:18.388 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:18.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:18.392 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:17 smithi107 ceph-mon[212750]: pgmap v532: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T06:18:18.392 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:17 smithi107 ceph-mon[220921]: pgmap v532: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T06:18:18.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:17 smithi150 ceph-mon[194178]: pgmap v532: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-09T06:18:18.493 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:18.557 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:18.827 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 30409 30393 146.069 113.312 0.00602665 0.00684132 2023-10-09T06:18:18.922 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:18.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:19.027 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:19.053 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:19.502 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:19.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:19.611 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:19.665 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:19.775 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:19.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:19.827 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 31838 31822 142.016 89.3125 0.0103017 0.00703651 2023-10-09T06:18:19.852 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:19.887 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:19.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.017 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:20.052 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:19 smithi107 ceph-mon[212750]: pgmap v533: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:18:20.052 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:19 smithi107 ceph-mon[220921]: pgmap v533: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:18:20.125 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:20.185 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:19 smithi150 ceph-mon[194178]: pgmap v533: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T06:18:20.516 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:20.627 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:20.729 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:20.758 INFO:teuthology.orchestra.run.smithi107.stdout:moved 0 / 267 (0%) 2023-10-09T06:18:20.758 INFO:teuthology.orchestra.run.smithi107.stdout:avg 33.375 2023-10-09T06:18:20.758 INFO:teuthology.orchestra.run.smithi107.stdout:stddev 6.22369 -> 6.22369 (expected baseline 5.40399) 2023-10-09T06:18:20.758 INFO:teuthology.orchestra.run.smithi107.stdout:min osd.2 with 21 -> 21 pgs (0.629214 -> 0.629214 * mean) 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:oload 1000 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:max_change 1 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:max_change_osds 4 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:average_utilization 0.0109 2023-10-09T06:18:20.759 INFO:teuthology.orchestra.run.smithi107.stdout:overload_utilization 0.1092 2023-10-09T06:18:20.768 INFO:teuthology.orchestra.run.smithi107.stderr:no change 2023-10-09T06:18:20.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 33478 33462 139.357 102.5 0.0110726 0.00717022 2023-10-09T06:18:20.881 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:20.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:20.987 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:21.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 35453 35437 138.362 123.438 0.00240838 0.00721857 2023-10-09T06:18:21.873 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:21.928 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.026 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:18:22.027 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:22.135 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[212750]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[212750]: pgmap v534: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:22] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.43.0" 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[220921]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[220921]: pgmap v534: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:18:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:22 smithi150 ceph-mon[194178]: from='client.35379 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:22 smithi150 ceph-mon[194178]: pgmap v534: 89 pgs: 89 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-09T06:18:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:18:22.179 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.222 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:22.328 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:22.387 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.794 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:22.828 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T06:18:22.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-09T06:18:22.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 36796 36780 135.161 83.9375 0.00646159 0.00739369 2023-10-09T06:18:22.901 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:22.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:22.982 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:22.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:23.087 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:23.113 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[212750]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:23 smithi107 ceph-mon[220921]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.154 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:23.260 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:23.295 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2073480481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:23 smithi150 ceph-mon[194178]: failed to encode map e157 with expected crc 2023-10-09T06:18:23.596 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:23.702 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:23.754 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:23.806 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:23.827 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:23.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 37944 37928 131.639 71.75 0.0081315 0.00758978 2023-10-09T06:18:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:23.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:23.858 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:23.859 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:23.889 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:23.932 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:23.992 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:24 smithi107 ceph-mon[220921]: pgmap v536: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T06:18:24.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:24 smithi107 ceph-mon[212750]: pgmap v536: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T06:18:24.299 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:24.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:24.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:24 smithi150 ceph-mon[194178]: pgmap v536: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-10-09T06:18:24.471 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.562 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:24.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:24.675 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:24.724 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:24.806 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:24.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:24.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:24.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 39720 39704 130.553 111 0.00708856 0.00765542 2023-10-09T06:18:24.914 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:24.970 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:25.041 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:25.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:25.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:25.049 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:18:25.147 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:25.191 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:25.265 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:25.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:25.372 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:25.433 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:25.608 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:18:25.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:18:25.830869+0000 min lat: 0.00127393 max lat: 0.269441 avg lat: 0.00773586 2023-10-09T06:18:25.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:25.830 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 41371 41355 129.185 103.188 0.00513642 0.00773586 2023-10-09T06:18:26.292 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:26 smithi107 ceph-mon[212750]: pgmap v537: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:18:26.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:26 smithi107 ceph-mon[220921]: pgmap v537: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:18:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:26 smithi150 ceph-mon[194178]: pgmap v537: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T06:18:26.701 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:26.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:26.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:26.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:26.831 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 42530 42514 126.474 72.4375 0.0115815 0.00790185 2023-10-09T06:18:26.882 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:26.986 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:26.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:27.084 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:27.147 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:27.201 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:27.286 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:27.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:27.352 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:27.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:27.465 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:27.600 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:27.635 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:27.672 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:27.678 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:27.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:27.724 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:27.763 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:27.781 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:27.792 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:27.822 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:27.831 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 43971 43955 124.819 90.0625 0.020877 0.00800501 2023-10-09T06:18:27.832 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:27.872 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:27.873 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.874 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:27.885 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:27.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:27.992 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:28.031 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:28 smithi107 ceph-mon[212750]: pgmap v538: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:18:28.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3455170937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:28.104 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:28.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:28 smithi150 ceph-mon[194178]: pgmap v538: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:18:28.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3455170937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:28.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:28.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:28.213 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:28.248 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.315 INFO:teuthology.orchestra.run.smithi107.stdout:moved 36 / 267 (13.4831%) 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:avg 33.375 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:stddev 6.22369 -> 8.09224 (expected baseline 5.40399) 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:min osd.2 with 21 -> 28 pgs (0.629214 -> 0.838951 * mean) 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:max osd.1 with 41 -> 36 pgs (1.22846 -> 1.07865 * mean) 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:oload 110 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:max_change 3 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:max_change_osds 4 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:average_utilization 0.0142 2023-10-09T06:18:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:overload_utilization 0.0156 2023-10-09T06:18:28.317 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5 weight 1.0000 -> 0.6054 2023-10-09T06:18:28.317 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7 weight 1.0000 -> 0.6999 2023-10-09T06:18:28.317 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1 weight 1.0000 -> 0.7721 2023-10-09T06:18:28.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:28 smithi107 ceph-mon[220921]: pgmap v538: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T06:18:28.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3455170937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:28.317 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:28 smithi107 conmon[200038]: 2023-10-09T06:18:28.315+0000 7f5488b69700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T06:18:28.324 INFO:teuthology.orchestra.run.smithi107.stderr:no change 2023-10-09T06:18:28.329 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:28.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:28.440 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:28.500 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.580 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:28.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:28.691 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:28.762 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:28.833 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 45503 45487 123.55 95.75 0.00825752 0.00808849 2023-10-09T06:18:29.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:29 smithi107 ceph-mon[212750]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:29.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:29 smithi107 ceph-mon[220921]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:29.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:29 smithi150 ceph-mon[194178]: from='client.45299 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T06:18:29.459 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:29.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:29.569 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:29.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:29.636 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:29.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:29.742 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:29.791 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:29.833 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 46698 46682 121.514 74.6875 0.00692058 0.00820593 2023-10-09T06:18:30.051 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:30 smithi107 ceph-mon[212750]: pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T06:18:30.308 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:30.312 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:30 smithi107 ceph-mon[220921]: pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T06:18:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:30.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:30 smithi150 ceph-mon[194178]: pgmap v539: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T06:18:30.413 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:30.472 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:30.833 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 47862 47846 119.561 72.75 0.00761343 0.00835903 2023-10-09T06:18:31.310 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:31.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:31.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:31.494 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:31.651 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:18:31.651 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93002 8.3 8:c23c1f88:::benchmark_data_smithi107_251897_object93001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 310.79563457099999, 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0136282910000001, 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.652 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93002 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:18:31.653 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851821+0000", 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851849+0000", 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.72e-05 2023-10-09T06:18:31.654 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851861+0000", 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2972999999999999e-05 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851870+0000", 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8480000000000007e-06 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.655 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851939+0000", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9049999999999998e-05 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851944+0000", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9509999999999997e-06 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854442+0000", 2023-10-09T06:18:31.656 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024976659999999999 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854451+0000", 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865368+0000", 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0109166650000001 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.657 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865451+0000", 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3094000000000001e-05 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93007 8.3 8:c788829f:::benchmark_data_smithi107_251897_object93006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:18:31.658 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 310.79337000599998, 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.025725453, 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93007 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.659 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854086+0000", 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.660 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854112+0000", 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5928e-05 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854125+0000", 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3168e-05 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.661 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854135+0000", 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0084999999999999e-05 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854657+0000", 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000521621 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.662 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854666+0000", 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8510000000000005e-06 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865819+0000", 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011153274 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865902+0000", 2023-10-09T06:18:31.663 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3090999999999994e-05 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.866012+0000", 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110571 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.875194+0000", 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091818119999999993 2023-10-09T06:18:31.664 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.875229+0000", 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4848999999999998e-05 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879743+0000", 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045137509999999999 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.665 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879777+0000", 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4147999999999999e-05 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879813+0000", 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5893999999999999e-05 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.666 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93019 8.3 8:c4546894:::benchmark_data_smithi107_251897_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 310.78304017300002, 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.017569357, 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.667 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93019 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864416+0000", 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864440+0000", 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3807000000000001e-05 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.669 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864450+0000", 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.045e-05 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864458+0000", 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9719999999999992e-06 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.670 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864513+0000", 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4622e-05 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864523+0000", 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0564000000000001e-05 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.867903+0000", 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0033802949999999 2023-10-09T06:18:31.671 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.868043+0000", 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013930299999999999 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.868202+0000", 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015874899999999999 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.672 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879826+0000", 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011624834000000001 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881863+0000", 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00203666 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.673 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881902+0000", 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9048000000000003e-05 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881945+0000", 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3384999999999999e-05 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.674 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881987+0000", 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1214999999999998e-05 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160720 8.7 8:eb35e384:::benchmark_data_smithi107_251897_object43806:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.49147870100001, 2023-10-09T06:18:31.675 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0247990149999999, 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160720 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.676 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155977+0000", 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155985+0000", 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4079999999999998e-06 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156014+0000", 2023-10-09T06:18:31.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8671e-05 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156024+0000", 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5349999999999993e-06 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169755+0000", 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013731729999999999 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169767+0000", 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1823e-05 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.180692+0000", 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0109248500000001 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.680 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.180778+0000", 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5822000000000006e-05 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160723 8.7 8:e87a05d0:::benchmark_data_smithi107_251897_object43809:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.49142262800001, 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0470548319999999, 2023-10-09T06:18:31.681 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160723 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:18:31.682 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156034+0000", 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.683 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156038+0000", 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1879999999999998e-06 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156044+0000", 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0290000000000002e-06 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.684 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156049+0000", 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9200000000000003e-06 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169787+0000", 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01373889 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.685 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169794+0000", 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9269999999999999e-06 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181389+0000", 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0115949829999999 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:18:31.686 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181521+0000", 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000131736 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.202970+0000", 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021449275 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203090+0000", 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119338 2023-10-09T06:18:31.687 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160724 8.7 8:e23977a0:::benchmark_data_smithi107_251897_object43810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.491405173, 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.047384474, 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.688 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160724 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.689 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.690 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156054+0000", 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2819999999999999e-06 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156060+0000", 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2480000000000004e-06 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.691 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156064+0000", 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3490000000000002e-06 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169814+0000", 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013749740999999999 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169820+0000", 2023-10-09T06:18:31.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5390000000000004e-06 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181584+0000", 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0117643000000001 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181682+0000", 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7349999999999995e-05 2023-10-09T06:18:31.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203344+0000", 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021662364 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203437+0000", 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2603999999999996e-05 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.694 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160729 8.7 8:e8d3cb53:::benchmark_data_smithi107_251897_object43815:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.491390088, 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0430160129999999, 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.695 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160729 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:18:31.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156069+0000", 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9250000000000002e-06 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156074+0000", 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0320000000000003e-06 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156078+0000", 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3610000000000003e-06 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169840+0000", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013761051999999999 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169846+0000", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8519999999999998e-06 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181733+0000", 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0118862280000001 2023-10-09T06:18:31.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181862+0000", 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000129691 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.182446+0000", 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058357799999999996 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.189884+0000", 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074385659999999998 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.195994+0000", 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0061091219999999998 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.701 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.198929+0000", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002935797 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.198987+0000", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8125000000000003e-05 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.199083+0000", 2023-10-09T06:18:31.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5947999999999997e-05 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160737 8.7 8:e2f37bf9:::benchmark_data_smithi107_251897_object43823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.491375635, 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0547179149999999, 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.703 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160737 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156084+0000", 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9369999999999998e-06 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156089+0000", 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4310000000000002e-06 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156093+0000", 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3499999999999999e-06 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169866+0000", 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013772863999999999 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169873+0000", 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6189999999999996e-06 2023-10-09T06:18:31.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196022+0000", 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.02614893 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196120+0000", 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7827999999999999e-05 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196205+0000", 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5265000000000004e-05 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203135+0000", 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006929724 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203302+0000", 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016770899999999999 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210674+0000", 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073718250000000003 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210726+0000", 2023-10-09T06:18:31.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1895999999999999e-05 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210800+0000", 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3824000000000003e-05 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.711 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162305 8.a 8:5d4cc9a4:::benchmark_data_smithi107_251897_object45391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.48714311099999, 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0500951030000001, 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162305 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160316+0000", 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6110000000000001e-06 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160322+0000", 2023-10-09T06:18:31.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6289999999999998e-06 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160327+0000", 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1930000000000001e-06 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160352+0000", 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4896e-05 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160550+0000", 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000197732 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160658+0000", 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108039 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.166029+0000", 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053705699999999999 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.176139+0000", 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010110887000000001 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.191860+0000", 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015721044 2023-10-09T06:18:31.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.191870+0000", 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8439999999999993e-06 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.210328+0000", 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0184577509999999 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.210409+0000", 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1397999999999996e-05 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162309 8.a 8:5870fc90:::benchmark_data_smithi107_251897_object45395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:18:31.719 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.47805069500001, 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.041964809, 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162309 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.720 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169405+0000", 2023-10-09T06:18:31.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169412+0000", 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2180000000000002e-06 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169433+0000", 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0420000000000001e-05 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169442+0000", 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4970000000000007e-06 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171660+0000", 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022180530000000002 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171801+0000", 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014037300000000001 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171929+0000", 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000127914 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.724 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.177701+0000", 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057724229999999996 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189331+0000", 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011629525999999999 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192876+0000", 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035456989999999998 2023-10-09T06:18:31.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192888+0000", 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1413000000000001e-05 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.211303+0000", 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.018415525 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.211372+0000", 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8337000000000007e-05 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162315 8.1 8:89f47883:::benchmark_data_smithi107_251897_object45401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.46119370700001, 2023-10-09T06:18:31.727 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0542757009999999, 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162315 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186262+0000", 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:31.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186268+0000", 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6119999999999998e-06 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186277+0000", 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0119999999999994e-06 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186283+0000", 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186447+0000", 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000164086 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.731 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186571+0000", 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000123607 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186690+0000", 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119653 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193070+0000", 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0063797419999999999 2023-10-09T06:18:31.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193078+0000", 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.222587+0000", 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0295085129999999 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.222700+0000", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0001127 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.240453+0000", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017753890000000001 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.240539+0000", 2023-10-09T06:18:31.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6069000000000006e-05 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162320 8.1 8:8b316482:::benchmark_data_smithi107_251897_object45406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.45523782500001, 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0519342679999999, 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.735 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162320 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192218+0000", 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192228+0000", 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.91e-06 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192247+0000", 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8972999999999999e-05 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192257+0000", 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0061000000000001e-05 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193540+0000", 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012824869999999999 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.740 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193547+0000", 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8329999999999994e-06 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223305+0000", 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0297573090000001 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223409+0000", 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010399300000000001 2023-10-09T06:18:31.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223520+0000", 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000111723 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.230986+0000", 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074656439999999996 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244030+0000", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013043602 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244077+0000", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6981000000000003e-05 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244154+0000", 2023-10-09T06:18:31.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7329000000000005e-05 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162329 8.1 8:835e27a5:::benchmark_data_smithi107_251897_object45415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.44194100999999, 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.038815201, 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.744 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162329 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205515+0000", 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205525+0000", 2023-10-09T06:18:31.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.059e-05 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205545+0000", 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9582e-05 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205556+0000", 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1056999999999999e-05 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205622+0000", 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5757000000000001e-05 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205636+0000", 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.435e-05 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227102+0000", 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0214658539999999 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227217+0000", 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115251 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227340+0000", 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012321899999999999 2023-10-09T06:18:31.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.235355+0000", 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0080145619999999994 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244200+0000", 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088452549999999998 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244254+0000", 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3495000000000002e-05 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244332+0000", 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8145000000000005e-05 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162333 8.1 8:88e0a74d:::benchmark_data_smithi107_251897_object45419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 220.429333571, 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0263455720000001, 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162333 2023-10-09T06:18:31.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218122+0000", 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218130+0000", 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0299999999999994e-06 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218145+0000", 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4691999999999999e-05 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218153+0000", 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0530000000000007e-06 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218193+0000", 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9475999999999999e-05 2023-10-09T06:18:31.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218203+0000", 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0229e-05 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231014+0000", 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0128109240000001 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231130+0000", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116248 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231267+0000", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013658999999999999 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.235360+0000", 2023-10-09T06:18:31.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00409377 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244375+0000", 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090142599999999996 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244414+0000", 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9273999999999997e-05 2023-10-09T06:18:31.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244469+0000", 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5528000000000001e-05 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176889 8.1 8:842966fd:::benchmark_data_smithi107_251897_object59975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:18:31.759 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 199.95582166099999, 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1818074730000001, 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176889 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691634+0000", 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691640+0000", 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8429999999999998e-06 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691652+0000", 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1778e-05 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691662+0000", 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691810+0000", 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014854000000000001 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691824+0000", 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3281e-05 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862557+0000", 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.170733891 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862570+0000", 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2087e-05 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859255+0000", 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9966857579999999 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859403+0000", 2023-10-09T06:18:31.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014815999999999999 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859545+0000", 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014124499999999999 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870614+0000", 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011069522 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870629+0000", 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4932999999999999e-05 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873223+0000", 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025939309999999998 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873276+0000", 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2948999999999997e-05 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873443+0000", 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016722699999999999 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176917 8.1 8:891de888:::benchmark_data_smithi107_251897_object60003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 199.925717222, 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1591737929999999, 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176917 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.769 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721739+0000", 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721747+0000", 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.422e-06 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721761+0000", 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3781000000000001e-05 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721770+0000", 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9169999999999995e-06 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721847+0000", 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7078999999999998e-05 2023-10-09T06:18:31.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721860+0000", 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2833000000000001e-05 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862594+0000", 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.140733884 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862602+0000", 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2179999999999999e-06 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863642+0000", 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.001040164 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863765+0000", 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012266100000000001 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863882+0000", 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011691 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.775 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.872363+0000", 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.008481255 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.872371+0000", 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.926e-06 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880690+0000", 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083193750000000004 2023-10-09T06:18:31.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880744+0000", 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3591000000000002e-05 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880914+0000", 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000170321 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.777 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176924 8.a 8:5db5ab82:::benchmark_data_smithi107_251897_object60010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 199.91437838799999, 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.141292129, 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.778 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176924 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733077+0000", 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733084+0000", 2023-10-09T06:18:31.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0650000000000001e-06 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733099+0000", 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4124999999999999e-05 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733108+0000", 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.825e-06 2023-10-09T06:18:31.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733179+0000", 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0592000000000004e-05 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733191+0000", 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1746000000000001e-05 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862654+0000", 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.12946349500000001 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862664+0000", 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5619999999999993e-06 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863267+0000", 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0006028649999998 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863408+0000", 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000141013 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863529+0000", 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000121663 2023-10-09T06:18:31.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.871583+0000", 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0080541659999999998 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873555+0000", 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001971492 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874145+0000", 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058958700000000003 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874191+0000", 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6397999999999999e-05 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874371+0000", 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018031000000000001 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176925 8.a 8:5360f822:::benchmark_data_smithi107_251897_object60011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 199.91414019800001, 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1412183040000001, 2023-10-09T06:18:31.787 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176925 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.788 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733321+0000", 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.211e-06 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733331+0000", 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0514e-05 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733339+0000", 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.289e-06 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733616+0000", 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000277275 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733622+0000", 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1349999999999997e-06 2023-10-09T06:18:31.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.863714+0000", 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.130092441 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.863724+0000", 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1020000000000003e-06 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867691+0000", 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0039678250000001 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867831+0000", 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000140061 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:31.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867946+0000", 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114079 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.871810+0000", 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003864606 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873593+0000", 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001783228 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874416+0000", 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082254599999999995 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874452+0000", 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6037000000000001e-05 2023-10-09T06:18:31.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874536+0000", 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3639000000000003e-05 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:227141 8.a 8:5db9bc41:::benchmark_data_smithi107_251897_object110227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:18:31.796 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 128.29709646000001, 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0313837400000001, 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 227141 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350360+0000", 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350367+0000", 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.61e-06 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350383+0000", 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6019e-05 2023-10-09T06:18:31.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350392+0000", 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3300000000000005e-06 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350443+0000", 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0949000000000003e-05 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350457+0000", 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3511e-05 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372276+0000", 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0218192319999999 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372402+0000", 2023-10-09T06:18:31.802 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:31.802 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012583700000000001 2023-10-09T06:18:31.802 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372491+0000", 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8690999999999995e-05 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.375873+0000", 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003382005 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381596+0000", 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057236939999999997 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381676+0000", 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9623000000000007e-05 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:31.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381710+0000", 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4304999999999998e-05 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381745+0000", 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4433e-05 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.805 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231190 8.7 8:ecf39326:::benchmark_data_smithi107_251897_object114276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 122.07285063499999, 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.104789587, 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231190 2023-10-09T06:18:31.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:31.807 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574606+0000", 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574611+0000", 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4269999999999999e-06 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574620+0000", 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2689999999999999e-06 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574625+0000", 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1510000000000003e-06 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579614+0000", 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0049893089999999999 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579782+0000", 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016760700000000001 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579925+0000", 2023-10-09T06:18:31.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014329999999999999 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588552+0000", 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0086273949999999995 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.593852+0000", 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0052992849999999999 2023-10-09T06:18:31.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.615463+0000", 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021611755 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.615477+0000", 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3352e-05 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.812 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679328+0000", 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0638517119999999 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679396+0000", 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8059999999999996e-05 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.813 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:31.814 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:31.814 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:18:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:31.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:31.837 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 49332 49316 118.493 91.875 0.00493 0.00843468 2023-10-09T06:18:31.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:31.981 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:32.088 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:32.200 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:32.252 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:32.267 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:32 smithi107 ceph-mon[212750]: pgmap v540: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:18:32.267 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:32.268 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:32] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-10-09T06:18:32.268 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:32 smithi107 ceph-mon[220921]: pgmap v540: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:18:32.268 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:32.378 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:32.383 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:32.384 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:32 smithi150 ceph-mon[194178]: pgmap v540: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:18:32.384 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:32.486 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:32.537 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:32.668 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:32.775 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:32.835 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 50748 50732 117.381 88.5 0.0158615 0.00851493 2023-10-09T06:18:32.842 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:32.942 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:32.943 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:18:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:32.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:33.063 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:33.134 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:33.151 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:18:33.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:33.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:33.230 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:33.835 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 51972 51956 115.922 76.5 0.00945375 0.00862186 2023-10-09T06:18:34.033 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:34.136 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:34.210 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: pgmap v541: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:18:34.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:34.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:18:34.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[212750]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: pgmap v541: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:18:34.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:34.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:18:34.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:34 smithi107 ceph-mon[220921]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.361 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:34.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: pgmap v541: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:18:34.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:18:34.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3352638532' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:34 smithi150 ceph-mon[194178]: failed to encode map e158 with expected crc 2023-10-09T06:18:34.473 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:34.478 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:34.521 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.626 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:34.733 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:34.759 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.800 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:34.835 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 53130 53114 114.42 72.375 0.00340538 0.00870319 2023-10-09T06:18:34.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:34.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:34.972 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:34.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:35.077 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:35.133 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:35.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:35.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:35.341 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:35.414 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:35.528 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:35.637 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:35.711 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:35.836 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 54359 54343 113.163 76.8125 0.00851935 0.00883188 2023-10-09T06:18:36.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:36 smithi107 ceph-mon[212750]: pgmap v543: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:18:36.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:36.137 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:36.137 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:36.137 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T06:18:36.173 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:18:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:36 smithi107 ceph-mon[220921]: pgmap v543: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:18:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:36 smithi150 ceph-mon[194178]: pgmap v543: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T06:18:36.554 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:36.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:36.662 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:36.738 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:36.832 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:36.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:36.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 55538 55522 111.879 73.6875 0.00950881 0.00893192 2023-10-09T06:18:36.937 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:36.977 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:36.995 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:37.039 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:37.043 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:37.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:37.087 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:37.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:37.145 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:37.180 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:37.216 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:37.222 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:37.224 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:37.272 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:37.274 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:37.312 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:37.318 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.318 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.318 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.318 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.319 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.319 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.319 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.319 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:37.341 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:37.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:37.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:37.377 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:37.377 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:37.447 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:37.535 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:37.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 56723 56707 110.698 74.0625 0.00571264 0.00902847 2023-10-09T06:18:38.196 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:38.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:37 smithi107 ceph-mon[212750]: pgmap v544: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:18:38.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1263591666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:38.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:37 smithi107 ceph-mon[220921]: pgmap v544: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:18:38.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1263591666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:38.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:38.300 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:38.327 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:37 smithi150 ceph-mon[194178]: pgmap v544: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T06:18:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1263591666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:38.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 57754 57738 109.296 64.4375 0.0113506 0.00914428 2023-10-09T06:18:39.278 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:39.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:39.367 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:18:39.386 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:39.509 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:39.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 59135 59119 108.62 86.3125 0.0158678 0.00920194 2023-10-09T06:18:40.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:39 smithi107 ceph-mon[212750]: pgmap v545: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:18:40.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:39 smithi107 ceph-mon[220921]: pgmap v545: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:18:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:39 smithi150 ceph-mon[194178]: pgmap v545: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T06:18:40.450 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:40.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:40.555 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:40.607 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:40.839 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 60419 60403 107.81 80.25 0.0091003 0.00927027 2023-10-09T06:18:40.972 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:40.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:41.081 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:41.139 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.235 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:41.342 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:41.389 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:41.460 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:18:41.660 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:41.769 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:41.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:41.845 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 61566 61550 106.789 71.6875 0.0143617 0.00935801 2023-10-09T06:18:41.885 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:41.928 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.003 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:42.056 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:42] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-10-09T06:18:42.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:41 smithi107 ceph-mon[212750]: pgmap v546: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:18:42.057 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:41 smithi107 ceph-mon[220921]: pgmap v546: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:18:42.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.176 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.372 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:42 smithi107 conmon[212706]: 2023-10-09T06:18:42.171+0000 7f7c794f9700 -1 mon.a@0(leader).osd e158 definitely_dead 0 2023-10-09T06:18:42.378 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:41 smithi150 ceph-mon[194178]: pgmap v546: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:18:42.484 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:42.538 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.647 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:42.756 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:42.783 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:42.826 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:42.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:42.845 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 62647 62631 105.729 67.5625 0.018012 0.00941789 2023-10-09T06:18:42.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:42.980 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.7. 2023-10-09T06:18:42.985 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:18:42.995 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.045 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:43.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/625217794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[212750]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/625217794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:43.047 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:42 smithi107 ceph-mon[220921]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:43.149 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:43.175 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.216 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:43.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/625217794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:18:43.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T06:18:43.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:42 smithi150 ceph-mon[194178]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T06:18:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:43.322 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:43.350 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.774 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:43.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:43.845 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 62725 62709 103.077 4.875 0.00464766 0.00947883 2023-10-09T06:18:43.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:43.910 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:43.959 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:43.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:44.064 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.091 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.187 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:44.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:44.278 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:44.291 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:44.314 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: pgmap v547: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T06:18:44.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:43 smithi150 ceph-mon[194178]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.317 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: pgmap v547: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:18:44.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:44.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[212750]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.338 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: pgmap v547: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/106968822' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: osdmap e159: 8 total, 8 up, 7 in 2023-10-09T06:18:44.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:43 smithi107 ceph-mon[220921]: failed to encode map e159 with expected crc 2023-10-09T06:18:44.359 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:44.464 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:18:44.490 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.530 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:18:44.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:44.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:18:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:44.566 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:44.638 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:44.671 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:44.846 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 62725 62709 100.435 0 - 0.00947883 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.339 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[220921]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.340 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:44 smithi107 ceph-mon[212750]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.371 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: osdmap e160: 8 total, 8 up, 7 in 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:44 smithi150 ceph-mon[194178]: failed to encode map e160 with expected crc 2023-10-09T06:18:45.589 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:45.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:45.701 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:45.730 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:45.792 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:45.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:45.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:45.848 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:18:45.848741+0000 min lat: 0.00127393 max lat: 2.27436 avg lat: 0.0101318 2023-10-09T06:18:45.848 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:18:45.848 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 63182 63166 98.6338 14.2812 0.0402886 0.0101318 2023-10-09T06:18:45.898 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:46.008 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:46.198 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:18:46.267 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: pgmap v550: 89 pgs: 3 remapped+peering, 1 unknown, 16 peering, 69 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[212750]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: pgmap v550: 89 pgs: 3 remapped+peering, 1 unknown, 16 peering, 69 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:18:46.268 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:45 smithi107 ceph-mon[220921]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: pgmap v550: 89 pgs: 3 remapped+peering, 1 unknown, 16 peering, 69 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: osdmap e161: 8 total, 8 up, 7 in 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:45 smithi150 ceph-mon[194178]: failed to encode map e161 with expected crc 2023-10-09T06:18:46.411 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:46.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:46.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:46.530 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:46.610 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:46.625 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:46.848 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 63853 63837 97.251 41.9375 0.0108463 0.0102766 2023-10-09T06:18:46.858 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:46.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:46.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:46.965 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:46.995 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.048 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:18:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:18:47.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:47.156 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:47.228 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.249 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:47.292 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:47.330 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:47.371 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: osdmap e162: 8 total, 8 up, 7 in 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: failed to encode map e162 with expected crc 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:18:47.405 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:47.465 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:47.510 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:47.559 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:47.565 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.583 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:18:47.672 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:47.705 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.767 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:47.852 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 64544 64528 95.9642 43.1875 0.017783 0.0104146 2023-10-09T06:18:47.873 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:47.909 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:47.973 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:47.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:48.082 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:48.120 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: pgmap v553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 unknown, 16 peering, 68 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 3936/188610 objects degraded (2.087%) 2023-10-09T06:18:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:18:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3075889586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 3936/188610 objects degraded (2.087%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:18:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: pgmap v553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 unknown, 16 peering, 68 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 3936/188610 objects degraded (2.087%) 2023-10-09T06:18:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:18:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3075889586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:48 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 3936/188610 objects degraded (2.087%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: pgmap v553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 unknown, 16 peering, 68 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 3936/188610 objects degraded (2.087%) 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: osdmap e163: 8 total, 8 up, 7 in 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: failed to encode map e163 with expected crc 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3075889586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:48 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 3936/188610 objects degraded (2.087%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:18:48.851 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 65088 65072 94.5179 34 0.0177984 0.010573 2023-10-09T06:18:49.149 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:49.149 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:49.149 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:18:49.149 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:18:49.150 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:18:49.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:18:49.215 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:49.379 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:49.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:49.574 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:49.709 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:49.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:49.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:49.826 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:18:49.851 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 65761 65745 93.3263 42.0625 0.0233407 0.0107095 2023-10-09T06:18:49.944 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:50.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:50 smithi107 ceph-mon[212750]: pgmap v555: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 79 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 29 MiB/s wr, 465 op/s; 23629/192342 objects degraded (12.285%); 3828/192342 objects misplaced (1.990%); 1.6 MiB/s, 2 keys/s, 28 objects/s recovering 2023-10-09T06:18:50.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:50 smithi107 ceph-mon[220921]: pgmap v555: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 79 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 29 MiB/s wr, 465 op/s; 23629/192342 objects degraded (12.285%); 3828/192342 objects misplaced (1.990%); 1.6 MiB/s, 2 keys/s, 28 objects/s recovering 2023-10-09T06:18:50.135 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:50.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:50.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:50.247 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:50.274 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:18:50.274 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.34644.0:185641 8.e e132/129 8:7237c94b:::benchmark_data_smithi107_251897_object68727:head v 132'11529, mlcod=124'10319)", 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:25.889281+0000", 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 204.379312906, 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.031877001000000002, 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.275 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889281+0000", 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.276 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889281+0000", 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889281+0000", 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.277 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889289+0000", 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2600000000000005e-06 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889301+0000", 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1691999999999999e-05 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.278 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.281 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889308+0000", 2023-10-09T06:18:50.281 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6629999999999993e-06 2023-10-09T06:18:50.281 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.281 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.281 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889369+0000", 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0788999999999997e-05 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889389+0000", 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0378999999999999e-05 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.282 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.921119+0000", 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.031730056 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.921158+0000", 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9011999999999999e-05 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.34644.0:185653 8.e e132/129 8:786b3826:::benchmark_data_smithi107_251897_object68739:head v 132'11530, mlcod=124'10319)", 2023-10-09T06:18:50.283 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:25.889323+0000", 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 204.379271187, 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.031859734000000001, 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889323+0000", 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.284 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889323+0000", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889323+0000", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889327+0000", 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.262e-06 2023-10-09T06:18:50.285 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889332+0000", 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3900000000000003e-06 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.889335+0000", 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2590000000000001e-06 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.286 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.900382+0000", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011047563 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.900411+0000", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8911000000000002e-05 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.921164+0000", 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.020753107999999999 2023-10-09T06:18:50.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:25.921183+0000", 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8538000000000001e-05 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.34644.0:193194 8.d e134/129 8:b85abec2:::benchmark_data_smithi107_251897_object76280:head v 134'11804, mlcod=124'10248)", 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:18:50.288 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 194.134687372, 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.037746214, 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:18:50.289 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133906+0000", 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133913+0000", 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7150000000000001e-06 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.290 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133923+0000", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5629999999999998e-06 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133928+0000", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0980000000000001e-06 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.146011+0000", 2023-10-09T06:18:50.291 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012083534999999999 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.146038+0000", 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7044000000000001e-05 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.171622+0000", 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025583854999999999 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.171653+0000", 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1019999999999998e-05 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.34644.0:196829 8.1 e134/129 8:83100061:::benchmark_data_smithi107_251897_object79915:head v 134'12377, mlcod=122'10181)", 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:40.780734+0000", 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 189.48786048700001, 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.032020420000000001, 2023-10-09T06:18:50.293 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780734+0000", 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780734+0000", 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.294 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780733+0000", 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780741+0000", 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.295 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780752+0000", 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0309000000000001e-05 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.780757+0000", 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7389999999999996e-06 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.788219+0000", 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074612009999999998 2023-10-09T06:18:50.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.788233+0000", 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.449e-05 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.812681+0000", 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024448293999999999 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.297 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:40.812754+0000", 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2951e-05 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231195 8.f 8:f3e206e0:::benchmark_data_smithi107_251897_object114281:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.66685340000001, 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.136219622, 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.298 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231195 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.299 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601739+0000", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601747+0000", 2023-10-09T06:18:50.300 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4429999999999997e-06 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601758+0000", 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1917e-05 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601768+0000", 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2909999999999996e-06 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.301 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.607700+0000", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005931955 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.607710+0000", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0655e-05 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.737872+0000", 2023-10-09T06:18:50.302 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1301620589999999 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.737961+0000", 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8196999999999996e-05 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231214 8.f 8:fb3281cb:::benchmark_data_smithi107_251897_object114300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:18:50.303 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.62019409600001, 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1060980840000001, 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231214 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.304 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648398+0000", 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.305 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648413+0000", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5296999999999999e-05 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648453+0000", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0074000000000003e-05 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648462+0000", 2023-10-09T06:18:50.306 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3799999999999994e-06 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648506+0000", 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.456e-05 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648514+0000", 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3270000000000001e-06 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.307 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738194+0000", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.089679364 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738327+0000", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013300700000000001 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738457+0000", 2023-10-09T06:18:50.308 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013044099999999999 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.751429+0000", 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012972099000000001 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754396+0000", 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002966602 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.309 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754430+0000", 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4335000000000002e-05 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754498+0000", 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8104999999999999e-05 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.310 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231225 8.f 8:ff40690e:::benchmark_data_smithi107_251897_object114311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 139.589508881, 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.075537689000000005, 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231225 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.311 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.312 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679084+0000", 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679091+0000", 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6279999999999999e-06 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679104+0000", 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2893e-05 2023-10-09T06:18:50.313 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679111+0000", 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4660000000000002e-06 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679156+0000", 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5113e-05 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.314 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679164+0000", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8690000000000004e-06 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741160+0000", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.061996277000000002 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741294+0000", 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000133638 2023-10-09T06:18:50.315 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741424+0000", 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000130307 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.751436+0000", 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010011483999999999 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:50.316 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754532+0000", 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030958510000000002 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754564+0000", 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1983999999999997e-05 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754623+0000", 2023-10-09T06:18:50.317 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9682000000000002e-05 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231232 8.f 8:f343d541:::benchmark_data_smithi107_251897_object114318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 139.572335125, 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.058893938999999999, 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.318 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231232 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.319 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696258+0000", 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696267+0000", 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9530000000000005e-06 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696280+0000", 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3593e-05 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696289+0000", 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9630000000000004e-06 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744056+0000", 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.047766955 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744155+0000", 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8794000000000003e-05 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744260+0000", 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000104666 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754645+0000", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010385079 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755033+0000", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000388296 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755075+0000", 2023-10-09T06:18:50.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2126000000000003e-05 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755153+0000", 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8034000000000005e-05 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:233225 8.f 8:f009cc95:::benchmark_data_smithi107_251897_object116311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:18:50.324 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 137.307608109, 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24422334800000001, 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 233225 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.325 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960985+0000", 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:18:50.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960991+0000", 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0589999999999999e-06 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961002+0000", 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0822000000000001e-05 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.327 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961008+0000", 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7350000000000001e-06 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961052+0000", 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4536999999999997e-05 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961155+0000", 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000102366 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961237+0000", 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2424999999999998e-05 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.963539+0000", 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023016949999999999 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:18:50.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205068+0000", 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24152984399999999 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205136+0000", 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7566999999999999e-05 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205210+0000", 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3555999999999998e-05 2023-10-09T06:18:50.330 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11016 9.6 e156/152 9:6568d624:::benchmark_data_smithi107_300016_object11015:head v 156'667, mlcod=156'665)", 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 40.909604547000001, 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.155686719, 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358989+0000", 2023-10-09T06:18:50.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359030+0000", 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1652e-05 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359047+0000", 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6206e-05 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359056+0000", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8460000000000003e-06 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359118+0000", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1297000000000006e-05 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359139+0000", 2023-10-09T06:18:50.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1719e-05 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359945+0000", 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00080596499999999996 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514677+0000", 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.154731128 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.335 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11011 9.0 e156/152 9:0e4dd42b:::benchmark_data_smithi107_300016_object11010:head v 156'682, mlcod=156'680)", 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 40.909505711000001, 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15559872299999999, 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:18:50.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359088+0000", 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359109+0000", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0608999999999999e-05 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359117+0000", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.858e-06 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359123+0000", 2023-10-09T06:18:50.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9139999999999996e-06 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359193+0000", 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0149999999999998e-05 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359216+0000", 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3261e-05 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514658+0000", 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.155441945 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514687+0000", 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9456999999999999e-05 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.340 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11024 9.6 e156/152 9:66e82b7d:::benchmark_data_smithi107_300016_object11023:head v 156'668, mlcod=156'665)", 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 40.9075779, 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15538681900000001, 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361085+0000", 2023-10-09T06:18:50.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9665999999999993e-05 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514654+0000", 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15356840499999999 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514665+0000", 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0968000000000001e-05 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514713+0000", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8148e-05 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514735+0000", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2272e-05 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516372+0000", 2023-10-09T06:18:50.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016370790000000001 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516403+0000", 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1127000000000002e-05 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11064 9.2 e156/152 9:46ba2048:::benchmark_data_smithi107_300016_object11063:head v 156'714, mlcod=156'712)", 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:18:50.345 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 40.894618932, 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.142028131, 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.346 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373974+0000", 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.374000+0000", 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5729999999999999e-05 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514671+0000", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.140670884 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514679+0000", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9270000000000005e-06 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514715+0000", 2023-10-09T06:18:50.348 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6514000000000003e-05 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514732+0000", 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6410999999999999e-05 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.515964+0000", 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012326259999999999 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.349 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516004+0000", 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9150000000000003e-05 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:14197 9.2 e156/152 9:409344a9:::benchmark_data_smithi107_300016_object14196:head v 156'904, mlcod=156'902)", 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 39.798691513000001, 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060887881999999997, 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.350 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.351 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469902+0000", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469926+0000", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4340000000000001e-05 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469940+0000", 2023-10-09T06:18:50.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4282e-05 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529007+0000", 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.059066891000000003 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529076+0000", 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8422000000000003e-05 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.353 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529092+0000", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6433e-05 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.530758+0000", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016657379999999999 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.530791+0000", 2023-10-09T06:18:50.354 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2855999999999998e-05 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:14199 9.6 e156/152 9:6698baa0:::benchmark_data_smithi107_300016_object14198:head v 156'865, mlcod=156'862)", 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 39.798196625999999, 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060791981000000002, 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.355 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470397+0000", 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470455+0000", 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7723999999999998e-05 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529020+0000", 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.058564931000000001 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.357 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529049+0000", 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9266999999999999e-05 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529102+0000", 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3115000000000002e-05 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.358 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529126+0000", 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3938e-05 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.531150+0000", 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020239699999999999 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.531190+0000", 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9928000000000001e-05 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.359 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:22877 9.0 e156/152 9:0619543b:::benchmark_data_smithi107_300016_object22876:head v 156'1402, mlcod=156'1400)", 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 36.033593265, 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.054981192999999998, 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.360 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235000+0000", 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.361 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235028+0000", 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7268999999999999e-05 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235040+0000", 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2818e-05 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.362 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235049+0000", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6899999999999998e-06 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235098+0000", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9070999999999998e-05 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235116+0000", 2023-10-09T06:18:50.363 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8114e-05 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.236050+0000", 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00093338299999999998 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289982+0000", 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.053932718999999997 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.364 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:22889 9.6 e156/152 9:62aac67c:::benchmark_data_smithi107_300016_object22888:head v 156'1423, mlcod=156'1421)", 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 36.028188342, 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.052153469000000001, 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.365 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240405+0000", 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.366 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240461+0000", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5998000000000002e-05 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289945+0000", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.049483965999999997 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289958+0000", 2023-10-09T06:18:50.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3225e-05 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.290008+0000", 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9505000000000001e-05 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.290031+0000", 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3249000000000001e-05 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.368 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.292526+0000", 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024948890000000001 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.292560+0000", 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3581000000000003e-05 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.369 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:24238 9.2 e156/152 9:4195be16:::benchmark_data_smithi107_300016_object24237:head v 156'1526, mlcod=156'1524)", 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 35.105633275999999, 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.040898361000000001, 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.370 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162960+0000", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162988+0000", 2023-10-09T06:18:50.371 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8102999999999998e-05 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163003+0000", 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4438e-05 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163010+0000", 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3239999999999997e-06 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163074+0000", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3561999999999999e-05 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163093+0000", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9729999999999999e-05 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.203828+0000", 2023-10-09T06:18:50.373 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.040734609999999997 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.203859+0000", 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1599e-05 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:24237 9.6 e156/152 9:69328eaf:::benchmark_data_smithi107_300016_object24236:head v 156'1516, mlcod=156'1513)", 2023-10-09T06:18:50.374 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 35.105630945999998, 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.041692811000000003, 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.375 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162958+0000", 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999948 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.376 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163188+0000", 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000229597 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163199+0000", 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0966e-05 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163206+0000", 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8000000000000001e-06 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.201974+0000", 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.038768449000000003 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.201997+0000", 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2906999999999999e-05 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.204634+0000", 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026364259999999999 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.204656+0000", 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2719000000000001e-05 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64349 9.7 e163/160 9:ea5eaa42:::benchmark_data_smithi107_300016_object64348:head v 163'4091, mlcod=0'0)", 2023-10-09T06:18:50.379 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:47.613629+0000", 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 2.6549651409999999, 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.032509471999999998, 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613629+0000", 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613629+0000", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613628+0000", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613681+0000", 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2883999999999997e-05 2023-10-09T06:18:50.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613693+0000", 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1627000000000001e-05 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.613702+0000", 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6540000000000005e-06 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:18:50.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.621575+0000", 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007873718 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.621599+0000", 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4057999999999999e-05 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.646104+0000", 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024504416000000001 2023-10-09T06:18:50.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:47.646139+0000", 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4975000000000003e-05 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:18:50.384 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:18:50.385 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:50 smithi150 ceph-mon[194178]: pgmap v555: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 79 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 29 MiB/s wr, 465 op/s; 23629/192342 objects degraded (12.285%); 3828/192342 objects misplaced (1.990%); 1.6 MiB/s, 2 keys/s, 28 objects/s recovering 2023-10-09T06:18:50.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:50.407 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:50.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:50.511 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:50.603 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:50.851 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 66492 66476 92.2682 45.6875 0.0160311 0.0108321 2023-10-09T06:18:51.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:51 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:51.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:51 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:51 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:18:51.517 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:18:51.851 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 67117 67101 91.1121 39.0625 0.00665185 0.0109656 2023-10-09T06:18:52.204 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:52.309 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:52.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:52 smithi107 ceph-mon[212750]: pgmap v556: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 31376/194013 objects degraded (16.172%); 3828/194013 objects misplaced (1.973%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-10-09T06:18:52.313 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:18:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:18:52] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2023-10-09T06:18:52.313 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:52 smithi107 ceph-mon[220921]: pgmap v556: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 31376/194013 objects degraded (16.172%); 3828/194013 objects misplaced (1.973%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-10-09T06:18:52.314 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:18:52.340 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:52 smithi150 ceph-mon[194178]: pgmap v556: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 31376/194013 objects degraded (16.172%); 3828/194013 objects misplaced (1.973%); 1.4 MiB/s, 1 keys/s, 23 objects/s recovering 2023-10-09T06:18:52.343 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.399 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:52.502 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:52.532 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.657 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:52.761 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:52.807 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:52.853 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 67771 67755 90.0401 40.875 0.0228993 0.0110996 2023-10-09T06:18:52.874 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:52.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:52.909 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:52.978 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:53.010 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.070 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:53.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:53 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 31376/196524 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:53.174 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.218 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.287 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:53.290 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:53 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 31376/196524 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:53.360 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:18:53.360 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:18:53.361 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:18:53.392 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:53 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 31376/196524 objects degraded (15.965%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:53.479 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.586 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:18:53.591 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:53.705 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:53.740 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:53.796 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:53.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:53.853 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 68407 68391 88.9929 39.75 0.00281244 0.0112244 2023-10-09T06:18:53.904 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:53.939 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:54.002 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:54.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:54.108 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:54.140 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[220921]: pgmap v557: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 31376/196524 objects degraded (15.965%); 3828/196524 objects misplaced (1.948%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-09T06:18:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2858920036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.213 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:18:54.269 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:54.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:54 smithi150 ceph-mon[194178]: pgmap v557: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 31376/196524 objects degraded (15.965%); 3828/196524 objects misplaced (1.948%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-09T06:18:54.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2858920036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:54.377 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:54.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[212750]: pgmap v557: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 31376/196524 objects degraded (15.965%); 3828/196524 objects misplaced (1.948%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-09T06:18:54.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2858920036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:18:54.406 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:54.856 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 69071 69055 88.0191 41.5 0.0155964 0.0113555 2023-10-09T06:18:54.931 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:54.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:55.039 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:18:55.071 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.129 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:18:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:18:55.235 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:18:55.269 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.330 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:18:55.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:55 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:55.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:55.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:55 smithi150 ceph-mon[194178]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:18:55.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:55 smithi150 ceph-mon[194178]: failed to encode map e164 with expected crc 2023-10-09T06:18:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:18:55.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[220921]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[220921]: failed to encode map e164 with expected crc 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[212750]: osdmap e164: 8 total, 8 up, 7 in 2023-10-09T06:18:55.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:55 smithi107 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T06:18:55.438 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:18:55.472 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.516 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:18:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:18:55.622 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:18:55.689 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:55.857 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 69756 69740 87.1142 42.8125 0.00977489 0.0114735 2023-10-09T06:18:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:18:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:55.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:18:56.076 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:56.180 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:56.264 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:56 smithi107 ceph-mon[212750]: failed to encode map e164 with expected crc 2023-10-09T06:18:56.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:56 smithi107 ceph-mon[212750]: pgmap v559: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 31376/203910 objects degraded (15.387%); 3619/203910 objects misplaced (1.775%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T06:18:56.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:56 smithi107 ceph-mon[220921]: failed to encode map e164 with expected crc 2023-10-09T06:18:56.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:56 smithi107 ceph-mon[220921]: pgmap v559: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 31376/203910 objects degraded (15.387%); 3619/203910 objects misplaced (1.775%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T06:18:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:56 smithi150 ceph-mon[194178]: failed to encode map e164 with expected crc 2023-10-09T06:18:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:56 smithi150 ceph-mon[194178]: pgmap v559: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 31376/203910 objects degraded (15.387%); 3619/203910 objects misplaced (1.775%); 2.7 MiB/s, 1 keys/s, 43 objects/s recovering 2023-10-09T06:18:56.857 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 70402 70386 86.1982 40.375 0.0161757 0.0115869 2023-10-09T06:18:57.057 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:18:57.057 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:18:57.057 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:18:57.057 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:18:57.063 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:18:57.114 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:18:57.155 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:18:57.193 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.195 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.196 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.196 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:18:57.255 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:57.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:57.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:57.362 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:18:57.447 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:57.567 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:18:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:57.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:18:57.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:18:57.742 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:57.832 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:57.832 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2262359137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:57.832 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[212750]: pgmap v560: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 31376/205698 objects degraded (15.253%); 3619/205698 objects misplaced (1.759%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T06:18:57.857 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 71026 71010 85.291 39 0.0206268 0.0117166 2023-10-09T06:18:58.105 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:18:59.397 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:18:59.397 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:18:59.398 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T06:18:59.398 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T06:18:59.399 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:59.399 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2262359137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:59.399 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:57 smithi150 ceph-mon[194178]: pgmap v560: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 31376/205698 objects degraded (15.253%); 3619/205698 objects misplaced (1.759%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T06:18:59.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:18:59.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2262359137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:18:59.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:57 smithi107 ceph-mon[220921]: pgmap v560: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 31376/205698 objects degraded (15.253%); 3619/205698 objects misplaced (1.759%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2023-10-09T06:18:59.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:59.401 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:18:59.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:18:59.401 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:18:59.401 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:18:59.402 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 71781 71765 84.5724 47.1875 0.0198197 0.0118188 2023-10-09T06:18:59.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:18:59 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 31376/211878 objects degraded (14.809%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:59.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:18:59 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 31376/211878 objects degraded (14.809%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:59.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:18:59 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 31376/211878 objects degraded (14.809%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:18:59.502 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:18:59.675 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:18:59.781 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:18:59.860 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 72297 72281 83.6002 32.25 0.0213563 0.0119555 2023-10-09T06:18:59.871 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:18:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:18:59.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:18:59.985 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:00.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:00 smithi107 ceph-mon[212750]: pgmap v561: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 31376/211878 objects degraded (14.809%); 3383/211878 objects misplaced (1.597%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:00.144 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:00.340 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:00.393 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:00.398 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:00 smithi107 ceph-mon[220921]: pgmap v561: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 31376/211878 objects degraded (14.809%); 3383/211878 objects misplaced (1.597%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:00.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:00.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:00 smithi150 ceph-mon[194178]: pgmap v561: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 31376/211878 objects degraded (14.809%); 3383/211878 objects misplaced (1.597%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:00.500 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:00.535 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:00.595 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:00.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:00.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:00.705 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:00.741 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:00.800 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:00.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:00.860 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 72937 72921 82.8079 40 0.0299924 0.0120699 2023-10-09T06:19:00.911 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:00.942 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.026 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:01.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:01.133 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.163 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.215 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:01.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:01.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:01.325 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:01.358 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.400 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:01.416 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:01.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:01.523 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:01.688 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:01.860 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 73609 73593 82.0794 42 0.0276125 0.0121788 2023-10-09T06:19:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:02] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-10-09T06:19:02.860 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 74298 74282 81.3952 43.0625 0.0247742 0.0122799 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[212750]: pgmap v562: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 657 op/s; 31376/213762 objects degraded (14.678%); 3383/213762 objects misplaced (1.583%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[220921]: pgmap v562: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 657 op/s; 31376/213762 objects degraded (14.678%); 3383/213762 objects misplaced (1.583%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:02 smithi150 ceph-mon[194178]: pgmap v562: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 657 op/s; 31376/213762 objects degraded (14.678%); 3383/213762 objects misplaced (1.583%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:03.344 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:03.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:03.464 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:03.552 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:03.646 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:03.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:03.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:03.768 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:03.799 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:03.860 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:03.863 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 74946 74930 80.6873 40.5 0.0298452 0.0123869 2023-10-09T06:19:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:03.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:03.971 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:04.011 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:04.078 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:04.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:04.186 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:04.295 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:04.631 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:19:04.864 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 75526 75510 79.9322 36.25 0.0139618 0.0125041 2023-10-09T06:19:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:04 smithi150 ceph-mon[194178]: pgmap v563: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 31376/216087 objects degraded (14.520%); 3383/216087 objects misplaced (1.566%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4028852044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:04.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[212750]: pgmap v563: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 31376/216087 objects degraded (14.520%); 3383/216087 objects misplaced (1.566%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:04.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4028852044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:04.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:04.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[220921]: pgmap v563: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 31376/216087 objects degraded (14.520%); 3383/216087 objects misplaced (1.566%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:19:04.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4028852044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:04.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:05.122 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:05.396 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:05.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:05.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:05.480 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 conmon[212706]: 2023-10-09T06:19:05.208+0000 7f7c794f9700 -1 mon.a@0(leader).osd e165 definitely_dead 0 2023-10-09T06:19:05.501 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:05.538 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:05.596 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:05.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:05.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:05.676 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.7. 2023-10-09T06:19:05.706 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:05.738 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:05.780 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:05.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:05.865 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:19:05.865498+0000 min lat: 0.00127393 max lat: 2.27436 avg lat: 0.0125842 2023-10-09T06:19:05.865 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:19:05.865 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 76066 76050 79.1629 33.75 0.0101478 0.0125842 2023-10-09T06:19:05.885 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:05.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:05.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[212750]: failed to encode map e165 with expected crc 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[212750]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[220921]: failed to encode map e165 with expected crc 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[220921]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:19:05.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:19:05.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:05 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:05 smithi150 ceph-mon[194178]: failed to encode map e165 with expected crc 2023-10-09T06:19:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:05 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:05 smithi150 ceph-mon[194178]: osdmap e165: 8 total, 8 up, 7 in 2023-10-09T06:19:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T06:19:05.914 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:06.496 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:06.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:06.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:06.627 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:06.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[212750]: pgmap v565: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 31376/223341 objects degraded (14.048%); 3141/223341 objects misplaced (1.406%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:19:06.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 31376/223341 objects degraded (14.048%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:06.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[212750]: failed to encode map e166 with expected crc 2023-10-09T06:19:06.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:06.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[212750]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T06:19:06.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[220921]: pgmap v565: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 31376/223341 objects degraded (14.048%); 3141/223341 objects misplaced (1.406%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:19:06.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 31376/223341 objects degraded (14.048%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:06.777 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[220921]: failed to encode map e166 with expected crc 2023-10-09T06:19:06.778 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:06.778 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:06 smithi107 ceph-mon[220921]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T06:19:06.873 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 76159 76143 77.9506 5.8125 0.0611584 0.0128224 2023-10-09T06:19:06.904 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T06:19:06.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:06 smithi150 ceph-mon[194178]: pgmap v565: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 40 MiB/s wr, 647 op/s; 31376/223341 objects degraded (14.048%); 3141/223341 objects misplaced (1.406%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:19:06.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 31376/223341 objects degraded (14.048%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:06.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:06 smithi150 ceph-mon[194178]: failed to encode map e166 with expected crc 2023-10-09T06:19:06.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1370599182' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T06:19:06.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:06 smithi150 ceph-mon[194178]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T06:19:06.984 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:06.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:07.092 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:07.173 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:07.254 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:07.254 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:07.259 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:07.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:07.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:07.443 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:07.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.875 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.876 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.876 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[212750]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.876 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.876 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.876 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:07 smithi107 ceph-mon[220921]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.878 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 77181 77165 77.7237 63.875 0.0367172 0.0128602 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:07.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:07 smithi150 ceph-mon[194178]: failed to encode map e167 with expected crc 2023-10-09T06:19:08.597 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:08.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:08.703 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:08.761 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:08.773 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:19:08.840 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:19:08.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: pgmap v568: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 629 op/s; 27440/225093 objects degraded (12.191%); 3141/225093 objects misplaced (1.395%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:19:08.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T06:19:08.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[212750]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: pgmap v568: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 629 op/s; 27440/225093 objects degraded (12.191%); 3141/225093 objects misplaced (1.395%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.865 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:08 smithi107 ceph-mon[220921]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.871 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:08.876 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 78145 78129 77.4425 60.25 0.0283864 0.0129064 2023-10-09T06:19:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:08.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: pgmap v568: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 39 MiB/s wr, 629 op/s; 27440/225093 objects degraded (12.191%); 3141/225093 objects misplaced (1.395%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:08 smithi150 ceph-mon[194178]: failed to encode map e168 with expected crc 2023-10-09T06:19:08.984 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:09.124 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.231 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:09.339 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.398 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.473 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:09.512 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:09.560 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:09.568 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:09.579 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:09.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:09.637 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:09.658 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:09.663 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:09.728 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:09.768 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:09.800 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:09.821 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:09.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:09.836 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:09.841 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.841 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:09.876 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 78994 78978 77.0618 53.0625 0.00983804 0.0129716 2023-10-09T06:19:09.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2637069439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2637069439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:09.927 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:09.972 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2637069439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:09.975 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:10.070 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:10.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:10.187 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:10.253 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:10.876 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 79992 79976 76.836 62.375 0.0120919 0.0130101 2023-10-09T06:19:10.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:10 smithi107 ceph-mon[220921]: pgmap v570: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 11841/230619 objects degraded (5.134%); 704/230619 objects misplaced (0.305%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:19:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:10 smithi107 ceph-mon[212750]: pgmap v570: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 11841/230619 objects degraded (5.134%); 704/230619 objects misplaced (0.305%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:19:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:10 smithi150 ceph-mon[194178]: pgmap v570: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 11841/230619 objects degraded (5.134%); 704/230619 objects misplaced (0.305%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T06:19:11.163 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:11.266 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:11.319 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.411 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:11.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:11.516 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:11.582 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.678 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:11.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:11.792 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:11.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:11 smithi150 ceph-mon[194178]: pgmap v571: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 77 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5799/232506 objects degraded (2.494%); 704/232506 objects misplaced (0.303%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:19:11.841 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:11 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5799/232506 objects degraded (2.494%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:11.844 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:11.876 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 80948 80932 76.5772 59.75 0.0115637 0.0130532 2023-10-09T06:19:11.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:11 smithi107 ceph-mon[212750]: pgmap v571: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 77 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5799/232506 objects degraded (2.494%); 704/232506 objects misplaced (0.303%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:19:11.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:11 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5799/232506 objects degraded (2.494%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:11.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:11 smithi107 ceph-mon[220921]: pgmap v571: 89 pgs: 1 remapped+peering, 1 peering, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 77 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5799/232506 objects degraded (2.494%); 704/232506 objects misplaced (0.303%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:19:11.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:11 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5799/232506 objects degraded (2.494%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:11.905 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:19:11.906 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:19:11.906 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:19:11.906 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:19:11.906 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:19:11.906 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:19:11.936 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:11.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.060 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:12.143 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.282 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:12.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:12.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:12] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-10-09T06:19:12.460 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.596 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:12.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:12.705 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:12.707 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:12.779 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:12.877 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 81892 81876 76.3149 59 0.0202995 0.0130992 2023-10-09T06:19:13.043 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:13.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:13.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:13.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:13.100 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:13.169 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:13.240 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:13.335 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:13.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:13.377 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:19:13.377 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:19:13.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:19:13.377 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:23996 9.8 9:1111029f:::benchmark_data_smithi107_300016_object23995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.860318+0000", 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 58.515094126000001, 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19312328200000001, 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 23996 2023-10-09T06:19:13.378 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860318+0000", 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860318+0000", 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860317+0000", 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860350+0000", 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2564999999999997e-05 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860362+0000", 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1559999999999999e-05 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860369+0000", 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3270000000000003e-06 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.381 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860516+0000", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000146675 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860630+0000", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011473599999999999 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.860773+0000", 2023-10-09T06:19:13.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014275900000000001 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.862005+0000", 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012319589999999999 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.862775+0000", 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00076966199999999995 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.053320+0000", 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.190545559 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.053394+0000", 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3796999999999998e-05 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.053442+0000", 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7700000000000001e-05 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:24017 9.8 9:16da97b6:::benchmark_data_smithi107_300016_object24016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.868265+0000", 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 58.507147056999997, 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.185758859, 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.385 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 24017 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868265+0000", 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.386 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868265+0000", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868265+0000", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868286+0000", 2023-10-09T06:19:13.387 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1240999999999999e-05 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868293+0000", 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8630000000000002e-06 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868299+0000", 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6590000000000004e-06 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.388 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868345+0000", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5812000000000001e-05 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868461+0000", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115962 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.868600+0000", 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013970399999999999 2023-10-09T06:19:13.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.870881+0000", 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022812330000000001 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.872088+0000", 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001206309 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.053937+0000", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.181848854 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.053988+0000", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1357000000000003e-05 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.054024+0000", 2023-10-09T06:19:13.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6340000000000001e-05 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:53098 9.3 e158/152 9:cb523319:::benchmark_data_smithi107_300016_object53097:head v 158'3355, mlcod=158'3353)", 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:34.719019+0000", 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.656393250000001, 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27358244599999998, 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.392 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719019+0000", 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719019+0000", 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.393 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719019+0000", 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719053+0000", 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4677e-05 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719062+0000", 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6740000000000003e-06 2023-10-09T06:19:13.394 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719067+0000", 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719103+0000", 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5871000000000003e-05 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.395 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719122+0000", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9378e-05 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992536+0000", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27341405000000002 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992602+0000", 2023-10-09T06:19:13.396 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5421999999999998e-05 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:53102 9.c 9:3b9fc6a3:::benchmark_data_smithi107_300016_object53101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:34.719404+0000", 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.656008452000002, 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27959716200000001, 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.397 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53102 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719404+0000", 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.398 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719404+0000", 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719403+0000", 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719428+0000", 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5137999999999999e-05 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719441+0000", 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3309000000000001e-05 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719449+0000", 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7789999999999996e-06 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.400 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719489+0000", 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0133999999999999e-05 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719616+0000", 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012737800000000001 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.719756+0000", 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013969500000000001 2023-10-09T06:19:13.401 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992652+0000", 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27289553599999999 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998827+0000", 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0061752150000000004 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.402 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998909+0000", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2528999999999995e-05 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998961+0000", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1718000000000002e-05 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999001+0000", 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0185999999999998e-05 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.403 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:53109 9.8 9:13067e6c:::benchmark_data_smithi107_300016_object53108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:34.720371+0000", 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.655041451999999, 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27874684100000002, 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.404 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53109 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720371+0000", 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.405 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720371+0000", 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720370+0000", 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720392+0000", 2023-10-09T06:19:13.406 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2238e-05 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720405+0000", 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2574e-05 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720414+0000", 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7150000000000004e-06 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.407 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720465+0000", 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0974999999999999e-05 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.720590+0000", 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012505599999999999 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.408 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992706+0000", 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27211665400000001 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.997132+0000", 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044262429999999998 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.409 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.997998+0000", 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00086543099999999995 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999041+0000", 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00104346 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999083+0000", 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1980999999999997e-05 2023-10-09T06:19:13.410 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999118+0000", 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4604999999999998e-05 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:53108 9.3 e158/152 9:c9f9b61a:::benchmark_data_smithi107_300016_object53107:head v 158'3356, mlcod=158'3353)", 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:34.721354+0000", 2023-10-09T06:19:13.411 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.654058495999998, 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27697758900000002, 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.721354+0000", 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.412 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.721354+0000", 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.721353+0000", 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.721379+0000", 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5851e-05 2023-10-09T06:19:13.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992535+0000", 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.271155698 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992558+0000", 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3502e-05 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992659+0000", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100659 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992715+0000", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6366999999999998e-05 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998302+0000", 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055866869999999999 2023-10-09T06:19:13.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998332+0000", 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9547e-05 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:53112 9.c 9:3b860d9b:::benchmark_data_smithi107_300016_object53111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:34.722461+0000", 2023-10-09T06:19:13.416 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.652951575000003, 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.276662613, 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53112 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.417 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.722461+0000", 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.722461+0000", 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.722460+0000", 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.418 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.722489+0000", 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9153e-05 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992486+0000", 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26999743100000001 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.419 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992516+0000", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0199999999999999e-05 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992728+0000", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00021183000000000001 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.992874+0000", 2023-10-09T06:19:13.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014586499999999999 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.993019+0000", 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014518800000000001 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.998868+0000", 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058489099999999997 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999040+0000", 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172156 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999048+0000", 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9990000000000008e-06 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.422 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999088+0000", 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9313000000000001e-05 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:34.999124+0000", 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5955999999999999e-05 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.423 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64723 9.2 9:4bf0c3c2:::benchmark_data_smithi107_300016_object64722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.296870287000001, 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.362261308, 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64723 2023-10-09T06:19:13.424 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.425 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078541+0000", 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078567+0000", 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6197e-05 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.426 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078577+0000", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078583+0000", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4130000000000003e-06 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079048+0000", 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046495299999999999 2023-10-09T06:19:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079196+0000", 2023-10-09T06:19:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014766599999999999 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079363+0000", 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016698200000000001 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.084453+0000", 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050901360000000003 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440670+0000", 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.356217269 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440709+0000", 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9119999999999998e-05 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440762+0000", 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2203000000000001e-05 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440804+0000", 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1916999999999999e-05 2023-10-09T06:19:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64724 9.7 e163/160 9:e68efaaf:::benchmark_data_smithi107_300016_object64723:head v 163'4118, mlcod=0'0)", 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.296771502999999, 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35938271500000002, 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078682+0000", 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0822999999999999e-05 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078689+0000", 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078694+0000", 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0590000000000002e-06 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081003+0000", 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023091069999999999 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081031+0000", 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8022e-05 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437591+0000", 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35656025299999999 2023-10-09T06:19:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438024+0000", 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043272699999999999 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64725 9.f e163/152 9:f1989478:::benchmark_data_smithi107_300016_object64724:head v 163'3999, mlcod=163'3997)", 2023-10-09T06:19:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.296705826, 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35934786400000002, 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967296 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078736+0000", 2023-10-09T06:19:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9133e-05 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078741+0000", 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5060000000000003e-06 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078745+0000", 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1790000000000001e-06 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083200+0000", 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044543509999999996 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083218+0000", 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8397999999999999e-05 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438028+0000", 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35480992500000003 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438055+0000", 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6554e-05 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.443 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64727 9.8 9:1ddec4f7:::benchmark_data_smithi107_300016_object64726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.293823038999999, 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35651537999999999, 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64727 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081588+0000", 2023-10-09T06:19:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081613+0000", 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5406e-05 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081626+0000", 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2520999999999999e-05 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081634+0000", 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081691+0000", 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6190999999999998e-05 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081807+0000", 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011626299999999999 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081946+0000", 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013932899999999999 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087000+0000", 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050537489999999997 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437968+0000", 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35096775800000002 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438021+0000", 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3038999999999998e-05 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438065+0000", 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4045999999999998e-05 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438105+0000", 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0014e-05 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64729 9.3 9:cba02ed8:::benchmark_data_smithi107_300016_object64728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.293760848000002, 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37691757999999997, 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64729 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081651+0000", 2023-10-09T06:19:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081673+0000", 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2178999999999999e-05 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081681+0000", 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9219999999999997e-06 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.453 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081688+0000", 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2469999999999997e-06 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085025+0000", 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033377429999999998 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.454 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085147+0000", 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000122089 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085280+0000", 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013249099999999999 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444158+0000", 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35887852100000001 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.455 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444204+0000", 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5355999999999997e-05 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456528+0000", 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01232391 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.456 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456591+0000", 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.458569+0000", 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019784849999999999 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64730 9.f e163/152 9:f3222a22:::benchmark_data_smithi107_300016_object64729:head v 163'4000, mlcod=163'3997)", 2023-10-09T06:19:13.457 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.293684484, 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35634943299999999, 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.458 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081727+0000", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081887+0000", 2023-10-09T06:19:13.459 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016013400000000001 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081898+0000", 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0875e-05 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081916+0000", 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7195000000000001e-05 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.460 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085279+0000", 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033639469999999999 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085297+0000", 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7669e-05 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.461 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438058+0000", 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35276105099999999 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438077+0000", 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9317000000000002e-05 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.462 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64733 9.2 9:4941674d:::benchmark_data_smithi107_300016_object64732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.288676512999999, 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.360635498, 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64733 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.463 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086735+0000", 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086763+0000", 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7846000000000001e-05 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086772+0000", 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9039999999999998e-06 2023-10-09T06:19:13.465 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086778+0000", 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2759999999999997e-06 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086950+0000", 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172048 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.466 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087082+0000", 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013214800000000001 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087233+0000", 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000151213 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437967+0000", 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35073413799999997 2023-10-09T06:19:13.467 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440845+0000", 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028772379999999998 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447284+0000", 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064393209999999996 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.468 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447333+0000", 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9299999999999999e-05 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447372+0000", 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8277e-05 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.469 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64729 9.3 e163/160 ondisk, result = 0)", 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.286588762000001, 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.355337442, 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.470 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088823+0000", 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088829+0000", 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5189999999999998e-06 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088840+0000", 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088848+0000", 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.391e-06 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444142+0000", 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35529475900000002 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444151+0000", 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:19:13.473 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444161+0000", 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0239e-05 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64727 9.8 e163/152 ondisk, result = 0)", 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:19:13.474 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.285467419, 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.348172755, 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.475 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089944+0000", 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089951+0000", 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3820000000000001e-06 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437572+0000", 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34762106599999998 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437921+0000", 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034914000000000001 2023-10-09T06:19:13.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438007+0000", 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6438000000000001e-05 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438015+0000", 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4359999999999996e-06 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.478 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438118+0000", 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103149 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64737 9.2 9:42e1c82a:::benchmark_data_smithi107_300016_object64736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:13.479 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.285057273, 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.359336867, 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64737 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.480 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.481 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090354+0000", 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090375+0000", 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1294e-05 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.482 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437567+0000", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34719175099999999 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437894+0000", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032692099999999998 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437999+0000", 2023-10-09T06:19:13.483 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010511299999999999 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438153+0000", 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015426700000000001 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438341+0000", 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018829099999999999 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.484 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447252+0000", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0089106849999999998 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447436+0000", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018348799999999999 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449593+0000", 2023-10-09T06:19:13.485 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002157413 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449650+0000", 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7450000000000001e-05 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449692+0000", 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1686e-05 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.486 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64736 9.a e163/160 9:553011b2:::benchmark_data_smithi107_300016_object64735:head v 163'4090, mlcod=0'0)", 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 25.284692227000001, 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35900552499999999, 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.487 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090719+0000", 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:19:13.488 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090765+0000", 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5793e-05 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437575+0000", 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34680993900000001 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.489 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437924+0000", 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034926999999999999 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437970+0000", 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5179999999999998e-05 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.490 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438005+0000", 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5361000000000002e-05 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449697+0000", 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011691906 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.491 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449726+0000", 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8932999999999999e-05 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:70384 9.1 9:8e7a1678:::benchmark_data_smithi107_300016_object70383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:56.795527+0000", 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 16.579886012999999, 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18323009400000001, 2023-10-09T06:19:13.492 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 70384 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795527+0000", 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795527+0000", 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795526+0000", 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795548+0000", 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2028999999999999e-05 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795554+0000", 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8810000000000001e-06 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.795558+0000", 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.301e-06 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.798333+0000", 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027749990000000002 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.798470+0000", 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013698399999999999 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.798630+0000", 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015956600000000001 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.806758+0000", 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0081285560000000003 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.806767+0000", 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.405e-06 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809672+0000", 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002905702 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809728+0000", 2023-10-09T06:19:13.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5121000000000003e-05 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.978757+0000", 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.16902903799999999 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:70394 9.1 9:8eaeec87:::benchmark_data_smithi107_300016_object70393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T06:19:13.499 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:56.809430+0000", 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 16.565982476999999, 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18329156299999999, 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 70394 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809430+0000", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809430+0000", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809429+0000", 2023-10-09T06:19:13.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809452+0000", 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2866000000000001e-05 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809461+0000", 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6969999999999999e-06 2023-10-09T06:19:13.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.809467+0000", 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3820000000000001e-06 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.981127+0000", 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.17165951400000001 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.981283+0000", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015571000000000001 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.981459+0000", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000176131 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.989778+0000", 2023-10-09T06:19:13.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083193250000000007 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.989808+0000", 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0049999999999999e-05 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:19:13.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.989884+0000", 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5761999999999997e-05 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.989937+0000", 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2745999999999999e-05 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:56.992722+0000", 2023-10-09T06:19:13.506 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027849649999999999 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:19:13.507 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:19:13.507 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:13.570 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:13.638 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:13.654 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:13.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:13.762 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:13.867 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:19:13.872 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:13.877 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 82747 82731 75.9786 53.4375 0.00713874 0.0131556 2023-10-09T06:19:14.012 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[212750]: pgmap v572: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 569 op/s; 6653/235797 objects degraded (2.821%); 704/235797 objects misplaced (0.299%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:19:14.013 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4259605997' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.013 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[220921]: pgmap v572: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 569 op/s; 6653/235797 objects degraded (2.821%); 704/235797 objects misplaced (0.299%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:19:14.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4259605997' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.013 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.144 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:14.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:14.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:13 smithi150 ceph-mon[194178]: pgmap v572: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 569 op/s; 6653/235797 objects degraded (2.821%); 704/235797 objects misplaced (0.299%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:19:14.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4259605997' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:14.249 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:14.310 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:14.588 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:19:14.624 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:14.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:14.732 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:14.791 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:14.877 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 83467 83451 75.5299 45 0.00420385 0.0132124 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: failed to encode map e169 with expected crc 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: failed to encode map e169 with expected crc 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: failed to encode map e169 with expected crc 2023-10-09T06:19:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: failed to encode map e169 with expected crc 2023-10-09T06:19:14.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:14 smithi150 ceph-mon[194178]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[212750]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:14 smithi107 ceph-mon[220921]: failed to encode map e169 with expected crc 2023-10-09T06:19:15.252 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:15.263 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:15.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:15.370 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:15.396 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.444 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:15.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:15.548 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:15.577 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:15.877 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 83718 83702 74.6756 15.6875 0.00358965 0.0132255 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[220921]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 6653/246402 objects degraded (2.700%); 173/246402 objects misplaced (0.070%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:15 smithi107 ceph-mon[212750]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 6653/246402 objects degraded (2.700%); 173/246402 objects misplaced (0.070%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:19:16.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: failed to encode map e170 with expected crc 2023-10-09T06:19:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:15 smithi150 ceph-mon[194178]: pgmap v575: 89 pgs: 1 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 6653/246402 objects degraded (2.700%); 173/246402 objects misplaced (0.070%); 6.7 MiB/s, 107 objects/s recovering 2023-10-09T06:19:16.203 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:16.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:16.307 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:16.408 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:16.536 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:16.548 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:16.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:16.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:16.731 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:16.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[220921]: failed to encode map e171 with expected crc 2023-10-09T06:19:16.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[220921]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:19:16.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[220921]: failed to encode map e171 with expected crc 2023-10-09T06:19:16.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:16.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:16.878 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 84377 84361 74.2028 41.1875 0.0141527 0.0134714 2023-10-09T06:19:16.939 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:16.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:17.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:17.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:16 smithi150 ceph-mon[194178]: failed to encode map e171 with expected crc 2023-10-09T06:19:17.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:16 smithi150 ceph-mon[194178]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:19:17.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:16 smithi150 ceph-mon[194178]: failed to encode map e171 with expected crc 2023-10-09T06:19:17.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:17.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:17.121 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:17.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T06:19:17.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[212750]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T06:19:17.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[212750]: failed to encode map e171 with expected crc 2023-10-09T06:19:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:17.171 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:17.171 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "" 2023-10-09T06:19:17.171 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:17.211 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:19:17.273 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:17.293 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:17.313 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:17.361 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:17.407 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:17.463 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:17.525 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:17.575 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:17.586 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.586 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.586 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.586 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.587 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.587 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.587 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.587 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:17.879 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 85143 85127 73.8374 47.875 0.0121895 0.0135385 2023-10-09T06:19:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[212750]: pgmap v577: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5799/248298 objects degraded (2.336%); 743/248298 objects misplaced (0.299%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:19:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6653/246402 objects degraded (2.700%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1499682004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:17.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[220921]: pgmap v577: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5799/248298 objects degraded (2.336%); 743/248298 objects misplaced (0.299%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:19:17.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6653/246402 objects degraded (2.700%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:17.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:17.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1499682004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:17 smithi150 ceph-mon[194178]: pgmap v577: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s; 5799/248298 objects degraded (2.336%); 743/248298 objects misplaced (0.299%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T06:19:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6653/246402 objects degraded (2.700%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:19:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1499682004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:18.200 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:18.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:18.304 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:18.342 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:18.515 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:18.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:18.620 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:18.675 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:18.860 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:18.879 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 85929 85913 73.4987 49.125 0.0141537 0.0136011 2023-10-09T06:19:19.064 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:19.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:19.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:19.253 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:19.437 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:19.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:19.508 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:19.508 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:19.545 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:19.628 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:19.882 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 86842 86826 73.2739 57.0625 0.00887792 0.0136427 2023-10-09T06:19:20.365 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:20.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:20.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:19 smithi150 ceph-mon[194178]: pgmap v578: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5799/254331 objects degraded (2.280%); 570/254331 objects misplaced (0.224%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:19:20.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:19 smithi107 ceph-mon[212750]: pgmap v578: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5799/254331 objects degraded (2.280%); 570/254331 objects misplaced (0.224%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:19:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:19 smithi107 ceph-mon[220921]: pgmap v578: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 5799/254331 objects degraded (2.280%); 570/254331 objects misplaced (0.224%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:19:20.473 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:19:20.478 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:20.542 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:20.660 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:20.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:20.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:20.777 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:20.843 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:20.882 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 87477 87461 72.8263 39.6875 0.0222167 0.0137255 2023-10-09T06:19:20.928 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:20.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:21.037 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:21.090 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:21.236 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:21.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:21.320 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:21.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:21.437 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:21.520 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:21.574 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:21.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:21.682 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:21.756 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:21.888 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 88271 88255 72.5208 49.625 0.0233733 0.0137848 2023-10-09T06:19:22.029 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:19:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:22] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.43.0" 2023-10-09T06:19:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[212750]: pgmap v579: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s; 5799/256155 objects degraded (2.264%); 570/256155 objects misplaced (0.223%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:19:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3463792796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5799/256155 objects degraded (2.264%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[220921]: pgmap v579: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s; 5799/256155 objects degraded (2.264%); 570/256155 objects misplaced (0.223%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3463792796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:22.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5799/256155 objects degraded (2.264%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:22.220 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:22.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:22.225 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:22 smithi150 ceph-mon[194178]: pgmap v579: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 580 op/s; 5799/256155 objects degraded (2.264%); 570/256155 objects misplaced (0.223%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T06:19:22.225 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3463792796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.225 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:22.225 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:22.225 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5799/256155 objects degraded (2.264%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:22.325 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:22.397 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:22.882 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 89194 89178 72.3282 57.6875 0.00975301 0.0138134 2023-10-09T06:19:23.013 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:23.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:23.117 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:23.216 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:23.316 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.320 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:23 smithi150 ceph-mon[194178]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:23.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:23.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[212750]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:23 smithi107 ceph-mon[220921]: failed to encode map e172 with expected crc 2023-10-09T06:19:23.422 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:23.454 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:23.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:23.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:23.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:23.753 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:23.819 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:23.881 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:23.881 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:23.887 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 89899 89883 71.962 44.0625 0.0232271 0.0138908 2023-10-09T06:19:24.052 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:24.072 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:24 smithi107 ceph-mon[212750]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 5799/259269 objects degraded (2.237%); 233/259269 objects misplaced (0.090%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:24.073 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:24 smithi107 ceph-mon[220921]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 5799/259269 objects degraded (2.237%); 233/259269 objects misplaced (0.090%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:24.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:24.123 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:19:24.123 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:19:24.123 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T06:19:24.123 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:19:24.174 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:24.220 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:24 smithi150 ceph-mon[194178]: pgmap v581: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33 MiB/s wr, 535 op/s; 5799/259269 objects degraded (2.237%); 233/259269 objects misplaced (0.090%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:24.226 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:24.815 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:24.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:24.888 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 90742 90726 71.7167 52.6875 0.0124041 0.0139387 2023-10-09T06:19:24.900 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:24.947 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:24.992 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:25.038 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:25.092 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:25.116 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:25.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:25.127 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:25.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.141 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.141 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.141 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:25.165 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:25.220 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:25.252 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:19:25.285 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2065462133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:25.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2065462133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:25.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2065462133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:25.464 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:25.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:25.570 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:25.601 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.756 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:25.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:25.859 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:25.888 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:19:25.889530+0000 min lat: 0.00127393 max lat: 2.27436 avg lat: 0.0139339 2023-10-09T06:19:25.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:19:25.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 91229 91213 71.2011 30.4375 0.00475265 0.0139339 2023-10-09T06:19:25.892 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:25.910 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:19:25.911 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[212750]: pgmap v582: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5799/268008 objects degraded (2.164%); 233/268008 objects misplaced (0.087%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[212750]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[212750]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[220921]: pgmap v582: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5799/268008 objects degraded (2.164%); 233/268008 objects misplaced (0.087%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:19:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[220921]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[220921]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:19:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[220921]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:26 smithi107 ceph-mon[220921]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.324 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:26 smithi150 ceph-mon[194178]: pgmap v582: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 5799/268008 objects degraded (2.164%); 233/268008 objects misplaced (0.087%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:19:26.324 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:26 smithi150 ceph-mon[194178]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.324 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:26 smithi150 ceph-mon[194178]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T06:19:26.324 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:26 smithi150 ceph-mon[194178]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.324 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:26 smithi150 ceph-mon[194178]: failed to encode map e173 with expected crc 2023-10-09T06:19:26.680 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:26.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:26.790 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:26.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 91749 91733 70.7236 32.5 0.0211788 0.0141335 2023-10-09T06:19:26.907 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:26.955 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:27.059 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:27.087 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5799/268008 objects degraded (2.164%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5799/268008 objects degraded (2.164%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: failed to encode map e174 with expected crc 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:27.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5799/268008 objects degraded (2.164%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:27.417 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:27.521 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:27.550 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:27.595 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:27.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:27.772 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:27.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 92313 92297 70.2913 35.25 0.0186279 0.0142192 2023-10-09T06:19:27.918 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:27.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:27.943 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:28.028 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:28.097 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:28.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.242 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:48686 8.2 8:4cde1e59:::benchmark_data_smithi107_251897_object48685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 407.20061023699998, 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20579032899999999, 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 48686 2023-10-09T06:19:28.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:19:28.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036653+0000", 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036694+0000", 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1156999999999999e-05 2023-10-09T06:19:28.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036709+0000", 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5041e-05 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036715+0000", 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7830000000000004e-06 2023-10-09T06:19:28.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036752+0000", 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7527999999999997e-05 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036856+0000", 2023-10-09T06:19:28.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103558 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036956+0000", 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100714 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.038727+0000", 2023-10-09T06:19:28.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017710110000000001 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.038924+0000", 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000196911 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242375+0000", 2023-10-09T06:19:28.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20345057799999999 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242410+0000", 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4947999999999998e-05 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242444+0000", 2023-10-09T06:19:28.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4567999999999998e-05 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:62892 8.2 8:4af4e149:::benchmark_data_smithi107_251897_object62891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 396.76559217200003, 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.309153503, 2023-10-09T06:19:28.251 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 62892 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471671+0000", 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471735+0000", 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4172999999999997e-05 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.254 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471752+0000", 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7258e-05 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471758+0000", 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4949999999999996e-06 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.255 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471816+0000", 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7766000000000002e-05 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471935+0000", 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000118294 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.256 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.472060+0000", 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012510700000000001 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.476873+0000", 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048130719999999998 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.257 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.476879+0000", 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5810000000000002e-06 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780714+0000", 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30383470600000001 2023-10-09T06:19:28.258 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780762+0000", 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7963999999999997e-05 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780826+0000", 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3746999999999998e-05 2023-10-09T06:19:28.259 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:62902 8.2 8:4da830c8:::benchmark_data_smithi107_251897_object62901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 396.75825679600001, 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30722065199999998, 2023-10-09T06:19:28.260 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 62902 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:19:28.261 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.262 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479005+0000", 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479071+0000", 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5686999999999996e-05 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479084+0000", 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3267e-05 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479092+0000", 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3429999999999998e-06 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479138+0000", 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5998999999999998e-05 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479259+0000", 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012126299999999999 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.265 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479386+0000", 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127069 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.481372+0000", 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019865870000000002 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.266 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.481910+0000", 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000537631 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786138+0000", 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30422770599999999 2023-10-09T06:19:28.267 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786189+0000", 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1708e-05 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.268 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786228+0000", 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8720000000000002e-05 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.269 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:75955 8.2 8:4848e1b1:::benchmark_data_smithi107_251897_object75954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 386.54099212300002, 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28918334099999998, 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.270 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 75955 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.271 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:19:28.272 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696270+0000", 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.273 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696336+0000", 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5167999999999997e-05 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.274 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696350+0000", 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4881e-05 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696357+0000", 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3289999999999999e-06 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696404+0000", 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.702e-05 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696519+0000", 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0001149 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696647+0000", 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012799099999999999 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.699310+0000", 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026631950000000001 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.701937+0000", 2023-10-09T06:19:28.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026272370000000001 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985263+0000", 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28332602200000001 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985393+0000", 2023-10-09T06:19:28.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012951400000000001 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985456+0000", 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2872000000000004e-05 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.279 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:75967 8.2 8:411c9d04:::benchmark_data_smithi107_251897_object75966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 386.53211606100001, 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28091582799999998, 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 75967 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705147+0000", 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:28.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705184+0000", 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7125000000000001e-05 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705200+0000", 2023-10-09T06:19:28.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5871000000000001e-05 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705207+0000", 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0090000000000001e-06 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705258+0000", 2023-10-09T06:19:28.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1437999999999998e-05 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705400+0000", 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014183500000000001 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705531+0000", 2023-10-09T06:19:28.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013155600000000001 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.708330+0000", 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027981019999999998 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.708685+0000", 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00035532100000000002 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986008+0000", 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27732359699999998 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986034+0000", 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5587000000000001e-05 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986064+0000", 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0083e-05 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.288 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:167069 8.2 8:4fc8af52:::benchmark_data_smithi107_251897_object50155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 270.13617605100001, 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25242683300000002, 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.289 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 167069 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101087+0000", 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101093+0000", 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3070000000000002e-06 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101104+0000", 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1467999999999999e-05 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101110+0000", 2023-10-09T06:19:28.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3899999999999998e-06 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101158+0000", 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8038999999999999e-05 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101306+0000", 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014781899999999999 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101453+0000", 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014681399999999999 2023-10-09T06:19:28.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.105161+0000", 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037081760000000001 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.105887+0000", 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072546799999999995 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353412+0000", 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247525683 2023-10-09T06:19:28.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353471+0000", 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8606000000000001e-05 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353515+0000", 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4168999999999997e-05 2023-10-09T06:19:28.298 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:167076 8.2 8:404a6a5b:::benchmark_data_smithi107_251897_object50162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 270.12066579700002, 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23699437900000001, 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.299 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 167076 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:19:28.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116597+0000", 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116606+0000", 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6690000000000001e-06 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116632+0000", 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6047e-05 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116645+0000", 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2262999999999999e-05 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116733+0000", 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8582e-05 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116979+0000", 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024556900000000001 2023-10-09T06:19:28.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.117103+0000", 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012399700000000001 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.121152+0000", 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004049708 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.121436+0000", 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00028356799999999999 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353539+0000", 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23210345499999999 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353569+0000", 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9068e-05 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353593+0000", 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4399000000000001e-05 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231202 8.b 8:da53e5b5:::benchmark_data_smithi107_251897_object114288:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:19:28.309 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 178.64036698500001, 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.094812632, 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231202 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.310 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:19:28.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596896+0000", 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596901+0000", 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7650000000000001e-06 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596912+0000", 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0589e-05 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596918+0000", 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9939999999999997e-06 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596959+0000", 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1072000000000002e-05 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596966+0000", 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8079999999999999e-06 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.676763+0000", 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0797972950000001 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.680993+0000", 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042302499999999996 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.681374+0000", 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00038049299999999998 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.690404+0000", 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090298470000000006 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691560+0000", 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001156921 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691620+0000", 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9765000000000001e-05 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691710+0000", 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9808000000000005e-05 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233197 8.b 8:d053390f:::benchmark_data_smithi107_251897_object116283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.30515902100001, 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26465855500000002, 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233197 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932104+0000", 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932112+0000", 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8869999999999992e-06 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932128+0000", 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5917999999999999e-05 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932134+0000", 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5560000000000004e-06 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.936939+0000", 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048044639999999996 2023-10-09T06:19:28.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.937036+0000", 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7293000000000005e-05 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.937103+0000", 2023-10-09T06:19:28.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6816000000000005e-05 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941206+0000", 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041027770000000002 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196612+0000", 2023-10-09T06:19:28.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.255406313 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196671+0000", 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8581e-05 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196764+0000", 2023-10-09T06:19:28.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3227999999999996e-05 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233199 8.b 8:d15cf55a:::benchmark_data_smithi107_251897_object116285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:19:28.327 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.30511340300001, 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27300426900000002, 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233199 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.328 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932153+0000", 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7879999999999998e-06 2023-10-09T06:19:28.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932160+0000", 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932164+0000", 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2820000000000001e-06 2023-10-09T06:19:28.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941229+0000", 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090653000000000001 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941345+0000", 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116309 2023-10-09T06:19:28.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941461+0000", 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011604600000000001 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945642+0000", 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041809969999999997 2023-10-09T06:19:28.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.204998+0000", 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25935536799999998 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.205050+0000", 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1820000000000002e-05 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.205155+0000", 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105695 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233200 8.b 8:df5e5b62:::benchmark_data_smithi107_251897_object116286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.30509707300001, 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.284936368, 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233200 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932169+0000", 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8959999999999999e-06 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932174+0000", 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0200000000000002e-06 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932176+0000", 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2840000000000001e-06 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945722+0000", 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013545756000000001 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945858+0000", 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013620700000000001 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.946003+0000", 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014461399999999999 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.955361+0000", 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0093577720000000003 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.216959+0000", 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159835399999998 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217014+0000", 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4593999999999999e-05 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217104+0000", 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0131999999999997e-05 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233227 8.b 8:dfa0532a:::benchmark_data_smithi107_251897_object116313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.273738366, 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25373610600000002, 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.344 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233227 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963525+0000", 2023-10-09T06:19:28.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963534+0000", 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.918e-06 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963548+0000", 2023-10-09T06:19:28.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4175999999999999e-05 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963553+0000", 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2399999999999998e-06 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963604+0000", 2023-10-09T06:19:28.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.075e-05 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963732+0000", 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012822300000000001 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963849+0000", 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116897 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.967834+0000", 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0039854490000000003 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217144+0000", 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24931002699999999 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217186+0000", 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1783999999999998e-05 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217262+0000", 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5878999999999996e-05 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233231 8.b 8:d0139bd7:::benchmark_data_smithi107_251897_object116317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:19:28.352 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.26889647799999, 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24903313799999999, 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233231 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968367+0000", 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968371+0000", 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9659999999999998e-06 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968380+0000", 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4239999999999999e-06 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968384+0000", 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0790000000000002e-06 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968420+0000", 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5225999999999998e-05 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968521+0000", 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010161400000000001 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968599+0000", 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7506e-05 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.971331+0000", 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002732319 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217292+0000", 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245960447 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217326+0000", 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4758999999999997e-05 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217401+0000", 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4668999999999999e-05 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:65202 9.e 9:75736f69:::benchmark_data_smithi107_300016_object65201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:18:48.986317+0000", 2023-10-09T06:19:28.360 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 39.250947535999998, 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.112959529, 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 65202 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.361 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986317+0000", 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986317+0000", 2023-10-09T06:19:28.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986316+0000", 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986403+0000", 2023-10-09T06:19:28.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6884000000000004e-05 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986415+0000", 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1963e-05 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.986422+0000", 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8190000000000003e-06 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.988124+0000", 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001701812 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.988292+0000", 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016844599999999999 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.988485+0000", 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000193375 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:48.990339+0000", 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001853143 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:49.090061+0000", 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.099722859999999997 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:49.097947+0000", 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0078856380000000004 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:49.097991+0000", 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4057999999999997e-05 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:49.099276+0000", 2023-10-09T06:19:28.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0012852549999999999 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:68752 9.e 9:74c48cb7:::benchmark_data_smithi107_300016_object68751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 33.930350419, 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16018084899999999, 2023-10-09T06:19:28.369 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 68752 2023-10-09T06:19:28.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.373 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306912+0000", 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306959+0000", 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6937999999999997e-05 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306976+0000", 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6347000000000001e-05 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306982+0000", 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2419999999999997e-06 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307041+0000", 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9388999999999997e-05 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307224+0000", 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018262700000000001 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307418+0000", 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019383900000000001 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.309930+0000", 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002512065 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.312639+0000", 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00270893 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.464532+0000", 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15189308000000001 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.464601+0000", 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9190000000000007e-05 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.467095+0000", 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024937549999999998 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:82724 9.e 9:71b15de5:::benchmark_data_smithi107_300016_object82723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.382236291, 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14464223500000001, 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.381 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82724 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855027+0000", 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855057+0000", 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9502e-05 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855064+0000", 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7800000000000001e-06 2023-10-09T06:19:28.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855069+0000", 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1989999999999999e-06 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855121+0000", 2023-10-09T06:19:28.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2414999999999999e-05 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855304+0000", 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018354600000000001 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:19:28.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855547+0000", 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024204000000000001 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.859695+0000", 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041487620000000003 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.869371+0000", 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0096756419999999999 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.998875+0000", 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.12950377699999999 2023-10-09T06:19:28.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.998940+0000", 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5136999999999997e-05 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.999670+0000", 2023-10-09T06:19:28.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00073044400000000002 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:82748 9.e 9:7e63c7f7:::benchmark_data_smithi107_300016_object82747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:13.882175+0000", 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.355089021, 2023-10-09T06:19:28.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.124175357, 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82748 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882175+0000", 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882175+0000", 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882173+0000", 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882229+0000", 2023-10-09T06:19:28.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5587999999999997e-05 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882244+0000", 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5305999999999999e-05 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882253+0000", 2023-10-09T06:19:28.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6920000000000008e-06 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882302+0000", 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8699000000000002e-05 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882429+0000", 2023-10-09T06:19:28.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127207 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.882565+0000", 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013614599999999999 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.885317+0000", 2023-10-09T06:19:28.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027520029999999998 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.886679+0000", 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013618879999999999 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:14.005605+0000", 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.118925489 2023-10-09T06:19:28.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:14.005662+0000", 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7510999999999999e-05 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:14.006351+0000", 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00068867699999999995 2023-10-09T06:19:28.398 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:83753 9.e 9:7eb22873:::benchmark_data_smithi107_300016_object83752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 12.288858229000001, 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.276253264, 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.399 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 83753 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948404+0000", 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:28.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948466+0000", 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1511000000000001e-05 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948484+0000", 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8709999999999999e-05 2023-10-09T06:19:28.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948491+0000", 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0999999999999998e-06 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949392+0000", 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00090033300000000001 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949525+0000", 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013276299999999999 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949682+0000", 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015716800000000001 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.955464+0000", 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057822680000000001 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.956844+0000", 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013803540000000001 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.223817+0000", 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26697311299999998 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.223888+0000", 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0650999999999998e-05 2023-10-09T06:19:28.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.224659+0000", 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077126199999999999 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.407 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91488 9.e 9:71db0bc0:::benchmark_data_smithi107_300016_object91487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:26.473634+0000", 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.7636304919999999, 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.122690257, 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91488 2023-10-09T06:19:28.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473634+0000", 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473634+0000", 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473633+0000", 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473666+0000", 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2715000000000001e-05 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473674+0000", 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8690000000000004e-06 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.473679+0000", 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7779999999999998e-06 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.476982+0000", 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033038680000000002 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.477144+0000", 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161912 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.477310+0000", 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000165973 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.485543+0000", 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082326389999999999 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.488563+0000", 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030196400000000001 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.595522+0000", 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10695940299999999 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.595587+0000", 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5225e-05 2023-10-09T06:19:28.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.596324+0000", 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00073685899999999997 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91493 9.e 9:79a15459:::benchmark_data_smithi107_300016_object91492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:26.480377+0000", 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.756887066, 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.116422278, 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:28.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91493 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480377+0000", 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480377+0000", 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480376+0000", 2023-10-09T06:19:28.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480434+0000", 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7476999999999999e-05 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:28.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480444+0000", 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0020000000000001e-05 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.480449+0000", 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2950000000000001e-06 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.481288+0000", 2023-10-09T06:19:28.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00083956199999999997 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.481454+0000", 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016516799999999999 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:19:28.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.481631+0000", 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000177543 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.485552+0000", 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0039211539999999996 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.488623+0000", 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030703649999999998 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.596367+0000", 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10774464 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:19:28.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.596423+0000", 2023-10-09T06:19:28.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5470999999999998e-05 2023-10-09T06:19:28.425 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:28.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:28.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:28.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:19:28.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.596800+0000", 2023-10-09T06:19:28.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037679100000000002 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:28.427 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:28.505 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.609 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:28.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:28.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:28.721 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:28.750 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:28.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 93045 93029 69.9958 45.75 0.00453197 0.01428 2023-10-09T06:19:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: pgmap v585: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5799/269403 objects degraded (2.153%); 233/269403 objects misplaced (0.086%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:19:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: pgmap v585: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5799/269403 objects degraded (2.153%); 233/269403 objects misplaced (0.086%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: pgmap v585: 89 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 79 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 5799/269403 objects degraded (2.153%); 233/269403 objects misplaced (0.086%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:19:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:19:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:19:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:28.910 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:29.190 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:19:29.435 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:19:29.457 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:19:29.826 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:29.890 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 93784 93768 69.7125 46.1875 0.00451679 0.0143256 2023-10-09T06:19:30.236 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:30.346 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.371 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-10-09T06:19:28.944446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6035740576,"num_objects":92113,"num_object_clones":0,"num_object_copies":276339,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5015,"num_objects_misplaced":348,"num_objects_unfound":0,"num_objects_dirty":92113,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":92389,"num_write_kb":5896141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3083,"num_bytes_recovered":202845710,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92305,"ondisk_log_size":92305,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":287,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21373876,"kb_used_data":17835252,"kb_used_omap":20,"kb_used_meta":3538539,"kb_avail":728587340,"statfs":{"total":767960285184,"available":746073436160,"internally_reserved":0,"allocated":18263298048,"data_stored":18257789819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3623464590},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":390594560,"num_objects":5960,"num_object_clones":0,"num_object_copies":17880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-784,"num_objects_misplaced":115,"num_objects_unfound":0,"num_objects_dirty":5960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5960,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":669,"num_bytes_recovered":43843584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.940721"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079142+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:19:25.079142+0000","last_peered":"2023-10-09T06:19:25.079142+0000","last_clean":"2023-10-09T06:19:25.079142+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:19:25.079142+0000","last_undegraded":"2023-10-09T06:19:25.079142+0000","last_fullsized":"2023-10-09T06:19:25.079142+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":109,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.081374+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:19:25.081374+0000","last_peered":"2023-10-09T06:19:25.081374+0000","last_clean":"2023-10-09T06:19:25.081374+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:19:25.081374+0000","last_undegraded":"2023-10-09T06:19:25.081374+0000","last_fullsized":"2023-10-09T06:19:25.081374+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":143,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.085406+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:19:25.085406+0000","last_peered":"2023-10-09T06:19:25.085406+0000","last_clean":"2023-10-09T06:19:25.085406+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:19:25.085406+0000","last_undegraded":"2023-10-09T06:19:25.085406+0000","last_fullsized":"2023-10-09T06:19:25.085406+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":140,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860291+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:19:15.860291+0000","last_peered":"2023-10-09T06:19:15.860291+0000","last_clean":"2023-10-09T06:19:15.860291+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:19:15.860291+0000","last_undegraded":"2023-10-09T06:19:15.860291+0000","last_fullsized":"2023-10-09T06:19:15.860291+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":127,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.084180+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:19:25.084180+0000","last_peered":"2023-10-09T06:19:25.084180+0000","last_clean":"2023-10-09T06:19:25.084180+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:19:25.084180+0000","last_undegraded":"2023-10-09T06:19:25.084180+0000","last_fullsized":"2023-10-09T06:19:25.084180+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":133,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093654+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:19:25.093654+0000","last_peered":"2023-10-09T06:19:25.093654+0000","last_clean":"2023-10-09T06:19:25.093654+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:19:25.093654+0000","last_undegraded":"2023-10-09T06:19:25.093654+0000","last_fullsized":"2023-10-09T06:19:25.093654+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":121,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860382+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:19:15.860382+0000","last_peered":"2023-10-09T06:19:15.860382+0000","last_clean":"2023-10-09T06:19:15.860382+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:19:15.860382+0000","last_undegraded":"2023-10-09T06:19:15.860382+0000","last_fullsized":"2023-10-09T06:19:15.860382+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.084274+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:19:25.084274+0000","last_peered":"2023-10-09T06:19:25.084274+0000","last_clean":"2023-10-09T06:19:25.084274+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:19:25.084274+0000","last_undegraded":"2023-10-09T06:19:25.084274+0000","last_fullsized":"2023-10-09T06:19:25.084274+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072938+0000","last_change":"2023-10-09T06:19:06.673434+0000","last_active":"2023-10-09T06:19:25.072938+0000","last_peered":"2023-10-09T06:19:25.072938+0000","last_clean":"2023-10-09T06:19:25.072938+0000","last_became_active":"2023-10-09T06:19:06.673042+0000","last_became_peered":"2023-10-09T06:19:06.673042+0000","last_unstale":"2023-10-09T06:19:25.072938+0000","last_undegraded":"2023-10-09T06:19:25.072938+0000","last_fullsized":"2023-10-09T06:19:25.072938+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:30.387567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.087160+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:19:25.087160+0000","last_peered":"2023-10-09T06:19:25.087160+0000","last_clean":"2023-10-09T06:19:25.087160+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:19:25.087160+0000","last_undegraded":"2023-10-09T06:19:25.087160+0000","last_fullsized":"2023-10-09T06:19:25.087160+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":23,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.075547+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:19:25.075547+0000","last_peered":"2023-10-09T06:19:25.075547+0000","last_clean":"2023-10-09T06:19:25.075547+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:19:25.075547+0000","last_undegraded":"2023-10-09T06:19:25.075547+0000","last_fullsized":"2023-10-09T06:19:25.075547+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.086739+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:19:25.086739+0000","last_peered":"2023-10-09T06:19:25.086739+0000","last_clean":"2023-10-09T06:19:25.086739+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:19:25.086739+0000","last_undegraded":"2023-10-09T06:19:25.086739+0000","last_fullsized":"2023-10-09T06:19:25.086739+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":180,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079873+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:19:25.079873+0000","last_peered":"2023-10-09T06:19:25.079873+0000","last_clean":"2023-10-09T06:19:25.079873+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:19:25.079873+0000","last_undegraded":"2023-10-09T06:19:25.079873+0000","last_fullsized":"2023-10-09T06:19:25.079873+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079577+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:19:25.079577+0000","last_peered":"2023-10-09T06:19:25.079577+0000","last_clean":"2023-10-09T06:19:25.079577+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:19:25.079577+0000","last_undegraded":"2023-10-09T06:19:25.079577+0000","last_fullsized":"2023-10-09T06:19:25.079577+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":23,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.077516+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:19:25.077516+0000","last_peered":"2023-10-09T06:19:25.077516+0000","last_clean":"2023-10-09T06:19:25.077516+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:19:25.077516+0000","last_undegraded":"2023-10-09T06:19:25.077516+0000","last_fullsized":"2023-10-09T06:19:25.077516+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.088998+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:19:25.088998+0000","last_peered":"2023-10-09T06:19:25.088998+0000","last_clean":"2023-10-09T06:19:25.088998+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:19:25.088998+0000","last_undegraded":"2023-10-09T06:19:25.088998+0000","last_fullsized":"2023-10-09T06:19:25.088998+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":139,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.859839+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:19:15.859839+0000","last_peered":"2023-10-09T06:19:15.859839+0000","last_clean":"2023-10-09T06:19:15.859839+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:19:15.859839+0000","last_undegraded":"2023-10-09T06:19:15.859839+0000","last_fullsized":"2023-10-09T06:19:15.859839+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.073066+0000","last_change":"2023-10-09T06:19:06.758564+0000","last_active":"2023-10-09T06:19:25.073066+0000","last_peered":"2023-10-09T06:19:25.073066+0000","last_clean":"2023-10-09T06:19:25.073066+0000","last_became_active":"2023-10-09T06:19:06.758270+0000","last_became_peered":"2023-10-09T06:19:06.758270+0000","last_unstale":"2023-10-09T06:19:25.073066+0000","last_undegraded":"2023-10-09T06:19:25.073066+0000","last_fullsized":"2023-10-09T06:19:25.073066+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.823391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.076920+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:19:25.076920+0000","last_peered":"2023-10-09T06:19:25.076920+0000","last_clean":"2023-10-09T06:19:25.076920+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:19:25.076920+0000","last_undegraded":"2023-10-09T06:19:25.076920+0000","last_fullsized":"2023-10-09T06:19:25.076920+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":139,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860413+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:19:15.860413+0000","last_peered":"2023-10-09T06:19:15.860413+0000","last_clean":"2023-10-09T06:19:15.860413+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:19:15.860413+0000","last_undegraded":"2023-10-09T06:19:15.860413+0000","last_fullsized":"2023-10-09T06:19:15.860413+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":142,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.091864+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:19:25.091864+0000","last_peered":"2023-10-09T06:19:25.091864+0000","last_clean":"2023-10-09T06:19:25.091864+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:19:25.091864+0000","last_undegraded":"2023-10-09T06:19:25.091864+0000","last_fullsized":"2023-10-09T06:19:25.091864+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.085881+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:19:25.085881+0000","last_peered":"2023-10-09T06:19:25.085881+0000","last_clean":"2023-10-09T06:19:25.085881+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:19:25.085881+0000","last_undegraded":"2023-10-09T06:19:25.085881+0000","last_fullsized":"2023-10-09T06:19:25.085881+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"172'5603","reported_seq":5650,"reported_epoch":172,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:24.893466+0000","last_change":"2023-10-09T06:19:06.794798+0000","last_active":"2023-10-09T06:19:24.893466+0000","last_peered":"2023-10-09T06:19:24.893466+0000","last_clean":"2023-10-09T06:18:42.927798+0000","last_became_active":"2023-10-09T06:19:06.788715+0000","last_became_peered":"2023-10-09T06:19:06.788715+0000","last_unstale":"2023-10-09T06:19:24.893466+0000","last_undegraded":"2023-10-09T06:19:06.683032+0000","last_fullsized":"2023-10-09T06:19:06.640087+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:06.396489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367198208,"num_objects":5603,"num_object_clones":0,"num_object_copies":16809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5603,"num_write_kb":358592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,1,5","objects":873},{"shards":"0,5","objects":4730}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":21,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.077231+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:19:25.077231+0000","last_peered":"2023-10-09T06:19:25.077231+0000","last_clean":"2023-10-09T06:19:25.077231+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:19:25.077231+0000","last_undegraded":"2023-10-09T06:19:25.077231+0000","last_fullsized":"2023-10-09T06:19:25.077231+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072255+0000","last_change":"2023-10-09T06:19:06.673411+0000","last_active":"2023-10-09T06:19:25.072255+0000","last_peered":"2023-10-09T06:19:25.072255+0000","last_clean":"2023-10-09T06:19:25.072255+0000","last_became_active":"2023-10-09T06:19:06.673113+0000","last_became_peered":"2023-10-09T06:19:06.673113+0000","last_unstale":"2023-10-09T06:19:25.072255+0000","last_undegraded":"2023-10-09T06:19:25.072255+0000","last_fullsized":"2023-10-09T06:19:25.072255+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:39.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"174'5705","reported_seq":5736,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:26.224828+0000","last_change":"2023-10-09T06:17:57.020187+0000","last_active":"2023-10-09T06:19:26.224828+0000","last_peered":"2023-10-09T06:19:26.224828+0000","last_clean":"2023-10-09T06:19:26.224828+0000","last_became_active":"2023-10-09T06:17:57.019892+0000","last_became_peered":"2023-10-09T06:17:57.019892+0000","last_unstale":"2023-10-09T06:19:26.224828+0000","last_undegraded":"2023-10-09T06:19:26.224828+0000","last_fullsized":"2023-10-09T06:19:26.224828+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:01.403006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373882880,"num_objects":5705,"num_object_clones":0,"num_object_copies":17115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5705,"num_write_kb":365120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":21,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.077215+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:19:25.077215+0000","last_peered":"2023-10-09T06:19:25.077215+0000","last_clean":"2023-10-09T06:19:25.077215+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:19:25.077215+0000","last_undegraded":"2023-10-09T06:19:25.077215+0000","last_fullsized":"2023-10-09T06:19:25.077215+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.078884+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:19:25.078884+0000","last_peered":"2023-10-09T06:19:25.078884+0000","last_clean":"2023-10-09T06:19:25.078884+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:19:25.078884+0000","last_undegraded":"2023-10-09T06:19:25.078884+0000","last_fullsized":"2023-10-09T06:19:25.078884+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"174'5864","reported_seq":5911,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:28.015693+0000","last_change":"2023-10-09T06:19:06.768740+0000","last_active":"2023-10-09T06:19:28.015693+0000","last_peered":"2023-10-09T06:19:28.015693+0000","last_clean":"2023-10-09T06:18:42.965906+0000","last_became_active":"2023-10-09T06:19:06.763469+0000","last_became_peered":"2023-10-09T06:19:06.763469+0000","last_unstale":"2023-10-09T06:19:28.015693+0000","last_undegraded":"2023-10-09T06:19:06.661874+0000","last_fullsized":"2023-10-09T06:19:06.644760+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:38.702698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384303104,"num_objects":5864,"num_object_clones":0,"num_object_copies":17592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":824,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5864,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"2,3,5","objects":824},{"shards":"3,5","objects":5040}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":173,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T06:19:25.075292+0000","last_change":"2023-10-09T06:19:06.758153+0000","last_active":"2023-10-09T06:19:25.075292+0000","last_peered":"2023-10-09T06:19:25.075292+0000","last_clean":"2023-10-09T05:57:55.834431+0000","last_became_active":"2023-10-09T06:19:06.757480+0000","last_became_peered":"2023-10-09T06:19:06.757480+0000","last_unstale":"2023-10-09T06:19:25.075292+0000","last_undegraded":"2023-10-09T06:19:06.676823+0000","last_fullsized":"2023-10-09T06:19:25.075292+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:57.712635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.092041+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:19:25.092041+0000","last_peered":"2023-10-09T06:19:25.092041+0000","last_clean":"2023-10-09T06:19:25.092041+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:19:25.092041+0000","last_undegraded":"2023-10-09T06:19:25.092041+0000","last_fullsized":"2023-10-09T06:19:25.092041+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"174'5866","reported_seq":5896,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:27.999039+0000","last_change":"2023-10-09T06:17:57.217507+0000","last_active":"2023-10-09T06:19:27.999039+0000","last_peered":"2023-10-09T06:19:27.999039+0000","last_clean":"2023-10-09T06:19:27.999039+0000","last_became_active":"2023-10-09T06:17:57.217183+0000","last_became_peered":"2023-10-09T06:17:57.217183+0000","last_unstale":"2023-10-09T06:19:27.999039+0000","last_undegraded":"2023-10-09T06:19:27.999039+0000","last_fullsized":"2023-10-09T06:19:27.999039+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5866,"log_dups_size":0,"ondisk_log_size":5866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:56.905622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384434176,"num_objects":5866,"num_object_clones":0,"num_object_copies":17598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5866,"num_write_kb":375424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072449+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:19:25.072449+0000","last_peered":"2023-10-09T06:19:25.072449+0000","last_clean":"2023-10-09T06:19:25.072449+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:19:25.072449+0000","last_undegraded":"2023-10-09T06:19:25.072449+0000","last_fullsized":"2023-10-09T06:19:25.072449+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860043+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:19:15.860043+0000","last_peered":"2023-10-09T06:19:15.860043+0000","last_clean":"2023-10-09T06:19:15.860043+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:19:15.860043+0000","last_undegraded":"2023-10-09T06:19:15.860043+0000","last_fullsized":"2023-10-09T06:19:15.860043+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"174'5843","reported_seq":5890,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:28.499432+0000","last_change":"2023-10-09T06:19:06.799481+0000","last_active":"2023-10-09T06:19:28.499432+0000","last_peered":"2023-10-09T06:19:28.499432+0000","last_clean":"2023-10-09T06:18:42.980943+0000","last_became_active":"2023-10-09T06:19:06.792382+0000","last_became_peered":"2023-10-09T06:19:06.792382+0000","last_unstale":"2023-10-09T06:19:28.499432+0000","last_undegraded":"2023-10-09T06:19:06.664330+0000","last_fullsized":"2023-10-09T06:19:06.650008+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5843,"log_dups_size":0,"ondisk_log_size":5843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:41.232240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382926848,"num_objects":5843,"num_object_clones":0,"num_object_copies":17529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":804,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5843,"num_write_kb":373952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,4,6","objects":804},{"shards":"3,6","objects":5039}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.082352+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:19:25.082352+0000","last_peered":"2023-10-09T06:19:25.082352+0000","last_clean":"2023-10-09T06:19:25.082352+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:19:25.082352+0000","last_undegraded":"2023-10-09T06:19:25.082352+0000","last_fullsized":"2023-10-09T06:19:25.082352+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079866+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:19:25.079866+0000","last_peered":"2023-10-09T06:19:25.079866+0000","last_clean":"2023-10-09T06:19:25.079866+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:19:25.079866+0000","last_undegraded":"2023-10-09T06:19:25.079866+0000","last_fullsized":"2023-10-09T06:19:25.079866+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"174'5600","reported_seq":5631,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:26.224237+0000","last_change":"2023-10-09T06:17:57.236716+0000","last_active":"2023-10-09T06:19:26.224237+0000","last_peered":"2023-10-09T06:19:26.224237+0000","last_clean":"2023-10-09T06:19:26.224237+0000","last_became_active":"2023-10-09T06:17:57.236363+0000","last_became_peered":"2023-10-09T06:17:57.236363+0000","last_unstale":"2023-10-09T06:19:26.224237+0000","last_undegraded":"2023-10-09T06:19:26.224237+0000","last_fullsized":"2023-10-09T06:19:26.224237+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:53.348003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367001600,"num_objects":5600,"num_object_clones":0,"num_object_copies":16800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5600,"num_write_kb":358400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.092164+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:19:25.092164+0000","last_peered":"2023-10-09T06:19:25.092164+0000","last_clean":"2023-10-09T06:19:25.092164+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:19:25.092164+0000","last_undegraded":"2023-10-09T06:19:25.092164+0000","last_fullsized":"2023-10-09T06:19:25.092164+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079310+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:19:25.079310+0000","last_peered":"2023-10-09T06:19:25.079310+0000","last_clean":"2023-10-09T06:19:25.079310+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:19:25.079310+0000","last_undegraded":"2023-10-09T06:19:25.079310+0000","last_fullsized":"2023-10-09T06:19:25.079310+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"174'5812","reported_seq":7691,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:28.525924+0000","last_change":"2023-10-09T06:19:06.769027+0000","last_active":"2023-10-09T06:19:28.525924+0000","last_peered":"2023-10-09T06:19:28.525924+0000","last_clean":"2023-10-09T06:18:42.941195+0000","last_became_active":"2023-10-09T06:19:06.762925+0000","last_became_peered":"2023-10-09T06:19:06.762925+0000","last_unstale":"2023-10-09T06:19:28.525924+0000","last_undegraded":"2023-10-09T06:19:06.659862+0000","last_fullsized":"2023-10-09T06:19:06.643401+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5812,"log_dups_size":0,"ondisk_log_size":5812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:56.217919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380895232,"num_objects":5812,"num_object_clones":0,"num_object_copies":17436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5812,"num_write_kb":371968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":917,"num_bytes_recovered":60096512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":4985},{"shards":"1,3,5","objects":827}],"blocked_by":[],"up_primary":7,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.078769+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:19:25.078769+0000","last_peered":"2023-10-09T06:19:25.078769+0000","last_clean":"2023-10-09T06:19:25.078769+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:19:25.078769+0000","last_undegraded":"2023-10-09T06:19:25.078769+0000","last_fullsized":"2023-10-09T06:19:25.078769+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.082970+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:19:25.082970+0000","last_peered":"2023-10-09T06:19:25.082970+0000","last_clean":"2023-10-09T06:19:25.082970+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:19:25.082970+0000","last_undegraded":"2023-10-09T06:19:25.082970+0000","last_fullsized":"2023-10-09T06:19:25.082970+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"174'5674","reported_seq":5705,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:28.476048+0000","last_change":"2023-10-09T06:17:57.217463+0000","last_active":"2023-10-09T06:19:28.476048+0000","last_peered":"2023-10-09T06:19:28.476048+0000","last_clean":"2023-10-09T06:19:28.476048+0000","last_became_active":"2023-10-09T06:17:57.217100+0000","last_became_peered":"2023-10-09T06:17:57.217100+0000","last_unstale":"2023-10-09T06:19:28.476048+0000","last_undegraded":"2023-10-09T06:19:28.476048+0000","last_fullsized":"2023-10-09T06:19:28.476048+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:23.024261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371851264,"num_objects":5674,"num_object_clones":0,"num_object_copies":17022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5674,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.084347+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:19:25.084347+0000","last_peered":"2023-10-09T06:19:25.084347+0000","last_clean":"2023-10-09T06:19:25.084347+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:19:25.084347+0000","last_undegraded":"2023-10-09T06:19:25.084347+0000","last_fullsized":"2023-10-09T06:19:25.084347+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.092847+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:19:25.092847+0000","last_peered":"2023-10-09T06:19:25.092847+0000","last_clean":"2023-10-09T06:19:25.092847+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:19:25.092847+0000","last_undegraded":"2023-10-09T06:19:25.092847+0000","last_fullsized":"2023-10-09T06:19:25.092847+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"174'5720","reported_seq":5750,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:27.958130+0000","last_change":"2023-10-09T06:17:57.022383+0000","last_active":"2023-10-09T06:19:27.958130+0000","last_peered":"2023-10-09T06:19:27.958130+0000","last_clean":"2023-10-09T06:19:27.958130+0000","last_became_active":"2023-10-09T06:17:57.022128+0000","last_became_peered":"2023-10-09T06:17:57.022128+0000","last_unstale":"2023-10-09T06:19:27.958130+0000","last_undegraded":"2023-10-09T06:19:27.958130+0000","last_fullsized":"2023-10-09T06:19:27.958130+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.131013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374865920,"num_objects":5720,"num_object_clones":0,"num_object_copies":17160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5720,"num_write_kb":366080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093310+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:19:25.093310+0000","last_peered":"2023-10-09T06:19:25.093310+0000","last_clean":"2023-10-09T06:19:25.093310+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:19:25.093310+0000","last_undegraded":"2023-10-09T06:19:25.093310+0000","last_fullsized":"2023-10-09T06:19:25.093310+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.073146+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:19:25.073146+0000","last_peered":"2023-10-09T06:19:25.073146+0000","last_clean":"2023-10-09T06:19:25.073146+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:19:25.073146+0000","last_undegraded":"2023-10-09T06:19:25.073146+0000","last_fullsized":"2023-10-09T06:19:25.073146+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":88,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.086477+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:19:25.086477+0000","last_peered":"2023-10-09T06:19:25.086477+0000","last_clean":"2023-10-09T06:19:25.086477+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:19:25.086477+0000","last_undegraded":"2023-10-09T06:19:25.086477+0000","last_fullsized":"2023-10-09T06:19:25.086477+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"172'5698","reported_seq":5743,"reported_epoch":172,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:24.894662+0000","last_change":"2023-10-09T06:19:06.776203+0000","last_active":"2023-10-09T06:19:24.894662+0000","last_peered":"2023-10-09T06:19:24.894662+0000","last_clean":"2023-10-09T06:18:42.947964+0000","last_became_active":"2023-10-09T06:19:06.770806+0000","last_became_peered":"2023-10-09T06:19:06.770806+0000","last_unstale":"2023-10-09T06:19:24.894662+0000","last_undegraded":"2023-10-09T06:19:06.649303+0000","last_fullsized":"2023-10-09T06:19:06.637848+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5698,"log_dups_size":0,"ondisk_log_size":5698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:57.755567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373424128,"num_objects":5698,"num_object_clones":0,"num_object_copies":17094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":830,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":364672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,5","objects":830},{"shards":"0,5","objects":4868}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072578+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:19:25.072578+0000","last_peered":"2023-10-09T06:19:25.072578+0000","last_clean":"2023-10-09T06:19:25.072578+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:19:25.072578+0000","last_undegraded":"2023-10-09T06:19:25.072578+0000","last_fullsized":"2023-10-09T06:19:25.072578+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079454+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:19:25.079454+0000","last_peered":"2023-10-09T06:19:25.079454+0000","last_clean":"2023-10-09T06:19:25.079454+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:19:25.079454+0000","last_undegraded":"2023-10-09T06:19:25.079454+0000","last_fullsized":"2023-10-09T06:19:25.079454+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":118,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093375+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:19:25.093375+0000","last_peered":"2023-10-09T06:19:25.093375+0000","last_clean":"2023-10-09T06:19:25.093375+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:19:25.093375+0000","last_undegraded":"2023-10-09T06:19:25.093375+0000","last_fullsized":"2023-10-09T06:19:25.093375+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"174'5785","reported_seq":7543,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:28.025298+0000","last_change":"2023-10-09T06:19:15.900457+0000","last_active":"2023-10-09T06:19:28.025298+0000","last_peered":"2023-10-09T06:19:28.025298+0000","last_clean":"2023-10-09T06:19:28.025298+0000","last_became_active":"2023-10-09T06:19:15.893387+0000","last_became_peered":"2023-10-09T06:19:15.893387+0000","last_unstale":"2023-10-09T06:19:28.025298+0000","last_undegraded":"2023-10-09T06:19:28.025298+0000","last_fullsized":"2023-10-09T06:19:28.025298+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:09.689997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379125760,"num_objects":5785,"num_object_clones":0,"num_object_copies":17355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5785,"num_write_kb":370240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":850,"num_bytes_recovered":55705600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":131,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:19:22.027183+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:19:22.027183+0000","last_peered":"2023-10-09T06:19:22.027183+0000","last_clean":"2023-10-09T06:19:22.027183+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:19:22.027183+0000","last_undegraded":"2023-10-09T06:19:22.027183+0000","last_fullsized":"2023-10-09T06:19:22.027183+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.085690+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:19:25.085690+0000","last_peered":"2023-10-09T06:19:25.085690+0000","last_clean":"2023-10-09T06:19:25.085690+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:19:25.085690+0000","last_undegraded":"2023-10-09T06:19:25.085690+0000","last_fullsized":"2023-10-09T06:19:25.085690+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":130,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093019+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:19:25.093019+0000","last_peered":"2023-10-09T06:19:25.093019+0000","last_clean":"2023-10-09T06:19:25.093019+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:19:25.093019+0000","last_undegraded":"2023-10-09T06:19:25.093019+0000","last_fullsized":"2023-10-09T06:19:25.093019+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"174'5836","reported_seq":5867,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:28.545273+0000","last_change":"2023-10-09T06:17:57.016464+0000","last_active":"2023-10-09T06:19:28.545273+0000","last_peered":"2023-10-09T06:19:28.545273+0000","last_clean":"2023-10-09T06:19:28.545273+0000","last_became_active":"2023-10-09T06:17:57.016166+0000","last_became_peered":"2023-10-09T06:17:57.016166+0000","last_unstale":"2023-10-09T06:19:28.545273+0000","last_undegraded":"2023-10-09T06:19:28.545273+0000","last_fullsized":"2023-10-09T06:19:28.545273+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:53.029953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382468096,"num_objects":5836,"num_object_clones":0,"num_object_copies":17508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5836,"num_write_kb":373504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":158,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072767+0000","last_change":"2023-10-09T06:19:06.672499+0000","last_active":"2023-10-09T06:19:25.072767+0000","last_peered":"2023-10-09T06:19:25.072767+0000","last_clean":"2023-10-09T06:19:25.072767+0000","last_became_active":"2023-10-09T06:19:06.672182+0000","last_became_peered":"2023-10-09T06:19:06.672182+0000","last_unstale":"2023-10-09T06:19:25.072767+0000","last_undegraded":"2023-10-09T06:19:25.072767+0000","last_fullsized":"2023-10-09T06:19:25.072767+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:07.246328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":442,"reported_epoch":171,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:15.859808+0000","last_change":"2023-10-09T06:19:07.672426+0000","last_active":"2023-10-09T06:19:15.859808+0000","last_peered":"2023-10-09T06:19:15.859808+0000","last_clean":"2023-10-09T06:19:04.606807+0000","last_became_active":"2023-10-09T06:19:07.672145+0000","last_became_peered":"2023-10-09T06:19:07.672145+0000","last_unstale":"2023-10-09T06:19:15.859808+0000","last_undegraded":"2023-10-09T06:19:07.650505+0000","last_fullsized":"2023-10-09T06:19:07.649893+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":163,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:13:14.553285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.082038+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:19:25.082038+0000","last_peered":"2023-10-09T06:19:25.082038+0000","last_clean":"2023-10-09T06:19:25.082038+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:19:25.082038+0000","last_undegraded":"2023-10-09T06:19:25.082038+0000","last_fullsized":"2023-10-09T06:19:25.082038+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093974+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:19:25.093974+0000","last_peered":"2023-10-09T06:19:25.093974+0000","last_clean":"2023-10-09T06:19:25.093974+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:19:25.093974+0000","last_undegraded":"2023-10-09T06:19:25.093974+0000","last_fullsized":"2023-10-09T06:19:25.093974+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"172'5785","reported_seq":5814,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:19:24.863798+0000","last_change":"2023-10-09T06:17:57.236464+0000","last_active":"2023-10-09T06:19:24.863798+0000","last_peered":"2023-10-09T06:19:24.863798+0000","last_clean":"2023-10-09T06:19:24.863798+0000","last_became_active":"2023-10-09T06:17:57.236124+0000","last_became_peered":"2023-10-09T06:17:57.236124+0000","last_unstale":"2023-10-09T06:19:24.863798+0000","last_undegraded":"2023-10-09T06:19:24.863798+0000","last_fullsized":"2023-10-09T06:19:24.863798+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:33.469044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379125760,"num_objects":5785,"num_object_clones":0,"num_object_copies":17355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5785,"num_write_kb":370240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.074382+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:19:25.074382+0000","last_peered":"2023-10-09T06:19:25.074382+0000","last_clean":"2023-10-09T06:19:25.074382+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:19:25.074382+0000","last_undegraded":"2023-10-09T06:19:25.074382+0000","last_fullsized":"2023-10-09T06:19:25.074382+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.859761+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:19:15.859761+0000","last_peered":"2023-10-09T06:19:15.859761+0000","last_clean":"2023-10-09T06:19:15.859761+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:19:15.859761+0000","last_undegraded":"2023-10-09T06:19:15.859761+0000","last_fullsized":"2023-10-09T06:19:15.859761+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":136,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860352+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:19:15.860352+0000","last_peered":"2023-10-09T06:19:15.860352+0000","last_clean":"2023-10-09T06:19:15.860352+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:19:15.860352+0000","last_undegraded":"2023-10-09T06:19:15.860352+0000","last_fullsized":"2023-10-09T06:19:15.860352+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"174'5822","reported_seq":5865,"reported_epoch":174,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:19:27.788938+0000","last_change":"2023-10-09T06:19:06.768480+0000","last_active":"2023-10-09T06:19:27.788938+0000","last_peered":"2023-10-09T06:19:27.788938+0000","last_clean":"2023-10-09T06:18:42.946249+0000","last_became_active":"2023-10-09T06:19:06.761917+0000","last_became_peered":"2023-10-09T06:19:06.761917+0000","last_unstale":"2023-10-09T06:19:27.788938+0000","last_undegraded":"2023-10-09T06:19:06.657097+0000","last_fullsized":"2023-10-09T06:19:06.646060+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:59.664720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381550592,"num_objects":5822,"num_object_clones":0,"num_object_copies":17466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":854,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5822,"num_write_kb":372608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"0,1,4","objects":853},{"shards":"1,4","objects":4968}],"blocked_by":[],"up_primary":7,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.075473+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:19:25.075473+0000","last_peered":"2023-10-09T06:19:25.075473+0000","last_clean":"2023-10-09T06:19:25.075473+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:19:25.075473+0000","last_undegraded":"2023-10-09T06:19:25.075473+0000","last_fullsized":"2023-10-09T06:19:25.075473+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":144,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093446+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:19:25.093446+0000","last_peered":"2023-10-09T06:19:25.093446+0000","last_clean":"2023-10-09T06:19:25.093446+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:19:25.093446+0000","last_undegraded":"2023-10-09T06:19:25.093446+0000","last_fullsized":"2023-10-09T06:19:25.093446+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":125,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.859898+0000","last_change":"2023-10-09T06:19:06.667489+0000","last_active":"2023-10-09T06:19:15.859898+0000","last_peered":"2023-10-09T06:19:15.859898+0000","last_clean":"2023-10-09T06:19:15.859898+0000","last_became_active":"2023-10-09T06:19:06.667029+0000","last_became_peered":"2023-10-09T06:19:06.667029+0000","last_unstale":"2023-10-09T06:19:15.859898+0000","last_undegraded":"2023-10-09T06:19:15.859898+0000","last_fullsized":"2023-10-09T06:19:15.859898+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:35.989088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"174'5794","reported_seq":7560,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-10-09T06:19:27.839826+0000","last_change":"2023-10-09T06:19:26.121898+0000","last_active":"2023-10-09T06:19:27.839826+0000","last_peered":"2023-10-09T06:19:27.839826+0000","last_clean":"2023-10-09T06:19:27.839826+0000","last_became_active":"2023-10-09T06:19:26.079797+0000","last_became_peered":"2023-10-09T06:19:26.079797+0000","last_unstale":"2023-10-09T06:19:27.839826+0000","last_undegraded":"2023-10-09T06:19:27.839826+0000","last_fullsized":"2023-10-09T06:19:27.839826+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:05.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379715584,"num_objects":5794,"num_object_clones":0,"num_object_copies":17382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":370816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":161,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.085968+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:19:25.085968+0000","last_peered":"2023-10-09T06:19:25.085968+0000","last_clean":"2023-10-09T06:19:25.085968+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:19:25.085968+0000","last_undegraded":"2023-10-09T06:19:25.085968+0000","last_fullsized":"2023-10-09T06:19:25.085968+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":129,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.092982+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:19:25.092982+0000","last_peered":"2023-10-09T06:19:25.092982+0000","last_clean":"2023-10-09T06:19:25.092982+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:19:25.092982+0000","last_undegraded":"2023-10-09T06:19:25.092982+0000","last_fullsized":"2023-10-09T06:19:25.092982+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.084410+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:19:25.084410+0000","last_peered":"2023-10-09T06:19:25.084410+0000","last_clean":"2023-10-09T06:19:25.084410+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:19:25.084410+0000","last_undegraded":"2023-10-09T06:19:25.084410+0000","last_fullsized":"2023-10-09T06:19:25.084410+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"174'5682","reported_seq":6601,"reported_epoch":174,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:19:28.537943+0000","last_change":"2023-10-09T06:19:24.121478+0000","last_active":"2023-10-09T06:19:28.537943+0000","last_peered":"2023-10-09T06:19:28.537943+0000","last_clean":"2023-10-09T06:18:42.943505+0000","last_became_active":"2023-10-09T06:19:06.761691+0000","last_became_peered":"2023-10-09T06:19:06.761691+0000","last_unstale":"2023-10-09T06:19:28.537943+0000","last_undegraded":"2023-10-09T06:19:28.537943+0000","last_fullsized":"2023-10-09T06:19:06.648441+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:20.646514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372375552,"num_objects":5682,"num_object_clones":0,"num_object_copies":17046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":348,"num_objects_unfound":0,"num_objects_dirty":5682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5682,"num_write_kb":363648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":436,"num_bytes_recovered":28573696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"0,3,4","objects":348},{"shards":"3,4","objects":5334}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":158,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.093580+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:19:25.093580+0000","last_peered":"2023-10-09T06:19:25.093580+0000","last_clean":"2023-10-09T06:19:25.093580+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:19:25.093580+0000","last_undegraded":"2023-10-09T06:19:25.093580+0000","last_fullsized":"2023-10-09T06:19:25.093580+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":126,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860216+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:19:15.860216+0000","last_peered":"2023-10-09T06:19:15.860216+0000","last_clean":"2023-10-09T06:19:15.860216+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:19:15.860216+0000","last_undegraded":"2023-10-09T06:19:15.860216+0000","last_fullsized":"2023-10-09T06:19:15.860216+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":117,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.078858+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:19:25.078858+0000","last_peered":"2023-10-09T06:19:25.078858+0000","last_clean":"2023-10-09T06:19:25.078858+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:19:25.078858+0000","last_undegraded":"2023-10-09T06:19:25.078858+0000","last_fullsized":"2023-10-09T06:19:25.078858+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":128,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860002+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:19:15.860002+0000","last_peered":"2023-10-09T06:19:15.860002+0000","last_clean":"2023-10-09T06:19:15.860002+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:19:15.860002+0000","last_undegraded":"2023-10-09T06:19:15.860002+0000","last_fullsized":"2023-10-09T06:19:15.860002+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":172,"state":"active+clean","last_fresh":"2023-10-09T06:19:22.026289+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:19:22.026289+0000","last_peered":"2023-10-09T06:19:22.026289+0000","last_clean":"2023-10-09T06:19:22.026289+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:19:22.026289+0000","last_undegraded":"2023-10-09T06:19:22.026289+0000","last_fullsized":"2023-10-09T06:19:22.026289+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":136,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.072556+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:19:25.072556+0000","last_peered":"2023-10-09T06:19:25.072556+0000","last_clean":"2023-10-09T06:19:25.072556+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:19:25.072556+0000","last_undegraded":"2023-10-09T06:19:25.072556+0000","last_fullsized":"2023-10-09T06:19:25.072556+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":161,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079387+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:19:25.079387+0000","last_peered":"2023-10-09T06:19:25.079387+0000","last_clean":"2023-10-09T06:19:25.079387+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:19:25.079387+0000","last_undegraded":"2023-10-09T06:19:25.079387+0000","last_fullsized":"2023-10-09T06:19:25.079387+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":119,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.073212+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:19:25.073212+0000","last_peered":"2023-10-09T06:19:25.073212+0000","last_clean":"2023-10-09T06:19:25.073212+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:19:25.073212+0000","last_undegraded":"2023-10-09T06:19:25.073212+0000","last_fullsized":"2023-10-09T06:19:25.073212+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.079063+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:19:25.079063+0000","last_peered":"2023-10-09T06:19:25.079063+0000","last_clean":"2023-10-09T06:19:25.079063+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:19:25.079063+0000","last_undegraded":"2023-10-09T06:19:25.079063+0000","last_fullsized":"2023-10-09T06:19:25.079063+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":129,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.081744+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:19:25.081744+0000","last_peered":"2023-10-09T06:19:25.081744+0000","last_clean":"2023-10-09T06:19:25.081744+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:19:25.081744+0000","last_undegraded":"2023-10-09T06:19:25.081744+0000","last_fullsized":"2023-10-09T06:19:25.081744+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":144,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.087331+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:19:25.087331+0000","last_peered":"2023-10-09T06:19:25.087331+0000","last_clean":"2023-10-09T06:19:25.087331+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:19:25.087331+0000","last_undegraded":"2023-10-09T06:19:25.087331+0000","last_fullsized":"2023-10-09T06:19:25.087331+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":126,"reported_epoch":171,"state":"active+clean","last_fresh":"2023-10-09T06:19:15.860247+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:19:15.860247+0000","last_peered":"2023-10-09T06:19:15.860247+0000","last_clean":"2023-10-09T06:19:15.860247+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:19:15.860247+0000","last_undegraded":"2023-10-09T06:19:15.860247+0000","last_fullsized":"2023-10-09T06:19:15.860247+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":173,"state":"active+clean","last_fresh":"2023-10-09T06:19:25.078626+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:19:25.078626+0000","last_peered":"2023-10-09T06:19:25.078626+0000","last_clean":"2023-10-09T06:19:25.078626+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:19:25.078626+0000","last_undegraded":"2023-10-09T06:19:25.078626+0000","last_fullsized":"2023-10-09T06:19:25.078626+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":9614,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6035144704,"num_objects":92089,"num_object_clones":0,"num_object_copies":276267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5012,"num_objects_misplaced":348,"num_objects_unfound":0,"num_objects_dirty":92089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92089,"num_write_kb":5893696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3059,"num_bytes_recovered":200474624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18288279552,"data_stored":18288279552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92089,"ondisk_log_size":92089,"up":48,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234266,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3570640,"kb_used_data":2972680,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":90174512,"statfs":{"total":95995035648,"available":92338700288,"internally_reserved":0,"allocated":3044024320,"data_stored":3043410336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:32 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.4039999999999999,"15min":1.2470000000000001},"min":{"1min":0.249,"5min":0.23300000000000001,"15min":0.22600000000000001},"max":{"1min":1.393,"5min":11.785,"15min":11.785},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.109,"15min":1.345},"min":{"1min":0.311,"5min":0.27400000000000002,"15min":0.251},"max":{"1min":2.8530000000000002,"5min":10.396000000000001,"15min":10.784000000000001},"last":2.3849999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:19:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8570000000000002,"5min":2.738,"15min":1.8919999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":31.869,"5min":31.869,"15min":31.869},"last":1.5449999999999999},{"interface":"front","average":{"1min":3.8860000000000001,"5min":2.266,"15min":1.629},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":29.486999999999998,"5min":29.486999999999998,"15min":29.486999999999998},"last":3.6280000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:19:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":1.5840000000000001,"15min":1.8939999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.6950000000000003,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.7030000000000001,"15min":2.0310000000000001},"min":{"1min":0.40400000000000003,"5min":0.378,"15min":0.378},"max":{"1min":4.3159999999999998,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":2.367}]},{"osd":3,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.69699999999999995,"15min":0.623},"min":{"1min":0.40999999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.6059999999999999,"5min":4.6059999999999999,"15min":4.6059999999999999},"last":8.1240000000000006},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.72199999999999998,"15min":0.65600000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0270000000000001,"5min":4.0270000000000001,"15min":4.0270000000000001},"last":2.3439999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.47,"15min":1.2010000000000001},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.251},"max":{"1min":3.9350000000000001,"5min":11.853999999999999,"15min":11.853999999999999},"last":1.3220000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.89000000000000001,"15min":1.222},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":2.6899999999999999,"5min":6.1500000000000004,"15min":9.234},"last":2.2000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.6559999999999999,"15min":1.3140000000000001},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":5.1219999999999999,"5min":9.8629999999999995,"15min":9.8629999999999995},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":1.335,"15min":1.1459999999999999},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.4420000000000002,"5min":11.837,"15min":11.837},"last":0.36399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.359,"15min":1.2509999999999999},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":3.4849999999999999,"5min":11.878,"15min":11.878},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.1240000000000001,"15min":1.2909999999999999},"min":{"1min":0.39700000000000002,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":2.6419999999999999,"5min":7.9340000000000002,"15min":9.2050000000000001},"last":2.2349999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495903,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1423236,"kb_used_data":1109564,"kb_used_omap":0,"kb_used_meta":313664,"kb_avail":92321916,"statfs":{"total":95995035648,"available":94537641984,"internally_reserved":0,"allocated":1136193536,"data_stored":1135591720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321191936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.4019999999999999,"15min":1.5229999999999999},"min":{"1min":0.32900000000000001,"5min":0.222,"15min":0.214},"max":{"1min":2.649,"5min":12.808,"15min":12.808},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.7310000000000001,"15min":1.6180000000000001},"min":{"1min":0.38500000000000001,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":3.327,"5min":12.847,"15min":12.847},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2389999999999999,"5min":2.6640000000000001,"15min":2.3220000000000001},"min":{"1min":0.40699999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":47.841000000000001,"5min":47.841000000000001,"15min":47.841000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":4.25,"5min":2.5089999999999999,"15min":2.383},"min":{"1min":0.253,"5min":0.246,"15min":0.246},"max":{"1min":47.820999999999998,"5min":47.820999999999998,"15min":47.820999999999998},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:19:18 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.8360000000000001,"15min":2.2730000000000001},"min":{"1min":0.44500000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.0039999999999996,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.7290000000000001,"15min":2.3730000000000002},"min":{"1min":0.36799999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.9870000000000001,"5min":8.9870000000000001,"15min":8.9870000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:18:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.64400000000000002,"15min":0.57499999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.4140000000000001,"5min":3.4140000000000001,"15min":3.4140000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.65700000000000003,"15min":0.59299999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.157,"5min":3.157,"15min":3.157},"last":0.434}]},{"osd":4,"last update":"Mon Oct 9 06:19:22 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":1.3819999999999999,"15min":1.056},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.245},"max":{"1min":5.6790000000000003,"5min":10.507,"15min":10.75},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.86399999999999999,"5min":1.198,"15min":1.2290000000000001},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.26400000000000001},"max":{"1min":4.944,"5min":8.9239999999999995,"15min":9.4100000000000001},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:19:22 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.2909999999999999,"15min":1.1919999999999999},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":5.0919999999999996,"5min":9.1839999999999993,"15min":10.773},"last":2.2690000000000001},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.2909999999999999,"15min":1.784},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":4.9699999999999998,"5min":11.798999999999999,"15min":159.06800000000001},"last":0.60699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":1.05,"15min":1.089},"min":{"1min":0.31900000000000001,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":2.7280000000000002,"5min":10.962,"15min":14.837},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.016,"5min":1.45,"15min":1.4350000000000001},"min":{"1min":0.371,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.2389999999999999,"5min":10.891,"15min":10.891},"last":0.71699999999999997}]}]},{"osd":1,"up_from":110,"seq":472446402639,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3593020,"kb_used_data":3326324,"kb_used_omap":4,"kb_used_meta":266683,"kb_avail":90152132,"statfs":{"total":95995035648,"available":92315783168,"internally_reserved":0,"allocated":3406155776,"data_stored":3405379282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":273084302},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.5,"15min":2.3660000000000001},"min":{"1min":0.214,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":15.489000000000001,"5min":15.489000000000001,"15min":15.489000000000001},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.397,"5min":1.772,"15min":2.6909999999999998},"min":{"1min":0.23999999999999999,"5min":0.21199999999999999,"15min":0.20100000000000001},"max":{"1min":4.8550000000000004,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.24099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:19:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":2.0129999999999999,"15min":1.827},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":15.56,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.318},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.2730000000000001,"15min":2.6749999999999998},"min":{"1min":0.24199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.372999999999999,"5min":49.783000000000001,"15min":49.783000000000001},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":0.67600000000000005,"15min":0.47999999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.1549999999999994,"5min":8.1549999999999994,"15min":8.1549999999999994},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.4830000000000001,"5min":0.623,"15min":0.47899999999999998},"min":{"1min":0.29999999999999999,"5min":0.25,"15min":0.25},"max":{"1min":8.6999999999999993,"5min":8.6999999999999993,"15min":8.6999999999999993},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.411,"15min":2.573},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.3120000000000003,"5min":6.3739999999999997,"15min":6.3739999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.6499999999999999,"15min":2.6179999999999999},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":3.7919999999999998,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.365,"15min":2.8889999999999998},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.4009999999999998,"5min":7.8890000000000002,"15min":10.92},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.837,"15min":2.8929999999999998},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.3869999999999996,"5min":10.795999999999999,"15min":10.795999999999999},"last":0.441}]},{"osd":6,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":2.0670000000000002,"15min":2.7120000000000002},"min":{"1min":0.436,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.4569999999999999,"5min":12.91,"15min":12.91},"last":0.64500000000000002},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.7430000000000001,"15min":2.3900000000000001},"min":{"1min":0.39200000000000002,"5min":0.374,"15min":0.374},"max":{"1min":8.1059999999999999,"5min":8.1059999999999999,"15min":8.1059999999999999},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:19:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.6970000000000001,"15min":2.5329999999999999},"min":{"1min":0.42999999999999999,"5min":0.376,"15min":0.376},"max":{"1min":6.3920000000000003,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.522,"15min":1.9670000000000001},"min":{"1min":0.46100000000000002,"5min":0.40899999999999997,"15min":0.378},"max":{"1min":4.4409999999999998,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.45200000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024072,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014008,"kb_used_data":1564656,"kb_used_omap":5,"kb_used_meta":449338,"kb_avail":91731144,"statfs":{"total":95995035648,"available":93932691456,"internally_reserved":0,"allocated":1602207744,"data_stored":1601314016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":460122746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:19:09 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":1.798,"15min":1.911},"min":{"1min":0.193,"5min":0.19,"15min":0.19},"max":{"1min":30.733000000000001,"5min":30.733000000000001,"15min":30.733000000000001},"last":0.375},{"interface":"front","average":{"1min":4.6909999999999998,"5min":2.7229999999999999,"15min":2.653},"min":{"1min":0.28499999999999998,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":30.279,"5min":30.279,"15min":30.279},"last":1.1120000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:19:22 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.8660000000000001,"15min":2.4079999999999999},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":17.588999999999999,"5min":17.588999999999999,"15min":17.588999999999999},"last":2.6059999999999999},{"interface":"front","average":{"1min":2.1560000000000001,"5min":2.0529999999999999,"15min":2.4510000000000001},"min":{"1min":0.22900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.8559999999999999,"5min":21.198,"15min":21.198},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":0.61499999999999999,"15min":0.46100000000000002},"min":{"1min":0.311,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":9.0709999999999997},"last":2.3010000000000002},{"interface":"front","average":{"1min":1.5629999999999999,"5min":0.63200000000000001,"15min":0.47699999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.9050000000000002,"5min":7.9050000000000002,"15min":7.9050000000000002},"last":2.3140000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:18:40 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.94899999999999995,"15min":1.653},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":3.9140000000000001,"5min":5.351,"15min":15.835000000000001},"last":0.316},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.889,"15min":2.0859999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":2.5630000000000002,"5min":11.566000000000001,"15min":15.023},"last":2.4420000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:18:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.3919999999999999,"15min":1.853},"min":{"1min":0.39100000000000001,"5min":0.31900000000000001,"15min":0.29299999999999998},"max":{"1min":3.887,"5min":11.222,"15min":15.858000000000001},"last":5.5659999999999998},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.0150000000000001,"15min":2.1139999999999999},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.26700000000000002},"max":{"1min":6.7949999999999999,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:18:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.1699999999999999,"15min":2.1160000000000001},"min":{"1min":0.35599999999999998,"5min":0.33100000000000002,"15min":0.29299999999999998},"max":{"1min":8.8559999999999999,"5min":21.088999999999999,"15min":32.079000000000001},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.4530000000000001,"15min":1.946},"min":{"1min":0.41899999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.6420000000000003,"5min":6.6390000000000002,"15min":15.893000000000001},"last":1.6379999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:18:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":1.925,"15min":2.1930000000000001},"min":{"1min":0.437,"5min":0.39600000000000002,"15min":0.28899999999999998},"max":{"1min":2.4100000000000001,"5min":11.515000000000001,"15min":15.050000000000001},"last":2.653},{"interface":"front","average":{"1min":1.0720000000000001,"5min":2.234,"15min":2.2240000000000002},"min":{"1min":0.40799999999999997,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":6.7519999999999998,"5min":21.135999999999999,"15min":32.055999999999997},"last":0.45500000000000002}]}]},{"osd":2,"up_from":128,"seq":549755813941,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364884,"kb_used_data":1135308,"kb_used_omap":7,"kb_used_meta":229560,"kb_avail":92380268,"statfs":{"total":95995035648,"available":94597394432,"internally_reserved":0,"allocated":1162555392,"data_stored":1161814317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":235069521},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":7.0990000000000002,"5min":4.5129999999999999,"15min":2.7930000000000001},"min":{"1min":0.25600000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":91.515000000000001,"5min":163.40100000000001,"15min":163.40100000000001},"last":4.3650000000000002},{"interface":"front","average":{"1min":7.6879999999999997,"5min":4.8600000000000003,"15min":3.4420000000000002},"min":{"1min":0.24199999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":99.494,"5min":163.363,"15min":163.363},"last":0.63}]},{"osd":1,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":6.4269999999999996,"5min":4.4429999999999996,"15min":2.718},"min":{"1min":0.20599999999999999,"5min":0.188,"15min":0.188},"max":{"1min":61.255000000000003,"5min":222.50299999999999,"15min":222.50299999999999},"last":4.5679999999999996},{"interface":"front","average":{"1min":6.6130000000000004,"5min":4.742,"15min":3.4220000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":61.762,"5min":222.19399999999999,"15min":222.19399999999999},"last":3.8069999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:18:50 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":0.78500000000000003,"15min":0.496},"min":{"1min":0.35299999999999998,"5min":0.245,"15min":0.245},"max":{"1min":13.973000000000001,"5min":13.973000000000001,"15min":13.973000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.64000000000000001,"15min":0.49399999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.3650000000000002,"5min":5.3650000000000002,"15min":5.3650000000000002},"last":0.72899999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.48,"15min":1.9530000000000001},"min":{"1min":0.29099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.2009999999999996,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.839,"5min":1.6359999999999999,"15min":2.214},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.3799999999999999,"5min":15.183,"15min":15.183},"last":0.57599999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.375,"15min":1.6719999999999999},"min":{"1min":0.38300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":5.7199999999999998,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.70899999999999996},{"interface":"front","average":{"1min":2.6400000000000001,"5min":1.9119999999999999,"15min":2.3929999999999998},"min":{"1min":0.54900000000000004,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6210000000000004,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.79100000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":1.736,"15min":2.2010000000000001},"min":{"1min":0.41799999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.4260000000000002,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.72,"5min":1.343,"15min":1.591},"min":{"1min":0.39900000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.726,"5min":7.327,"15min":7.327},"last":4.4080000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:19:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.4199999999999999,"15min":1.631},"min":{"1min":0.46100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.8860000000000001,"5min":6.8860000000000001,"15min":6.8860000000000001},"last":4.4370000000000003},{"interface":"front","average":{"1min":2.0880000000000001,"5min":1.9710000000000001,"15min":2.835},"min":{"1min":0.501,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.5730000000000004,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.755}]}]},{"osd":3,"up_from":149,"seq":639950127139,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3262864,"kb_used_data":3068872,"kb_used_omap":2,"kb_used_meta":193981,"kb_avail":90482288,"statfs":{"total":95995035648,"available":92653862912,"internally_reserved":0,"allocated":3142524928,"data_stored":3141852032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":198636597},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.68899999999999995,"15min":0.64300000000000002},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":5.8929999999999998,"5min":5.8929999999999998,"15min":5.8929999999999998},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.0369999999999999,"15min":0.97699999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":5.7859999999999996,"5min":11.696,"15min":11.696},"last":1.5940000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.125,"15min":1.0069999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.7110000000000003,"5min":11.632,"15min":11.632},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":0.70599999999999996,"15min":0.64200000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.952,"5min":7.952,"15min":7.952},"last":0.46800000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.97799999999999998,"15min":0.95499999999999996},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.0950000000000006,"5min":11.667999999999999,"15min":11.667999999999999},"last":1.51},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.47899999999999998,"15min":0.44},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.1320000000000001,"5min":3.1320000000000001,"15min":3.1320000000000001},"last":0.36399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.68200000000000005,"15min":0.63800000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.8820000000000001,"5min":3.8820000000000001,"15min":3.8820000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.147,"5min":1.1830000000000001,"15min":1.1890000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.3039999999999998,"5min":11.742000000000001,"15min":11.742000000000001},"last":1.157}]},{"osd":5,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":1.238,"15min":1.228},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.218,"5min":11.772,"15min":11.772},"last":1.667},{"interface":"front","average":{"1min":1.157,"5min":1.2130000000000001,"15min":1.2230000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.577,"5min":11.598000000000001,"15min":11.598000000000001},"last":1.54}]},{"osd":6,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.2230000000000001,"15min":1.204},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.1760000000000002,"5min":11.798,"15min":11.798},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.147,"5min":1.1870000000000001,"15min":1.1930000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":3.9169999999999998,"5min":11.552,"15min":11.552},"last":1.117}]},{"osd":7,"last update":"Mon Oct 9 06:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.69299999999999995,"15min":0.65800000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.048,"5min":3.048,"15min":3.048},"last":1.032},{"interface":"front","average":{"1min":1.381,"5min":1.2490000000000001,"15min":1.2270000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.375,"5min":11.823,"15min":11.823},"last":1.2150000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019175,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802636,"kb_used_data":1208772,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":91942516,"statfs":{"total":95995035648,"available":94149136384,"internally_reserved":0,"allocated":1237782528,"data_stored":1237172375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:38 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.486,"15min":1.3819999999999999},"min":{"1min":0.22900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.8520000000000003,"5min":8.8520000000000003,"15min":15.484999999999999},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.8460000000000001,"15min":1.6990000000000001},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.23999999999999999},"max":{"1min":8.8879999999999999,"5min":11.234,"15min":15.289},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:19:17 2023","interfaces":[{"interface":"back","average":{"1min":6.4299999999999997,"5min":2.3479999999999999,"15min":1.8620000000000001},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":91.849999999999994,"5min":91.849999999999994,"15min":91.849999999999994},"last":0.78600000000000003},{"interface":"front","average":{"1min":6.2869999999999999,"5min":2.2759999999999998,"15min":1.8640000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":91.885000000000005,"5min":91.885000000000005,"15min":91.885000000000005},"last":0.28199999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:19:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.486,"15min":1.766},"min":{"1min":0.34000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.577,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":2.528},{"interface":"front","average":{"1min":2.6000000000000001,"5min":1.6399999999999999,"15min":1.891},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":15.894,"5min":15.894,"15min":15.894},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:18:46 2023","interfaces":[{"interface":"back","average":{"1min":5.6159999999999997,"5min":1.573,"15min":0.89900000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":91.697000000000003,"5min":91.697000000000003,"15min":91.697000000000003},"last":0.75},{"interface":"front","average":{"1min":5.8899999999999997,"5min":1.675,"15min":0.97299999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":94.356999999999999,"5min":94.356999999999999,"15min":94.356999999999999},"last":4.1210000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:19:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.1739999999999999,"15min":1.026},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.22500000000000001},"max":{"1min":2.7810000000000001,"5min":9.6790000000000003,"15min":11.130000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.3460000000000001,"15min":1.2929999999999999},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":5.9039999999999999,"5min":11.282,"15min":11.282},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:18:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.097,"15min":0.98699999999999999},"min":{"1min":0.312,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.9689999999999999,"5min":9.7680000000000007,"15min":9.7680000000000007},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.81200000000000006,"5min":1.5289999999999999,"15min":1.226},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.25},"max":{"1min":3.2789999999999999,"5min":14.202999999999999,"15min":14.202999999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.0369999999999999,"15min":1.0349999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.27900000000000003},"max":{"1min":3.3159999999999998,"5min":6.6849999999999996,"15min":8.5020000000000007},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.3680000000000001,"15min":1.1619999999999999},"min":{"1min":0.39000000000000001,"5min":0.34100000000000003,"15min":0.32800000000000001},"max":{"1min":3.294,"5min":12.965999999999999,"15min":12.965999999999999},"last":0.436}]}]},{"osd":5,"up_from":38,"seq":163208757540,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4342588,"kb_used_data":3449076,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":89402564,"statfs":{"total":95995035648,"available":91548225536,"internally_reserved":0,"allocated":3531853824,"data_stored":3531255741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:18:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.4930000000000001,"15min":1.607},"min":{"1min":0.25900000000000001,"5min":0.215,"15min":0.215},"max":{"1min":3.0529999999999999,"5min":8.5139999999999993,"15min":18.637},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.7729999999999999,"15min":1.54},"min":{"1min":0.32800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.0369999999999999,"5min":9.9939999999999998,"15min":9.9939999999999998},"last":0.60499999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:19:19 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.417,"15min":2.137},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.4820000000000002,"5min":8.8170000000000002,"15min":11.292},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.7110000000000001,"15min":2.0230000000000001},"min":{"1min":0.33000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.1239999999999997,"5min":7.9539999999999997,"15min":7.9539999999999997},"last":0.52900000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:19:19 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":1.6759999999999999,"15min":2.2170000000000001},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":7.9409999999999998,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.508,"15min":2.0710000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.7779999999999996,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":0.57599999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:18:48 2023","interfaces":[{"interface":"back","average":{"1min":18.068000000000001,"5min":4.1130000000000004,"15min":1.7869999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":190.411,"5min":190.411,"15min":190.411},"last":0.85899999999999999},{"interface":"front","average":{"1min":18.175000000000001,"5min":4.1390000000000002,"15min":1.7989999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":190.70699999999999,"5min":190.70699999999999,"15min":190.70699999999999},"last":0.59099999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:19:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":1.4019999999999999,"15min":1.143},"min":{"1min":0.27400000000000002,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":2.355,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.008,"15min":1.0029999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.25900000000000001},"max":{"1min":3.1549999999999998,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":0.55700000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:18:31 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.4059999999999999,"15min":1.171},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":1.7290000000000001,"5min":8.8149999999999995,"15min":8.8149999999999995},"last":1.5009999999999999},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.3759999999999999,"15min":1.0720000000000001},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":2.6219999999999999,"5min":7.3479999999999999,"15min":7.3479999999999999},"last":0.496}]},{"osd":7,"last update":"Mon Oct 9 06:18:52 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.355,"15min":1.087},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":3.5139999999999998,"5min":8.8409999999999993,"15min":8.8409999999999993},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.212,"15min":1.0429999999999999},"min":{"1min":0.371,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":3.077,"5min":7.7809999999999997,"15min":9.4580000000000002},"last":0.625}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1600716800,"data_stored":1600716800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3415932928,"data_stored":3415932928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1163460608,"data_stored":1163460608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3142385664,"data_stored":3142385664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1237712896,"data_stored":1237712896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3539009536,"data_stored":3539009536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1136918528,"data_stored":1136918528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3052142592,"data_stored":3052142592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:19:30.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T06:19:30.414 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:30.505 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:30.610 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:30.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:30 smithi107 ceph-mon[212750]: pgmap v586: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s; 5015/276339 objects degraded (1.815%); 348/276339 objects misplaced (0.126%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:19:30.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:30 smithi107 ceph-mon[212750]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:30.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:30 smithi107 ceph-mon[220921]: pgmap v586: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s; 5015/276339 objects degraded (1.815%); 348/276339 objects misplaced (0.126%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:19:30.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:30 smithi107 ceph-mon[220921]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:30.683 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:30.707 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:30.836 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:30.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:30.889 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 15 94503 94488 69.4218 45 0.0420998 0.0143959 2023-10-09T06:19:30.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:30 smithi150 ceph-mon[194178]: pgmap v586: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s; 5015/276339 objects degraded (1.815%); 348/276339 objects misplaced (0.126%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:19:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:30 smithi150 ceph-mon[194178]: from='client.35457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:19:30.945 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:30.993 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:31.233 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:31.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:31.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:31.338 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:31.400 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:31.511 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:31.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:31.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:31.541 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:19:31.625 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:31.649 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2535372028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2535372028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.652 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:31.695 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:31.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:31.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:95066 9.f 9:ffc7331b:::benchmark_data_smithi107_300016_object95065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:31.779246+0000", 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.0043486949999999996, 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043844260000000003, 2023-10-09T06:19:31.794 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 95066 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779246+0000", 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:31.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779246+0000", 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779244+0000", 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779318+0000", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3627999999999994e-05 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779336+0000", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8173e-05 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779346+0000", 2023-10-09T06:19:31.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7580000000000004e-06 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779458+0000", 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011219599999999999 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779621+0000", 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016276499999999999 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.779805+0000", 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018425599999999999 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783169+0000", 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033644930000000001 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45275.0:95065 9.8 e174/152 9:12c5c3fd:::benchmark_data_smithi107_300016_object95064:head v 174'5982, mlcod=174'5980)", 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:31.783498+0000", 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 9.6812999999999996e-05, 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022795199999999999, 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783498+0000", 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:31.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783498+0000", 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783497+0000", 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:19:31.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783555+0000", 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8393000000000001e-05 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783566+0000", 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.115e-05 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783571+0000", 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6959999999999996e-06 2023-10-09T06:19:31.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783620+0000", 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8851999999999999e-05 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:31.783645+0000", 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5148999999999999e-05 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:31.803 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:19:31.804 INFO:teuthology.orchestra.run.smithi150.stdout: ], 2023-10-09T06:19:31.804 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 2 2023-10-09T06:19:31.804 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:31.804 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:31.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2535372028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:31.859 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:31.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 95165 95149 69.0919 41.3125 0.0150895 0.0144682 2023-10-09T06:19:32.171 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:32.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:32.278 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:32.301 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:32] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2023-10-09T06:19:32.304 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 5015/278025 objects degraded (1.804%); 348/278025 objects misplaced (0.125%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: failed to encode map e175 with expected crc 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:19:32.611 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5015/278025 objects degraded (1.804%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 5015/278025 objects degraded (1.804%); 348/278025 objects misplaced (0.125%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: failed to encode map e175 with expected crc 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:32.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5015/278025 objects degraded (1.804%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:32.743 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:32.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:32.763 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:19:32.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: pgmap v587: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 5015/278025 objects degraded (1.804%); 348/278025 objects misplaced (0.125%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:19:32.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:32.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: failed to encode map e175 with expected crc 2023-10-09T06:19:32.826 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:32.827 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T06:19:32.827 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:32.827 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:32.827 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5015/278025 objects degraded (1.804%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:32.838 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:32.851 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:32.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 95885 95869 68.8151 45 0.0160199 0.0145264 2023-10-09T06:19:32.906 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:33.262 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:33.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:33.342 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:33.378 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:33.380 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:33.391 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:33.410 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:33.441 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:33.449 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:33.483 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:33.540 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:33.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:33.544 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/274820231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:33.548 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:33.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:33.641 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:33.643 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.643 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.643 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:33.644 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:33.698 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:33.704 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/274820231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:33.772 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:33.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:33.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:33.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 96628 96612 68.5609 46.4375 0.00370025 0.0145496 2023-10-09T06:19:33.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/274820231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:33.904 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:34.880 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:19:34.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 96846 96830 67.9441 13.625 0.0119685 0.014713 2023-10-09T06:19:34.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[212750]: pgmap v589: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 534 op/s; 5015/280608 objects degraded (1.787%); 348/280608 objects misplaced (0.124%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:19:34.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[212750]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:19:34.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[212750]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:34 smithi150 ceph-mon[194178]: pgmap v589: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 534 op/s; 5015/280608 objects degraded (1.787%); 348/280608 objects misplaced (0.124%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:34 smithi150 ceph-mon[194178]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:34 smithi150 ceph-mon[194178]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:34 smithi150 ceph-mon[194178]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:34 smithi150 ceph-mon[194178]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[220921]: pgmap v589: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 33 MiB/s wr, 534 op/s; 5015/280608 objects degraded (1.787%); 348/280608 objects misplaced (0.124%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:19:34.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[220921]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[220921]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T06:19:34.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[220921]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:34 smithi107 ceph-mon[220921]: failed to encode map e176 with expected crc 2023-10-09T06:19:34.932 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:34.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:35.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:35.097 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.193 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:35.303 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:35.384 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.486 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:35.486 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:35.589 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:35.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:19:35.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[220921]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:35 smithi107 ceph-mon[212750]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.683 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:35.702 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:35.757 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:35.829 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.831 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:35 smithi150 ceph-mon[194178]: failed to encode map e177 with expected crc 2023-10-09T06:19:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:35.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:35.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 90 15 97569 97554 67.6921 45.25 0.00672847 0.0147557 2023-10-09T06:19:35.933 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:35.963 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:36.112 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:36.143 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.291 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:36.336 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:19:36.394 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:36.421 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.465 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:36.568 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:36.581 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 1 pgp_num to 1 2023-10-09T06:19:36.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.641 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:36.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:36.696 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:36.746 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:36.773 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:36.819 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:36.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:36 smithi150 ceph-mon[194178]: pgmap v592: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4142/288390 objects degraded (1.436%); 691/288390 objects misplaced (0.240%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:36.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:19:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:36.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.1726 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 97569 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 67.6266 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 42.2247 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 210.625 2023-10-09T06:19:36.894 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1082 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 675.602 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3370 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0147813 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0413763 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.27436 2023-10-09T06:19:36.895 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00127393 2023-10-09T06:19:36.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:36 smithi107 ceph-mon[212750]: pgmap v592: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4142/288390 objects degraded (1.436%); 691/288390 objects misplaced (0.240%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:19:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:36 smithi107 ceph-mon[220921]: pgmap v592: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 669 op/s; 4142/288390 objects degraded (1.436%); 691/288390 objects misplaced (0.240%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T06:19:36.901 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:19:36.923 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:36.980 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:37.053 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:37.695 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:37.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:37.802 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:37.810 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.810 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.810 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T06:19:37.810 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4142/288390 objects degraded (1.436%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T06:19:37.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:19:37.812 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.812 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4142/288390 objects degraded (1.436%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1315278691' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: failed to encode map e178 with expected crc 2023-10-09T06:19:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4142/288390 objects degraded (1.436%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:37.906 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:38.102 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:38.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:38.181 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:38.211 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:38.281 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:38.482 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:38.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:38.573 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:38.574 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:38.590 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:38.674 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:38.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:38 smithi107 ceph-mon[212750]: pgmap v594: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 4142/289449 objects degraded (1.431%); 691/289449 objects misplaced (0.239%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:19:38.839 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:38 smithi107 ceph-mon[220921]: pgmap v594: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 4142/289449 objects degraded (1.431%); 691/289449 objects misplaced (0.239%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:19:38.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:38.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:38 smithi150 ceph-mon[194178]: pgmap v594: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 621 op/s; 4142/289449 objects degraded (1.431%); 691/289449 objects misplaced (0.239%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:19:38.960 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:39.045 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:39.377 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:39.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:39.488 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:39.545 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:39.554 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:19:39.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:39.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:39.662 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:19:39.679 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:39.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:39.791 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:39.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:19:39.872 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:39.976 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:39.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:40.087 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:40.114 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:40.136 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:40.159 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:40.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:40.268 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:40.331 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:40.477 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:40.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:40.592 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:40.660 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:40.794 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:40.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 41 MiB/s wr, 781 op/s; 4142/290028 objects degraded (1.428%); 691/290028 objects misplaced (0.238%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.819 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:40 smithi150 ceph-mon[194178]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.899 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 41 MiB/s wr, 781 op/s; 4142/290028 objects degraded (1.428%); 691/290028 objects misplaced (0.238%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[212750]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 170 B/s rd, 41 MiB/s wr, 781 op/s; 4142/290028 objects degraded (1.428%); 691/290028 objects misplaced (0.238%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1477059272' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:40 smithi107 ceph-mon[220921]: failed to encode map e179 with expected crc 2023-10-09T06:19:40.960 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:41.039 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:41.084 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:41.088 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:41.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:41.122 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:41.178 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:41.197 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:41.214 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:41.251 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:41.263 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:41.293 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:41.328 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:41.513 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:41.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:41.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:41.627 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:41.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3946650915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3946650915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:41.719 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:41.858 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:41.861 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3946650915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:41.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:41.963 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:41.992 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.138 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:42.247 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:42.279 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:42.279 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:19:42.280 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:19:42.280 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:42.289 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:42] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2023-10-09T06:19:42.305 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.397 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:42.509 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:42.562 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.610 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:42.611 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:19:42.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:42.718 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:42.742 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:42 smithi150 ceph-mon[194178]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 160 B/s rd, 20 MiB/s wr, 516 op/s; 4142/289056 objects degraded (1.433%); 162/289056 objects misplaced (0.056%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T06:19:42.742 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4142/289056 objects degraded (1.433%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:42.745 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:42.789 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:42.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:42.842 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:19:42.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:19:42.892 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:42.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:42 smithi107 ceph-mon[212750]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 160 B/s rd, 20 MiB/s wr, 516 op/s; 4142/289056 objects degraded (1.433%); 162/289056 objects misplaced (0.056%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T06:19:42.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4142/289056 objects degraded (1.433%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:42.894 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:42 smithi107 ceph-mon[220921]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 81 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 160 B/s rd, 20 MiB/s wr, 516 op/s; 4142/289056 objects degraded (1.433%); 162/289056 objects misplaced (0.056%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T06:19:42.894 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4142/289056 objects degraded (1.433%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:42.942 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.158 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:43.289 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:43.296 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:19:43.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:43.428 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.479 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:43.583 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:43.614 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.715 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:43.755 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:43.818 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:43.844 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:43.886 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:43.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:43.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 457 op/s; 3288/286191 objects degraded (1.149%); 994/286191 objects misplaced (0.347%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:19:43.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:19:43.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T06:19:43.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T06:19:43.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T06:19:43.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T06:19:43.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[212750]: failed to encode map e180 with expected crc 2023-10-09T06:19:43.989 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:44.023 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 457 op/s; 3288/286191 objects degraded (1.149%); 994/286191 objects misplaced (0.347%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:43 smithi150 ceph-mon[194178]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.296 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:44.314 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:44.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:44.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 457 op/s; 3288/286191 objects degraded (1.149%); 994/286191 objects misplaced (0.347%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:19:44.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T06:19:44.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:43 smithi107 ceph-mon[220921]: failed to encode map e180 with expected crc 2023-10-09T06:19:44.348 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:19:44.423 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:44.505 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:44.651 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:44.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:44.759 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:44.810 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[212750]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:19:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[212750]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[220921]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[220921]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[220921]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:19:45.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[220921]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:44 smithi107 ceph-mon[220921]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.203 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:45.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:45.308 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:45.335 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:45.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:44 smithi150 ceph-mon[194178]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:44 smithi150 ceph-mon[194178]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:44 smithi150 ceph-mon[194178]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T06:19:45.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:44 smithi150 ceph-mon[194178]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:44 smithi150 ceph-mon[194178]: failed to encode map e181 with expected crc 2023-10-09T06:19:45.383 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:45.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:45.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:45.486 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:45.540 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:45.803 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:45.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:45.911 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:45.954 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:45.962 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:45.962 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:19:45.962 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:19:45.963 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:19:45.963 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:45.987 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:46.012 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:19:46.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:45 smithi107 ceph-mon[212750]: pgmap v601: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 1.5 MiB/s wr, 738 op/s; 3288/277578 objects degraded (1.185%); 832/277578 objects misplaced (0.300%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:19:46.045 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:46.045 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:45 smithi107 ceph-mon[220921]: pgmap v601: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 1.5 MiB/s wr, 738 op/s; 3288/277578 objects degraded (1.185%); 832/277578 objects misplaced (0.300%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:19:46.045 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:46.296 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:46.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:46.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:46.308 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:45 smithi150 ceph-mon[194178]: pgmap v601: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 1.5 MiB/s wr, 738 op/s; 3288/277578 objects degraded (1.185%); 832/277578 objects misplaced (0.300%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:19:46.308 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:19:46.406 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:46.441 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:46.982 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:46.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:46.993 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:19:47.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:47.168 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:47.300 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:47.314 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:47.314 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.314 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:47.314 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:47.315 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:46 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3288/277578 objects degraded (1.185%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:47.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.320 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3288/277578 objects degraded (1.185%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3927011406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: failed to encode map e182 with expected crc 2023-10-09T06:19:47.322 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:19:47.322 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:19:47.322 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:46 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3288/277578 objects degraded (1.185%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:47.419 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:47.523 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:47.628 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:47.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:47.681 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:47.735 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:47.797 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:47.971 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:47.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:48.080 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:48.142 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.255 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:48.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:48.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:48.362 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:48.421 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.488 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:48.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:48.592 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:48.605 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:19:48.662 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:48 smithi107 ceph-mon[212750]: pgmap v603: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3288/276387 objects degraded (1.190%); 832/276387 objects misplaced (0.301%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:19:48.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:19:48.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:48 smithi107 ceph-mon[220921]: pgmap v603: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3288/276387 objects degraded (1.190%); 832/276387 objects misplaced (0.301%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:19:48.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:19:48.763 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:48.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:48.774 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:48 smithi150 ceph-mon[194178]: pgmap v603: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3288/276387 objects degraded (1.190%); 832/276387 objects misplaced (0.301%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:19:48.774 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:19:48.875 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:48.933 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:48.938 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:48.958 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:48.977 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:48.978 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:48.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:48.997 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:49.015 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:49.059 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:49.064 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:19:49.081 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:49.096 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:49.140 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.142 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.143 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:19:49.155 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:49.455 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:49.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:49.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:49.577 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:49.762 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2579011364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:49.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:19:49.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2579011364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:49.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/204116717' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: failed to encode map e183 with expected crc 2023-10-09T06:19:49.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2579011364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:50.116 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:50.189 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:50.189 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:50.211 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:19:50.224 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:50.287 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:50.429 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:50.430 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:50.537 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:50.616 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:50.776 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:50.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:50.880 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:50.898 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:50 smithi107 ceph-mon[212750]: pgmap v605: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3288/269460 objects degraded (1.220%); 301/269460 objects misplaced (0.112%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:19:50.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:50 smithi107 ceph-mon[220921]: pgmap v605: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3288/269460 objects degraded (1.220%); 301/269460 objects misplaced (0.112%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:19:50.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:50 smithi150 ceph-mon[194178]: pgmap v605: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3288/269460 objects degraded (1.220%); 301/269460 objects misplaced (0.112%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:19:50.934 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.182 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:51.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:51.218 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:19:51.294 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:51.322 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.363 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:51.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:51.466 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:51.523 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:51.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:51.743 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:51.767 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:51.796 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:51.843 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:51.880 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:51.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:51.988 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.017 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:52.028 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:19:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:19:52] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2023-10-09T06:19:52.115 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:52.220 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:52.247 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:52.277 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:19:52.495 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:52.600 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:52.632 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:52.693 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:52.800 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:52.825 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:52.867 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: pgmap v606: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3288/267312 objects degraded (1.230%); 301/267312 objects misplaced (0.113%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3288/267312 objects degraded (1.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:52.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: pgmap v606: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3288/267312 objects degraded (1.230%); 301/267312 objects misplaced (0.113%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3288/267312 objects degraded (1.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:52.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: pgmap v606: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 3288/267312 objects degraded (1.230%); 301/267312 objects misplaced (0.113%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: failed to encode map e184 with expected crc 2023-10-09T06:19:52.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3288/267312 objects degraded (1.230%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:52.971 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:53.054 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:53.162 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:53.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:53.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:53.350 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:53.390 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:53.475 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:53.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:53.580 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:19:53.633 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3462429535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3462429535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.670 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:19:53.753 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:19:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:53.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:19:53.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:19:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T06:19:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: failed to encode map e185 with expected crc 2023-10-09T06:19:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3462429535' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:19:53.954 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:54.011 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:19:54.011 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:19:54.673 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:19:54.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: pgmap v609: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 3287/264777 objects degraded (1.241%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T06:19:54.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:54.983 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: failed to encode map e186 with expected crc 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: failed to encode map e186 with expected crc 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: failed to encode map e186 with expected crc 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: failed to encode map e186 with expected crc 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2531504369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:54.984 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:55.003 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:19:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: pgmap v609: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 3287/264777 objects degraded (1.241%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T06:19:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2531504369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: pgmap v609: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 3287/264777 objects degraded (1.241%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: failed to encode map e186 with expected crc 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2531504369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:19:55.363 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:19:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:55.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:19:55.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:55.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:55.652 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:55.915 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[212750]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.917 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:55 smithi107 ceph-mon[220921]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.947 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:19:55.947 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T06:19:55.947 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:55.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:55 smithi150 ceph-mon[194178]: failed to encode map e187 with expected crc 2023-10-09T06:19:56.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:19:56.021 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:56.132 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:56.217 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:56.408 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:56.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:56.514 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:56.567 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:56.626 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:19:56.677 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:19:56.679 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:56.726 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:19:56.755 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:56.767 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:19:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:56.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:56.806 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:19:56.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:19:56.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:19:56.897 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:19:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:56 smithi150 ceph-mon[194178]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2463/258216 objects degraded (0.954%); 549/258216 objects misplaced (0.213%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T06:19:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/909640650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:56.933 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:19:56.950 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:56 smithi107 ceph-mon[212750]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2463/258216 objects degraded (0.954%); 549/258216 objects misplaced (0.213%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T06:19:56.950 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/909640650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:56.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:56 smithi107 ceph-mon[220921]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2463/258216 objects degraded (0.954%); 549/258216 objects misplaced (0.213%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T06:19:56.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/909640650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:19:56.960 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:57.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:19:57.492 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:19:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:57.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:19:57.600 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:19:57.641 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:57.853 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:19:57.853 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:19:57.853 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:19:57.853 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:19:57.853 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:19:57.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2463/258216 objects degraded (0.954%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:57.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2463/258216 objects degraded (0.954%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:57.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2463/258216 objects degraded (0.954%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:19:58.485 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:19:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:58.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:19:58.596 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:19:58.668 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:58.749 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:19:58.783 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:19:58.805 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:19:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:58.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:19:58.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:58 smithi107 ceph-mon[212750]: pgmap v613: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 2463/257025 objects degraded (0.958%); 549/257025 objects misplaced (0.214%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:19:58.838 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:58 smithi107 ceph-mon[220921]: pgmap v613: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 2463/257025 objects degraded (0.958%); 549/257025 objects misplaced (0.214%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:19:58.916 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:58.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:58 smithi150 ceph-mon[194178]: pgmap v613: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 2463/257025 objects degraded (0.958%); 549/257025 objects misplaced (0.214%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T06:19:59.011 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.114 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:19:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:19:59.225 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:19:59.290 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.442 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:19:59.492 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:19:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:19:59.605 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:19:59.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.709 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:19:59.745 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:19:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:19:59.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:19:59.849 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:19:59.876 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:19:59.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1602299944' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:19:59.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:19:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:19:59.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1602299944' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:19:59.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:19:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1602299944' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:19:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:20:00.387 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:00.491 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:00.518 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:00.666 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:00.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:00.767 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:00.830 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: pgmap v614: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 2463/249300 objects degraded (0.988%); 101/249300 objects misplaced (0.041%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: failed to encode map e188 with expected crc 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: failed to encode map e188 with expected crc 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: failed to encode map e188 with expected crc 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: [WRN] PG_DEGRADED: Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T06:20:00.875 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-10-09T06:20:00.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T06:20:00.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:00 smithi150 ceph-mon[194178]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-10-09T06:20:01.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: pgmap v614: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 2463/249300 objects degraded (0.988%); 101/249300 objects misplaced (0.041%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:20:01.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:01.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:20:01.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: [WRN] PG_DEGRADED: Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[212750]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-10-09T06:20:01.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: pgmap v614: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 2463/249300 objects degraded (0.988%); 101/249300 objects misplaced (0.041%); 10 MiB/s, 161 objects/s recovering 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: failed to encode map e188 with expected crc 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: [WRN] PG_DEGRADED: Degraded data redundancy: 2463/249300 objects degraded (0.988%), 4 pgs degraded 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [0,6] 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: pg 9.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: pg 9.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T06:20:01.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:00 smithi107 ceph-mon[220921]: pg 9.9 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2023-10-09T06:20:01.113 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:01.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:01.133 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:20:01.235 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:01.305 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.453 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:01.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:01.532 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:20:01.532 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204169 7.5 7:aceb55b0:::benchmark_data_smithi107_237467_object101280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:00.998621+0000", 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.531108426, 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0434366239999999, 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.533 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204169 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998621+0000", 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998621+0000", 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998619+0000", 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998626+0000", 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6359999999999994e-06 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998643+0000", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7059e-05 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998651+0000", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1620000000000002e-06 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998705+0000", 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3876999999999999e-05 2023-10-09T06:20:01.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:00.998718+0000", 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3156000000000001e-05 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.005561+0000", 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0068437299999999 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.005670+0000", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108826 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.005748+0000", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8114999999999993e-05 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022661+0000", 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016912994000000001 2023-10-09T06:20:01.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.041973+0000", 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019311354999999999 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042000+0000", 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6934e-05 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042032+0000", 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2384e-05 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042057+0000", 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5245e-05 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204180 7.5 7:a9d5c092:::benchmark_data_smithi107_237467_object101291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.017240+0000", 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.51248947200003, 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0249817290000001, 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204180 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017240+0000", 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017240+0000", 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017238+0000", 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017245+0000", 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4289999999999998e-06 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.543 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017261+0000", 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6325e-05 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017269+0000", 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8450000000000003e-06 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017345+0000", 2023-10-09T06:20:01.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6237999999999998e-05 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.017356+0000", 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1066999999999999e-05 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.008881+0000", 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99152491300000001 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.008954+0000", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2630000000000001e-05 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.009028+0000", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4682000000000001e-05 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022671+0000", 2023-10-09T06:20:01.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013642331000000001 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042095+0000", 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019424231 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042171+0000", 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6661000000000003e-05 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.548 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042199+0000", 2023-10-09T06:20:01.548 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7804999999999999e-05 2023-10-09T06:20:01.548 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.548 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.548 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042221+0000", 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2001000000000002e-05 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204185 7.5 7:aa0bdde3:::benchmark_data_smithi107_237467_object101296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.026543+0000", 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.50318648000001, 2023-10-09T06:20:01.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0157804880000001, 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204185 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026543+0000", 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026543+0000", 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026541+0000", 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026547+0000", 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9909999999999999e-06 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026565+0000", 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7249999999999999e-05 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026572+0000", 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6359999999999994e-06 2023-10-09T06:20:01.553 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026629+0000", 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6484000000000003e-05 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.026637+0000", 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3659999999999992e-06 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.554 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.011644+0000", 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98500704800000005 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.011712+0000", 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8057000000000002e-05 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.011782+0000", 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9986000000000004e-05 2023-10-09T06:20:01.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022679+0000", 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010896553 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042127+0000", 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019448674999999999 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042277+0000", 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014965000000000001 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042303+0000", 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6089999999999999e-05 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042323+0000", 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9870000000000001e-05 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.557 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204213 7.5 7:a182cf1c:::benchmark_data_smithi107_237467_object101324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.063493+0000", 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.46623642600002, 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97896457599999998, 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.558 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.559 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204213 2023-10-09T06:20:01.559 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.559 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.559 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063493+0000", 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063493+0000", 2023-10-09T06:20:01.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063491+0000", 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063498+0000", 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7429999999999996e-06 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063510+0000", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2534999999999999e-05 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063516+0000", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5149999999999997e-06 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063566+0000", 2023-10-09T06:20:01.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0198000000000003e-05 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.063575+0000", 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7590000000000003e-06 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.013963+0000", 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95038818199999997 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.014065+0000", 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010171299999999999 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.014203+0000", 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000138703 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.027142+0000", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012938777 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042155+0000", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015013291999999999 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042402+0000", 2023-10-09T06:20:01.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000246388 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042427+0000", 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5040000000000001e-05 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042457+0000", 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0465000000000001e-05 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.567 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204250 7.5 7:a58e796a:::benchmark_data_smithi107_237467_object101361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.122333+0000", 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.40739575999999, 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92036412300000003, 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.568 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204250 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122333+0000", 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122333+0000", 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.569 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122332+0000", 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122339+0000", 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6990000000000003e-06 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.570 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122356+0000", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6917000000000001e-05 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122363+0000", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5749999999999997e-06 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122421+0000", 2023-10-09T06:20:01.571 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8341000000000003e-05 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.122430+0000", 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0019999999999995e-06 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.017943+0000", 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89551231099999995 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.572 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.018017+0000", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4659000000000005e-05 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.018092+0000", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5217999999999997e-05 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.027172+0000", 2023-10-09T06:20:01.573 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090792540000000001 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042354+0000", 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015182040000000001 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042604+0000", 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00025028399999999998 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.574 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042656+0000", 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1795999999999997e-05 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042697+0000", 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1579999999999998e-05 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.575 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204254 7.5 7:a877949d:::benchmark_data_smithi107_237467_object101365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.127549+0000", 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.40218003699999, 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.915705985, 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204254 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.576 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127549+0000", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127549+0000", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127548+0000", 2023-10-09T06:20:01.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127553+0000", 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5010000000000004e-06 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127568+0000", 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4622e-05 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127573+0000", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.502e-06 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127623+0000", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9604999999999997e-05 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.127631+0000", 2023-10-09T06:20:01.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3010000000000007e-06 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.020388+0000", 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89275659200000002 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.020457+0000", 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8779e-05 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.020526+0000", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9562999999999998e-05 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.030886+0000", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010359319000000001 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042380+0000", 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011494315 2023-10-09T06:20:01.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.043169+0000", 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00078929900000000003 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.043216+0000", 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6501000000000001e-05 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.043255+0000", 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9246999999999999e-05 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204261 7.5 7:a9cfed10:::benchmark_data_smithi107_237467_object101372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.135527+0000", 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.394201786, 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.90879202699999995, 2023-10-09T06:20:01.583 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204261 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135527+0000", 2023-10-09T06:20:01.584 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135527+0000", 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135526+0000", 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135532+0000", 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8170000000000004e-06 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135547+0000", 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.509e-05 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135555+0000", 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7400000000000004e-06 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135614+0000", 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9138000000000002e-05 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.135621+0000", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0670000000000003e-06 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022711+0000", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88709044999999997 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022794+0000", 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2107000000000005e-05 2023-10-09T06:20:01.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.022863+0000", 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9863999999999994e-05 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.030893+0000", 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080292899999999997 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042490+0000", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011597066 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044236+0000", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017462689999999999 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044282+0000", 2023-10-09T06:20:01.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6156000000000003e-05 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044319+0000", 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7110000000000002e-05 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204265 7.5 7:a39574f4:::benchmark_data_smithi107_237467_object101376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.145583+0000", 2023-10-09T06:20:01.591 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.38414588900002, 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89887442900000003, 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204265 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145583+0000", 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145583+0000", 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145582+0000", 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145588+0000", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4060000000000002e-06 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145606+0000", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7244e-05 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145614+0000", 2023-10-09T06:20:01.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1969999999999996e-06 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145682+0000", 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8666999999999998e-05 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.145694+0000", 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1819e-05 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.025137+0000", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.879442528 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.025208+0000", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0976000000000006e-05 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.025275+0000", 2023-10-09T06:20:01.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7537000000000001e-05 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.034429+0000", 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0091539280000000004 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042514+0000", 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080849990000000007 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044369+0000", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0018545860000000001 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044420+0000", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1431999999999998e-05 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.044458+0000", 2023-10-09T06:20:01.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7311000000000002e-05 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204299 7.5 7:a436d8a2:::benchmark_data_smithi107_237467_object101410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.202458+0000", 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.32727064400001, 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.845133735, 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204299 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202458+0000", 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202458+0000", 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202457+0000", 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202464+0000", 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4930000000000001e-06 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202480+0000", 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5620999999999998e-05 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202486+0000", 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.37e-06 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202551+0000", 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4567999999999996e-05 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.202562+0000", 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0900999999999999e-05 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.027193+0000", 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.82463139600000002 2023-10-09T06:20:01.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.027263+0000", 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9326999999999995e-05 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.027330+0000", 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7584999999999998e-05 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.604 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.034457+0000", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071270780000000002 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042541+0000", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080834050000000001 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047525+0000", 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049843429999999996 2023-10-09T06:20:01.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047567+0000", 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2128000000000001e-05 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047592+0000", 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5001e-05 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.606 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.607 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34272.0:204327 7.5 7:a8f606a9:::benchmark_data_smithi107_237467_object101438:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T06:20:01.607 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:01.250206+0000", 2023-10-09T06:20:01.607 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 480.27952327000003, 2023-10-09T06:20:01.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.79748240100000001, 2023-10-09T06:20:01.608 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34272", 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 204327 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250206+0000", 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250206+0000", 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250204+0000", 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250213+0000", 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0569999999999999e-06 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250232+0000", 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8442999999999999e-05 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250241+0000", 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3700000000000001e-06 2023-10-09T06:20:01.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250317+0000", 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5884999999999996e-05 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:01.250327+0000", 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5370000000000003e-06 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.029097+0000", 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77877069799999998 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.029160+0000", 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2421000000000004e-05 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.029230+0000", 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0059000000000001e-05 2023-10-09T06:20:01.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.038083+0000", 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0088531340000000004 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.042578+0000", 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0044952880000000001 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047626+0000", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0050475279999999999 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047657+0000", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0857e-05 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:02.047688+0000", 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1644999999999999e-05 2023-10-09T06:20:01.615 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90926 9.d 9:b3ca72e7:::benchmark_data_smithi107_300016_object90925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.388649819999998, 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98166545400000005, 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.616 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90926 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.617 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141077+0000", 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141118+0000", 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0886999999999997e-05 2023-10-09T06:20:01.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141136+0000", 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7960000000000001e-05 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141144+0000", 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3710000000000002e-06 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141226+0000", 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2038000000000003e-05 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141235+0000", 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7119999999999995e-06 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091572+0000", 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95033658300000001 2023-10-09T06:20:01.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091680+0000", 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108069 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091823+0000", 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014296400000000001 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.621 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.100838+0000", 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090148290000000002 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122479+0000", 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.021641525000000002 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.622 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122680+0000", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020095599999999999 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122712+0000", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1483999999999999e-05 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122745+0000", 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2886000000000003e-05 2023-10-09T06:20:01.623 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90932 9.d 9:bfbd009d:::benchmark_data_smithi107_300016_object90931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.387108017000003, 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98438990199999998, 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.624 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90932 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.625 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142620+0000", 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142659+0000", 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8958999999999997e-05 2023-10-09T06:20:01.626 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142675+0000", 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6461e-05 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142682+0000", 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4239999999999998e-06 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.627 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142740+0000", 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7855000000000001e-05 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142751+0000", 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1357e-05 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094465+0000", 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95171417800000002 2023-10-09T06:20:01.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094549+0000", 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4246999999999995e-05 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094636+0000", 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6484999999999996e-05 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.629 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104716+0000", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010080428000000001 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126844+0000", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022127320999999998 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126903+0000", 2023-10-09T06:20:01.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9360000000000001e-05 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126964+0000", 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0622999999999999e-05 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127011+0000", 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7321999999999999e-05 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.631 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90935 9.d 9:b2389883:::benchmark_data_smithi107_300016_object90934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.378983931999997, 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97652245800000004, 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.632 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90935 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:20:01.633 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150744+0000", 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150812+0000", 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8767000000000001e-05 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150826+0000", 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3434e-05 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150831+0000", 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0039999999999999e-06 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150893+0000", 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2515999999999996e-05 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150906+0000", 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2364999999999999e-05 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096362+0000", 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.945456729 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.636 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096452+0000", 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9395999999999996e-05 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096552+0000", 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100128 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.637 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104724+0000", 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0081719050000000001 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127064+0000", 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022340598 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127172+0000", 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000107358 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.638 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127223+0000", 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1517000000000002e-05 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127268+0000", 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4280000000000003e-05 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90943 9.d 9:b3157ca5:::benchmark_data_smithi107_300016_object90942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.371347172999997, 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96914109000000004, 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90943 2023-10-09T06:20:01.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158381+0000", 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158462+0000", 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1271000000000002e-05 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.642 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158475+0000", 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2584e-05 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158480+0000", 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7840000000000003e-06 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158523+0000", 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3368999999999998e-05 2023-10-09T06:20:01.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158532+0000", 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6829999999999997e-06 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.098947+0000", 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94041511099999997 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.099038+0000", 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0902000000000005e-05 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.099136+0000", 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8568999999999998e-05 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.108977+0000", 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.009841163 2023-10-09T06:20:01.645 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127329+0000", 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.018351781000000001 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127431+0000", 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101494 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127483+0000", 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2043999999999998e-05 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127523+0000", 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0329e-05 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.647 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90961 9.d 9:bd07329a:::benchmark_data_smithi107_300016_object90960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.190619+0000", 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.339109837999999, 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93710648399999996, 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90961 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.648 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190619+0000", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190619+0000", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190618+0000", 2023-10-09T06:20:01.649 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190665+0000", 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6964e-05 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190682+0000", 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7402e-05 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.650 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190689+0000", 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.844e-06 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190761+0000", 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2397999999999994e-05 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.651 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.190788+0000", 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7118999999999999e-05 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.100885+0000", 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91009693400000002 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.100972+0000", 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6748000000000004e-05 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.652 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.101073+0000", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100948 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.108988+0000", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0079147669999999996 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127385+0000", 2023-10-09T06:20:01.653 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.018397179 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127637+0000", 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00025230599999999999 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127690+0000", 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3195000000000001e-05 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.654 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127726+0000", 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5339999999999997e-05 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90995 9.d 9:bb274b35:::benchmark_data_smithi107_300016_object90994:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.235217+0000", 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.294511864999997, 2023-10-09T06:20:01.655 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89269336600000004, 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90995 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235217+0000", 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.656 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235217+0000", 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235216+0000", 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.657 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235244+0000", 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8104999999999999e-05 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235254+0000", 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0068000000000001e-05 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235259+0000", 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4299999999999997e-06 2023-10-09T06:20:01.658 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235317+0000", 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8139e-05 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.235327+0000", 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9920000000000002e-06 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.102857+0000", 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.86752944300000001 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.102946+0000", 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9197000000000007e-05 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.103039+0000", 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3185999999999999e-05 2023-10-09T06:20:01.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.115198+0000", 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012158795999999999 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127582+0000", 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012383916 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127838+0000", 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000255715 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127879+0000", 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1505000000000003e-05 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127911+0000", 2023-10-09T06:20:01.662 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1375999999999999e-05 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91031 9.d 9:b87400a4:::benchmark_data_smithi107_300016_object91030:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.297742+0000", 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.231986614, 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.83026530700000001, 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.663 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91031 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297742+0000", 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.664 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297742+0000", 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297740+0000", 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297810+0000", 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0368000000000001e-05 2023-10-09T06:20:01.665 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297839+0000", 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8739999999999999e-05 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297851+0000", 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1861999999999999e-05 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.666 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297939+0000", 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7775000000000006e-05 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.297952+0000", 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3692e-05 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104757+0000", 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.80680517200000001 2023-10-09T06:20:01.667 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104863+0000", 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105461 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104973+0000", 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000109966 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.668 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.115211+0000", 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010237598000000001 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127770+0000", 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012559259999999999 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127942+0000", 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017202 2023-10-09T06:20:01.669 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127978+0000", 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5960000000000001e-05 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128008+0000", 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9994000000000001e-05 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.670 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91047 9.d 9:b2cbd304:::benchmark_data_smithi107_300016_object91046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.324487+0000", 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.205241880999999, 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.80364825600000001, 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91047 2023-10-09T06:20:01.671 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324487+0000", 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324487+0000", 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.672 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324485+0000", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324523+0000", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7617000000000003e-05 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324541+0000", 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7982000000000002e-05 2023-10-09T06:20:01.673 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324548+0000", 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9650000000000002e-06 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324681+0000", 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000133189 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.674 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324693+0000", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1693e-05 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.106645+0000", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.781952596 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.106717+0000", 2023-10-09T06:20:01.675 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1297000000000005e-05 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.106823+0000", 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010683600000000001 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.121533+0000", 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014709808 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.676 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127811+0000", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006278011 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128066+0000", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000254589 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128103+0000", 2023-10-09T06:20:01.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6925000000000003e-05 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128135+0000", 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2727e-05 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.678 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91048 9.d 9:b160652a:::benchmark_data_smithi107_300016_object91047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.324565+0000", 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.205163769999999, 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.80369201000000001, 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91048 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.679 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324565+0000", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324565+0000", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324565+0000", 2023-10-09T06:20:01.680 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324595+0000", 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0598000000000001e-05 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324608+0000", 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2379e-05 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.681 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324613+0000", 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0429999999999998e-06 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324719+0000", 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010595 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.324725+0000", 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2809999999999997e-06 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.109023+0000", 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.78429765500000004 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.109110+0000", 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7464000000000001e-05 2023-10-09T06:20:01.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.109212+0000", 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010171299999999999 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.121539+0000", 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012327694 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128038+0000", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0064985260000000001 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128196+0000", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015775899999999999 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128229+0000", 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3216000000000002e-05 2023-10-09T06:20:01.685 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128257+0000", 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8376000000000001e-05 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:91054 9.d 9:bf279075:::benchmark_data_smithi107_300016_object91053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.333731+0000", 2023-10-09T06:20:01.686 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 36.195998484999997, 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.79465038799999999, 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 91054 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:20:01.687 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333731+0000", 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333731+0000", 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333729+0000", 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.688 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333812+0000", 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2645999999999994e-05 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333827+0000", 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4752e-05 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:20:01.689 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333833+0000", 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4489999999999996e-06 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333887+0000", 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3902e-05 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.333895+0000", 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0029999999999994e-06 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.110937+0000", 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77704207300000006 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.111047+0000", 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110115 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:20:01.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.111135+0000", 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7566999999999997e-05 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.121545+0000", 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010409750000000001 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128170+0000", 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0066251620000000004 2023-10-09T06:20:01.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128320+0000", 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000150507 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128353+0000", 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.235e-05 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:20:01.693 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.128381+0000", 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8225000000000002e-05 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:20:01.694 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:20:01.694 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:01.695 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:01.744 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:01.838 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:01.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:01.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:20:01.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.846 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: pgmap v617: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1633/247140 objects degraded (0.661%); 912/247140 objects misplaced (0.369%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:20:01.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:01.847 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:01.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: pgmap v617: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1633/247140 objects degraded (0.661%); 912/247140 objects misplaced (0.369%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:01.990 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: failed to encode map e189 with expected crc 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: pgmap v617: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1633/247140 objects degraded (0.661%); 912/247140 objects misplaced (0.369%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:01.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:02.031 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:02.173 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:02.289 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:02.307 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:02] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-10-09T06:20:02.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:02.337 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:20:02.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:02.464 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:02.548 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:02.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:02.668 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:02.745 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:02.772 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:20:03.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1633/247140 objects degraded (0.661%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:03.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1633/247140 objects degraded (0.661%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:03.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:03.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1633/247140 objects degraded (0.661%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:03.183 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:03.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:03.292 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:03.338 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:03.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:03.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:03.536 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:03.591 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:03.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:03.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:03.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:03.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:03.838 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: failed to encode map e190 with expected crc 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: failed to encode map e190 with expected crc 2023-10-09T06:20:03.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:03 smithi150 ceph-mon[194178]: pgmap v619: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 1633/244809 objects degraded (0.667%); 912/244809 objects misplaced (0.373%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:20:03.924 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:03.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: failed to encode map e190 with expected crc 2023-10-09T06:20:04.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[212750]: pgmap v619: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 1633/244809 objects degraded (0.667%); 912/244809 objects misplaced (0.373%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:20:04.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:04.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/189119852' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:04.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T06:20:04.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: failed to encode map e190 with expected crc 2023-10-09T06:20:04.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: failed to encode map e190 with expected crc 2023-10-09T06:20:04.005 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:03 smithi107 ceph-mon[220921]: pgmap v619: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 1633/244809 objects degraded (0.667%); 912/244809 objects misplaced (0.373%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T06:20:04.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:04.117 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:04.378 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:04.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:04.488 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:04.550 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:04.651 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:04.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:04.764 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:04.792 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:05.205 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:05.228 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:05.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:05.336 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:05.412 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:05.506 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:05.544 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:05.586 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:05.636 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:05.668 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:05.732 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:05.770 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:05.826 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:05.837 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.837 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.837 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.837 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.837 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.838 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.838 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.838 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:05.965 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:20:05.981 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:05.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:05.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:06.097 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:06.141 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:06.236 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:06.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:06.343 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:06.382 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:06 smithi150 ceph-mon[194178]: pgmap v620: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 1633/236979 objects degraded (0.689%); 407/236979 objects misplaced (0.172%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:20:06.382 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/163746230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:06.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:06 smithi107 ceph-mon[212750]: pgmap v620: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 1633/236979 objects degraded (0.689%); 407/236979 objects misplaced (0.172%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:20:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/163746230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:06 smithi107 ceph-mon[220921]: pgmap v620: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 1633/236979 objects degraded (0.689%); 407/236979 objects misplaced (0.172%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T06:20:06.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/163746230' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:06.425 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:06.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:06.532 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:06.602 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.063 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:20:07.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3605984814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1633/236979 objects degraded (0.689%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:07.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3605984814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.095 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.095 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1633/236979 objects degraded (0.689%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:07.371 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3605984814' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:20:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1633/236979 objects degraded (0.689%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:20:07.479 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:07.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:07.653 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:07.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:07.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:07.762 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:07.798 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: pgmap v621: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 1633/235566 objects degraded (0.693%); 407/235566 objects misplaced (0.173%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[212750]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.143 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:08.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:08.161 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:20:08.247 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:08.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: pgmap v621: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 1633/235566 objects degraded (0.693%); 407/235566 objects misplaced (0.173%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.318 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:08 smithi107 ceph-mon[220921]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.340 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: pgmap v621: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 1633/235566 objects degraded (0.693%); 407/235566 objects misplaced (0.173%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:08 smithi150 ceph-mon[194178]: failed to encode map e191 with expected crc 2023-10-09T06:20:08.436 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:08.473 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:08.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:08.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:08.585 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:08.601 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:08.601 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:08.669 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:08.861 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:08.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:08.966 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:09.025 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.087 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:09.194 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:09.253 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:09.455 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:09.878 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:09.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:09.979 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:10.004 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:10.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:10.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:10.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:10 smithi107 ceph-mon[212750]: pgmap v623: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 1633/227502 objects degraded (0.718%); 407/227502 objects misplaced (0.179%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:10.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:10 smithi107 ceph-mon[220921]: pgmap v623: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 1633/227502 objects degraded (0.718%); 407/227502 objects misplaced (0.179%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:10.154 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:10.208 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.230 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:20:10.269 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:10.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:10 smithi150 ceph-mon[194178]: pgmap v623: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 1633/227502 objects degraded (0.718%); 407/227502 objects misplaced (0.179%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:10.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:10.375 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.402 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.445 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:10.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:10.485 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:10.535 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:10.554 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:10.587 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.705 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:10.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:10.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:10.812 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:10.838 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:10.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:10.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:10.888 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:10.988 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:11.023 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:11 smithi150 ceph-mon[194178]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:11 smithi150 ceph-mon[194178]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:11 smithi150 ceph-mon[194178]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:20:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:11 smithi150 ceph-mon[194178]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.208 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[220921]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.209 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[220921]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.209 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[220921]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:20:11.209 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[220921]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.211 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[212750]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T06:20:11.211 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:11 smithi107 ceph-mon[212750]: failed to encode map e192 with expected crc 2023-10-09T06:20:11.504 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:11.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:11.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:11.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:11.673 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:11.779 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:11.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:11.894 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:11.931 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:12.079 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: pgmap v625: 89 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 1631/225381 objects degraded (0.724%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2345881166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1631/225381 objects degraded (0.724%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:12.366 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:12] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-10-09T06:20:12.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: pgmap v625: 89 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 1631/225381 objects degraded (0.724%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2345881166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1631/225381 objects degraded (0.724%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: pgmap v625: 89 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 1631/225381 objects degraded (0.724%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: failed to encode map e193 with expected crc 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2345881166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1631/225381 objects degraded (0.724%), 2 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:13.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[212750]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:13 smithi107 ceph-mon[220921]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:13 smithi150 ceph-mon[194178]: failed to encode map e194 with expected crc 2023-10-09T06:20:13.787 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:13.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:13.897 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:13.963 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.012 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:14.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:14.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:14.118 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:14.146 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:14 smithi107 ceph-mon[212750]: pgmap v628: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 1631/222918 objects degraded (0.732%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T06:20:14.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:14 smithi107 ceph-mon[220921]: pgmap v628: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 1631/222918 objects degraded (0.732%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T06:20:14.191 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:14.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:14.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:14.296 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:14.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:14 smithi150 ceph-mon[194178]: pgmap v628: 89 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 1631/222918 objects degraded (0.732%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T06:20:14.355 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.455 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:14.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:14.566 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:14.647 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:14.817 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:14.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:14.926 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:14.993 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:15.037 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:15.041 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:15.081 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:15.090 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1886680132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:15.103 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1886680132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:15.126 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:15.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:15.223 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:15.231 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:15.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:15.279 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:15.318 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:15.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:15.348 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:15.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1886680132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:15.438 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:15.550 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:20:15.550 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:20:15.551 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:20:15.551 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:20:15.551 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:20:15.564 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:15.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:15.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:15.608 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:20:15.676 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:15.771 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:16.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[220921]: pgmap v629: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 804/215826 objects degraded (0.373%); 448/215826 objects misplaced (0.208%); 8.3 MiB/s, 131 objects/s recovering 2023-10-09T06:20:16.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2045742212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:16.100 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:16.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[212750]: pgmap v629: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 804/215826 objects degraded (0.373%); 448/215826 objects misplaced (0.208%); 8.3 MiB/s, 131 objects/s recovering 2023-10-09T06:20:16.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2045742212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:16.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:16 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:16.116 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:20:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:16 smithi150 ceph-mon[194178]: pgmap v629: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 804/215826 objects degraded (0.373%); 448/215826 objects misplaced (0.208%); 8.3 MiB/s, 131 objects/s recovering 2023-10-09T06:20:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2045742212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:16 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:20:17.231 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:17.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:17.322 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.322 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:17.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 804/215826 objects degraded (0.373%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:17.324 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:17.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 804/215826 objects degraded (0.373%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:17.338 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.391 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.391 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:20:17.391 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: failed to encode map e195 with expected crc 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:17.392 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 804/215826 objects degraded (0.373%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:17.394 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:17.483 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:17.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:17.593 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:17.613 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:20:17.637 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:17.670 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:20:17.707 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:17.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:17.818 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:17.857 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:17.876 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:17.932 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:17.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:18.037 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:18.084 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:18.166 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:18.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:18.277 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:18.364 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:18 smithi107 ceph-mon[212750]: pgmap v631: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 804/214413 objects degraded (0.375%); 448/214413 objects misplaced (0.209%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:18.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:18 smithi107 ceph-mon[220921]: pgmap v631: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 804/214413 objects degraded (0.375%); 448/214413 objects misplaced (0.209%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:18.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:18 smithi150 ceph-mon[194178]: pgmap v631: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 804/214413 objects degraded (0.375%); 448/214413 objects misplaced (0.209%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T06:20:18.633 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:18.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:18.744 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:18.801 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:18.843 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:18.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:18.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:18.948 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:18.975 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.016 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:19.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:19.119 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:19.187 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.289 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:19.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:19.356 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:19.371 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:19.400 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:19.419 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:19.446 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.526 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:19.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:19.672 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:19.751 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:19.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:19.860 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:19.920 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.014 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:20.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:20.127 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:20.206 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:20 smithi107 ceph-mon[212750]: pgmap v632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 804/206820 objects degraded (0.389%); 32/206820 objects misplaced (0.015%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:20:20.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:20 smithi107 ceph-mon[220921]: pgmap v632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 804/206820 objects degraded (0.389%); 32/206820 objects misplaced (0.015%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:20:20.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:20 smithi150 ceph-mon[194178]: pgmap v632: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 804/206820 objects degraded (0.389%); 32/206820 objects misplaced (0.015%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:20:21.141 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:21.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:21.164 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:20:21.246 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:21.278 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:21.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:20:21.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1463006293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.346 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.346 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.346 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1463006293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.346 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.367 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:21.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: failed to encode map e196 with expected crc 2023-10-09T06:20:21.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1463006293' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:21.480 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:21.519 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:21.560 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:21.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:21.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:21.752 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: pgmap v634: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 804/204840 objects degraded (0.393%); 32/204840 objects misplaced (0.016%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:20:22.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:22.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.352 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:22.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:20:22.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 804/204840 objects degraded (0.393%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: pgmap v634: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 804/204840 objects degraded (0.393%); 32/204840 objects misplaced (0.016%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:20:22.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.355 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.355 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 804/204840 objects degraded (0.393%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:22.356 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:22] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: pgmap v634: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 804/204840 objects degraded (0.393%); 32/204840 objects misplaced (0.016%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T06:20:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: failed to encode map e197 with expected crc 2023-10-09T06:20:22.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 804/204840 objects degraded (0.393%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:20:22.528 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:22.643 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:22.707 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:22.784 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:22.905 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:22.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164028 7.6 7:6e57a499:::benchmark_data_smithi107_237467_object61139:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:20:22.914 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 563.03437484699998, 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0265854210000001, 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164028 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877322+0000", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877320+0000", 2023-10-09T06:20:22.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877332+0000", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1945e-05 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877355+0000", 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2385e-05 2023-10-09T06:20:22.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.877367+0000", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2782999999999999e-05 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879335+0000", 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019675890000000001 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879341+0000", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9309999999999996e-06 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.903847+0000", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0245063409999999 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.903908+0000", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0418000000000003e-05 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164036 7.6 7:68fb9b1f:::benchmark_data_smithi107_237467_object61147:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 563.03241568399994, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.070745885, 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.920 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164036 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879281+0000", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879280+0000", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879287+0000", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5100000000000004e-06 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879299+0000", 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2593e-05 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879314+0000", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4611e-05 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879352+0000", 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8498000000000003e-05 2023-10-09T06:20:22.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.879356+0000", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1970000000000001e-06 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904149+0000", 2023-10-09T06:20:22.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0247934160000001 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904218+0000", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9089000000000002e-05 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.949949+0000", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.045731115000000003 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.950027+0000", 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8215999999999998e-05 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.927 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34272.0:164055 7.6 7:68939ae2:::benchmark_data_smithi107_237467_object61166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e98)", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 563.01214222700003, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.014796101, 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.928 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34272", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4020881800", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 164055 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899555+0000", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899554+0000", 2023-10-09T06:20:22.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899559+0000", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7469999999999996e-06 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899570+0000", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.899577+0000", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1999999999999997e-06 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.903132+0000", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003554911 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:10:59.903144+0000", 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2489e-05 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904266+0000", 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0011211259999999 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.934 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904350+0000", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4363000000000005e-05 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.904437+0000", 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6550000000000003e-05 2023-10-09T06:20:22.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.911505+0000", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070688929999999997 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:22.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914120+0000", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002614487 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914178+0000", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8471999999999999e-05 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:22.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914232+0000", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3752999999999998e-05 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:11:00.914351+0000", 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011889 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.938 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53739 8.1 8:8560477b:::benchmark_data_smithi107_251897_object53738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 458.01607435800003, 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011737895, 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53739 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895623+0000", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895622+0000", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895646+0000", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4346e-05 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895658+0000", 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1829e-05 2023-10-09T06:20:22.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895667+0000", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0189999999999995e-06 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895725+0000", 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7825999999999998e-05 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895734+0000", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3729999999999999e-06 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907232+0000", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0114980069999999 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907361+0000", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012863999999999999 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53743 8.1 8:8dd73d4f:::benchmark_data_smithi107_251897_object53742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 458.01601046399998, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0119421850000001, 2023-10-09T06:20:22.944 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53743 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895687+0000", 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895686+0000", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895706+0000", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9834999999999999e-05 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895713+0000", 2023-10-09T06:20:22.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.691e-06 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895718+0000", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6189999999999999e-06 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8346000000000002e-05 2023-10-09T06:20:22.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895762+0000", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8189999999999997e-06 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907554+0000", 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0117913940000001 2023-10-09T06:20:22.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907629+0000", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4975999999999995e-05 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53745 8.1 8:89b04a6c:::benchmark_data_smithi107_251897_object53744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:20:22.950 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 458.01597365100002, 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011994064, 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53745 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895723+0000", 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895741+0000", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7733000000000001e-05 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895747+0000", 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.596e-06 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895753+0000", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2510000000000002e-06 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895784+0000", 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0846999999999998e-05 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895789+0000", 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6219999999999997e-06 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907652+0000", 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0118627149999999 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907718+0000", 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.567e-05 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:53746 8.1 8:8f4403ee:::benchmark_data_smithi107_251897_object53745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e107)", 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 458.01593984300001, 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.023758089, 2023-10-09T06:20:22.956 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 53746 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895757+0000", 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895775+0000", 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7768e-05 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895780+0000", 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6509999999999998e-06 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895785+0000", 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8169999999999998e-06 2023-10-09T06:20:22.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895808+0000", 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3003e-05 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:44.895813+0000", 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2610000000000001e-06 2023-10-09T06:20:22.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907740+0000", 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0119261020000001 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.907864+0000", 2023-10-09T06:20:22.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012438099999999999 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.919426+0000", 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011562413000000001 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:12:45.919515+0000", 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9061000000000007e-05 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75980 8.1 8:85ec2575:::benchmark_data_smithi107_251897_object75979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:20:22.964 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 440.08438030500002, 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0842550950000001, 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75980 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.965 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827317+0000", 2023-10-09T06:20:22.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827315+0000", 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827343+0000", 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7511000000000001e-05 2023-10-09T06:20:22.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827360+0000", 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7618999999999999e-05 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827372+0000", 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1697e-05 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827469+0000", 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7077999999999994e-05 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827478+0000", 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829307+0000", 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018292200000000001 2023-10-09T06:20:22.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829320+0000", 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2662e-05 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900507+0000", 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0711866210000001 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900688+0000", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018153099999999999 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.900863+0000", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017435399999999999 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907377+0000", 2023-10-09T06:20:22.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065140379999999998 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907392+0000", 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5113000000000001e-05 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910524+0000", 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031320279999999998 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910577+0000", 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3477999999999999e-05 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911572+0000", 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00099476300000000007 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75985 8.1 8:844c4be0:::benchmark_data_smithi107_251897_object75984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 440.08431036600001, 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0907094479999999, 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75985 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.974 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827387+0000", 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827386+0000", 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827408+0000", 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1869999999999999e-05 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827416+0000", 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3399999999999998e-06 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827423+0000", 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7270000000000001e-06 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829369+0000", 2023-10-09T06:20:22.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00194557 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829379+0000", 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.006e-05 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904462+0000", 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0750832909999999 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904613+0000", 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015070900000000001 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.904744+0000", 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013179500000000001 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910436+0000", 2023-10-09T06:20:22.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056916629999999996 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.910480+0000", 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4121e-05 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.917077+0000", 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065964780000000002 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.917136+0000", 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9684e-05 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918096+0000", 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00095980100000000004 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:75993 8.1 8:8297030a:::benchmark_data_smithi107_251897_object75992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 440.08378019899999, 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0907953589999999, 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.983 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 75993 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:20:22.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827917+0000", 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827916+0000", 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827963+0000", 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6594999999999997e-05 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827973+0000", 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0414999999999999e-05 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827982+0000", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.49e-06 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829408+0000", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014259850000000001 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829417+0000", 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1819999999999995e-06 2023-10-09T06:20:22.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907421+0000", 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.078004368 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907583+0000", 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016161499999999999 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.907778+0000", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000194971 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914689+0000", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069108240000000003 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914704+0000", 2023-10-09T06:20:22.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5427999999999999e-05 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918139+0000", 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034351619999999999 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918191+0000", 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1901000000000003e-05 2023-10-09T06:20:22.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918712+0000", 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052121800000000005 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.991 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:76002 8.1 8:8589f225:::benchmark_data_smithi107_251897_object76001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 440.08370851900003, 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.091170872, 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 76002 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827989+0000", 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.827988+0000", 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828043+0000", 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5210999999999997e-05 2023-10-09T06:20:22.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828051+0000", 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7829999999999999e-06 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.828057+0000", 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7989999999999999e-06 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829442+0000", 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013856490000000001 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:02.829448+0000", 2023-10-09T06:20:22.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8200000000000002e-06 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911665+0000", 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0822164009999999 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911818+0000", 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015297800000000001 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.911933+0000", 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115646 2023-10-09T06:20:22.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.914801+0000", 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028680709999999998 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.915159+0000", 2023-10-09T06:20:22.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035749499999999999 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918752+0000", 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003592961 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.918805+0000", 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3224999999999999e-05 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:03.919159+0000", 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035442399999999999 2023-10-09T06:20:23.001 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162302 8.e 8:7101b874:::benchmark_data_smithi107_251897_object45388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.75134606900002, 2023-10-09T06:20:23.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.060445393, 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162302 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160351+0000", 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160349+0000", 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160359+0000", 2023-10-09T06:20:23.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4290000000000007e-06 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160380+0000", 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1739999999999999e-05 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160390+0000", 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2269999999999999e-06 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161137+0000", 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00074714099999999999 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161255+0000", 2023-10-09T06:20:23.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000118217 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.161363+0000", 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108325 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.166561+0000", 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051975420000000003 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173923+0000", 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073616810000000001 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188983+0000", 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015060772 2023-10-09T06:20:23.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188993+0000", 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.005e-05 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.220709+0000", 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.031715916 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.220796+0000", 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7115999999999997e-05 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162314 8.e 8:779025f7:::benchmark_data_smithi107_251897_object45400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.73785478399998, 2023-10-09T06:20:23.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0473069500000001, 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162314 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173842+0000", 2023-10-09T06:20:23.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173844+0000", 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3350000000000001e-06 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173851+0000", 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4710000000000004e-06 2023-10-09T06:20:23.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173857+0000", 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8340000000000004e-06 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.173948+0000", 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0878000000000006e-05 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.174058+0000", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011074299999999999 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.174174+0000", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115643 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.178772+0000", 2023-10-09T06:20:23.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045980989999999996 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189024+0000", 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01025155 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189031+0000", 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1559999999999998e-06 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221086+0000", 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0320555389999999 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221149+0000", 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3066999999999998e-05 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162323 8.e 8:7b14d2dc:::benchmark_data_smithi107_251897_object45409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.72306368800002, 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.047554023, 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162323 2023-10-09T06:20:23.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188633+0000", 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188632+0000", 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188639+0000", 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2869999999999999e-06 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188651+0000", 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1702000000000001e-05 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.188660+0000", 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8060000000000001e-06 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189123+0000", 2023-10-09T06:20:23.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046345499999999999 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189130+0000", 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.579e-06 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221200+0000", 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0320701640000001 2023-10-09T06:20:23.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221288+0000", 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8208000000000007e-05 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4", 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.221369+0000", 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0900000000000001e-05 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231308+0000", 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099386189999999992 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236045+0000", 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047374510000000002 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236096+0000", 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0469000000000001e-05 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.236187+0000", 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1762999999999995e-05 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176913 8.e 8:751e71d0:::benchmark_data_smithi107_251897_object59999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 311.195551679, 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.154407339, 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.027 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176913 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716145+0000", 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:23.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716148+0000", 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.286e-06 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716155+0000", 2023-10-09T06:20:23.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3960000000000003e-06 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716160+0000", 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9740000000000001e-06 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716285+0000", 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012556700000000001 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.716296+0000", 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.098e-05 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.860697+0000", 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.14440092099999999 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.860709+0000", 2023-10-09T06:20:23.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1685000000000001e-05 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859539+0000", 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9988303840000001 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859613+0000", 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3980000000000007e-05 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859727+0000", 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114228 2023-10-09T06:20:23.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.866046+0000", 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0063190549999999996 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.866078+0000", 2023-10-09T06:20:23.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1996999999999999e-05 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870327+0000", 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0042490569999999997 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870388+0000", 2023-10-09T06:20:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1098000000000004e-05 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870553+0000", 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016430500000000001 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:192018 8.e 8:708941a0:::benchmark_data_smithi107_251897_object75104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 288.84327683700002, 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.012711275, 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 192018 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068420+0000", 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068423+0000", 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1190000000000001e-06 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068429+0000", 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4470000000000003e-06 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.068435+0000", 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7030000000000003e-06 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.069829+0000", 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001393949 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:34.069840+0000", 2023-10-09T06:20:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1068e-05 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.075899+0000", 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.006058517 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.076004+0000", 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105814 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.076113+0000", 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109028 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081010+0000", 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0048969770000000003 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081063+0000", 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2398999999999998e-05 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081074+0000", 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.128e-05 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081109+0000", 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4637000000000001e-05 2023-10-09T06:20:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:35.081132+0000", 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2836e-05 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231197 8.c 8:3d5e510f:::benchmark_data_smithi107_251897_object114283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:20:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 233.323911165, 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0702522830000001, 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231197 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587786+0000", 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587785+0000", 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587792+0000", 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0180000000000004e-06 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587807+0000", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4932e-05 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587817+0000", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0679999999999999e-05 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.587874+0000", 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6702e-05 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588034+0000", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016021899999999999 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588171+0000", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013668400000000001 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.598003+0000", 2023-10-09T06:20:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0098321499999999996 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.605004+0000", 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070009989999999999 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.605013+0000", 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1630000000000002e-06 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.657929+0000", 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0529163100000001 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658038+0000", 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108799 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231205 8.c 8:3765e502:::benchmark_data_smithi107_251897_object114291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 233.29938695800001, 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.056454129, 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231205 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612310+0000", 2023-10-09T06:20:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612308+0000", 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612315+0000", 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1620000000000002e-06 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612332+0000", 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6688e-05 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612343+0000", 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0988000000000001e-05 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612397+0000", 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3671000000000002e-05 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.612410+0000", 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3529e-05 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658355+0000", 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0459445469999999 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658467+0000", 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000111896 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 7", 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.658540+0000", 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3749000000000002e-05 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.665781+0000", 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072405739999999996 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668654+0000", 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028725000000000001 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668685+0000", 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1189999999999998e-05 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.668764+0000", 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9584000000000002e-05 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:102483 9.5 9:a4f32bc2:::benchmark_data_smithi107_300016_object4911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:42.983298+0000", 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 39.928398895999997, 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0065695429999999, 2023-10-09T06:20:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 102483 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983298+0000", 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983298+0000", 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983298+0000", 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983300+0000", 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5749999999999999e-06 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983306+0000", 2023-10-09T06:20:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7969999999999997e-06 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983312+0000", 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3800000000000002e-06 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983365+0000", 2023-10-09T06:20:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3717000000000001e-05 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.983374+0000", 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.976134+0000", 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99275967300000001 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.976266+0000", 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000132514 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.976367+0000", 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100797 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.982464+0000", 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0060973379999999999 2023-10-09T06:20:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.985510+0000", 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030456160000000001 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.989772+0000", 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0042621400000000002 2023-10-09T06:20:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.989826+0000", 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4147000000000002e-05 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.989868+0000", 2023-10-09T06:20:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1542000000000003e-05 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:102490 9.5 9:abfea5c3:::benchmark_data_smithi107_300016_object4918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:42.992973+0000", 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 39.918724071, 2023-10-09T06:20:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.009757094, 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 102490 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.992973+0000", 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.992973+0000", 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.992972+0000", 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.992978+0000", 2023-10-09T06:20:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9299999999999997e-06 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.992991+0000", 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2568000000000001e-05 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.993000+0000", 2023-10-09T06:20:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1039999999999996e-06 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.993054+0000", 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3569999999999997e-05 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:42.993065+0000", 2023-10-09T06:20:23.079 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1158000000000001e-05 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.979014+0000", 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98594945000000001 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.979100+0000", 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5827000000000003e-05 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,7", 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.979164+0000", 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4210999999999999e-05 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.985457+0000", 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0062926550000000003 2023-10-09T06:20:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:43.991976+0000", 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065185750000000004 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:44.002649+0000", 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010673369 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:44.002685+0000", 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6269000000000001e-05 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:44.002730+0000", 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4909e-05 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:23.083 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:20:23.127 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.200 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:23.306 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:23.367 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:23.571 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:23.617 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.704 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:23.816 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:23.866 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:23.965 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:23.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:23.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:24.076 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:24.155 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:24.217 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:20:24.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[220921]: pgmap v636: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 804/201966 objects degraded (0.398%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:20:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3214060245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[212750]: pgmap v636: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 804/201966 objects degraded (0.398%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:20:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3214060245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.479 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:20:24.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:24 smithi150 ceph-mon[194178]: pgmap v636: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 804/201966 objects degraded (0.398%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:20:24.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3214060245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:20:24.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:20:25.452 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:20:25.452 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.455 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:25 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.546 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:25.555 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:20:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:20:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T06:20:25.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:25 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:25.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:25.793 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:25.952 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:25.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:26.064 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:26.108 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:26.138 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:26.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:26.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:26.217 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:26.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:26.324 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:26.360 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:26.361 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:26.366 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:26 smithi150 ceph-mon[194178]: failed to encode map e198 with expected crc 2023-10-09T06:20:26.366 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:26 smithi150 ceph-mon[194178]: pgmap v638: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 354/195741 objects misplaced (0.181%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:20:26.372 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:26.406 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:26.440 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:26.448 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:26.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:26.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:26.468 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:26.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:26 smithi107 ceph-mon[212750]: failed to encode map e198 with expected crc 2023-10-09T06:20:26.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:26 smithi107 ceph-mon[212750]: pgmap v638: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 354/195741 objects misplaced (0.181%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:20:26.485 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:26 smithi107 ceph-mon[220921]: failed to encode map e198 with expected crc 2023-10-09T06:20:26.485 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:26 smithi107 ceph-mon[220921]: pgmap v638: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 354/195741 objects misplaced (0.181%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:20:26.509 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:26.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:26.557 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:26.588 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:26.609 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:26.641 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:26.958 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:26.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:26.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:27.065 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:27.136 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1656084949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:27.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:20:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1656084949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:27.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:20:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1656084949' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:20:28.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:28 smithi107 ceph-mon[212750]: pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 354/194379 objects misplaced (0.182%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:20:28.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:28 smithi107 ceph-mon[220921]: pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 354/194379 objects misplaced (0.182%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:20:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:28 smithi150 ceph-mon[194178]: pgmap v639: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 354/194379 objects misplaced (0.182%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:20:28.739 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:28.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:28.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:28.845 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:28.904 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.079 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:29.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:29.261 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.493 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:29 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:29.494 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:29 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:29.518 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:29.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:29.623 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:29.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:20:29.658 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.774 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:29.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:29.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:29.953 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:29.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:30.058 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:30.085 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:30.133 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:30.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:30.151 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:30.151 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:30.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:30.252 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:20:30.277 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:30.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 25/187137 objects misplaced (0.013%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/485048524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 25/187137 objects misplaced (0.013%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:20:30.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:20:30.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/485048524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:30 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.520 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:30.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:30.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: pgmap v640: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 25/187137 objects misplaced (0.013%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T06:20:30.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T06:20:30.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: failed to encode map e199 with expected crc 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/485048524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.528 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:30 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:20:30.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:30.629 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:30.679 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.272 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:20:31.310 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:20:31.320 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:31.463 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:31.626 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:31.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.628 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:20:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:20:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T06:20:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: failed to encode map e200 with expected crc 2023-10-09T06:20:31.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:20:31.727 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:31.793 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:31.863 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:31.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:31.966 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:32.039 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.263 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:32] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T06:20:32.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: pgmap v643: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25/185244 objects misplaced (0.013%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:20:32.268 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:32.532 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:20:32.532 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: pgmap v643: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25/185244 objects misplaced (0.013%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.533 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:32.571 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: pgmap v643: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 25/185244 objects misplaced (0.013%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3855831208' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: osdmap e201: 8 total, 8 up, 8 in 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: failed to encode map e201 with expected crc 2023-10-09T06:20:32.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:32.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:32.695 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:32.759 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:32.769 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:32.926 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:32.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:33.040 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:33.124 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.225 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:33.335 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:33.418 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.512 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:33.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:33.674 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:33.786 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:33.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:33.890 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:33.948 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[212750]: pgmap v645: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 25/182256 objects misplaced (0.014%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[220921]: pgmap v645: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 25/182256 objects misplaced (0.014%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:34 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:34 smithi150 ceph-mon[194178]: pgmap v645: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 25/182256 objects misplaced (0.014%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:20:34.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:34 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:34 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:34.458 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:34.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:34.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:34.492 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:20:34.568 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:34.629 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:35.039 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:35.040 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:35.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:35.147 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:35.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:35 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:20:35.267 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:35.388 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:35.488 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:35 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:20:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:35 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:20:35.740 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:35.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:35.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:35.892 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:35.955 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:35.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:36.046 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:36.064 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:36.078 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:36.078 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:36.094 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:36.102 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:36.136 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:36.175 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:36.176 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:36.209 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:36.253 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:36.287 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:36.310 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:36.324 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:36.340 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:36.354 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:36 smithi107 ceph-mon[212750]: pgmap v646: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 267 KiB/s, 4 objects/s recovering 2023-10-09T06:20:36.354 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/386967068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:36.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:36 smithi107 ceph-mon[220921]: pgmap v646: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 267 KiB/s, 4 objects/s recovering 2023-10-09T06:20:36.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/386967068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.365 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:36.409 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:36.537 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:36.590 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:36 smithi150 ceph-mon[194178]: pgmap v646: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 267 KiB/s, 4 objects/s recovering 2023-10-09T06:20:36.591 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/386967068' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:36.592 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:36.657 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:36.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:36.762 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:36.792 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.558 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:37.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:37.664 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:37.740 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:37.746 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:20:38.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:38 smithi107 ceph-mon[212750]: pgmap v647: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 206 KiB/s, 3 objects/s recovering 2023-10-09T06:20:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:38 smithi150 ceph-mon[194178]: pgmap v647: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 206 KiB/s, 3 objects/s recovering 2023-10-09T06:20:38.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:38 smithi107 ceph-mon[220921]: pgmap v647: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 206 KiB/s, 3 objects/s recovering 2023-10-09T06:20:38.954 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:38.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:39.063 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:39.120 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.235 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:39.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:39.357 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:39.400 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.487 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:39.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:39.598 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:39.653 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:39.750 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:39.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:39.772 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:39.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:39.952 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.149 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:40.238 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:20:40.257 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:40.295 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.354 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:40.468 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:40.511 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:40 smithi107 ceph-mon[212750]: pgmap v648: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 183 KiB/s, 2 objects/s recovering 2023-10-09T06:20:40.511 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:40 smithi107 ceph-mon[220921]: pgmap v648: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 183 KiB/s, 2 objects/s recovering 2023-10-09T06:20:40.557 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:40 smithi150 ceph-mon[194178]: pgmap v648: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 183 KiB/s, 2 objects/s recovering 2023-10-09T06:20:40.820 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:40.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:40.923 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:40.973 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.017 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:41.125 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:41.151 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.193 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:41.264 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:20:41.264 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:20:41.300 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:41.347 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.434 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:41.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:41.543 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:41.619 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:20:41.619 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/807589768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.619 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/807589768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:41.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:41.626 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:20:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/807589768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.116 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:42.221 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:42.228 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:20:42.228 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: pgmap v649: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.292 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.292 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.292 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.292 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.292 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:42] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: pgmap v649: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.293 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.294 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.294 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.294 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.435 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:42.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:42.547 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:42.614 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: pgmap v649: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1965568212' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: osdmap e202: 8 total, 8 up, 8 in 2023-10-09T06:20:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: failed to encode map e202 with expected crc 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:42.881 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:42.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:42.988 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:43.041 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.111 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:43.224 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:43.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.360 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:43.426 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:43.427 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93002 8.3 8:c23c1f88:::benchmark_data_smithi107_251897_object93001:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:20:43.490 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 442.63648115199999, 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0136282910000001, 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93002 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851823+0000", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851821+0000", 2023-10-09T06:20:43.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851849+0000", 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.72e-05 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851861+0000", 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2972999999999999e-05 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851870+0000", 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8480000000000007e-06 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851939+0000", 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9049999999999998e-05 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:20:43.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.851944+0000", 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9509999999999997e-06 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854442+0000", 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024976659999999999 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854451+0000", 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4490000000000005e-06 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865368+0000", 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0109166650000001 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865451+0000", 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3094000000000001e-05 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93007 8.3 8:c788829f:::benchmark_data_smithi107_251897_object93006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 442.63421658700003, 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.025725453, 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93007 2023-10-09T06:20:43.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854087+0000", 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854086+0000", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854112+0000", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5928e-05 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854125+0000", 2023-10-09T06:20:43.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3168e-05 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854135+0000", 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0084999999999999e-05 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854657+0000", 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000521621 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.854666+0000", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8510000000000005e-06 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865819+0000", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.011153274 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.865902+0000", 2023-10-09T06:20:43.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3090999999999994e-05 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.866012+0000", 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110571 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.875194+0000", 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091818119999999993 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.875229+0000", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4848999999999998e-05 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879743+0000", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045137509999999999 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879777+0000", 2023-10-09T06:20:43.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4147999999999999e-05 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879813+0000", 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5893999999999999e-05 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:93019 8.3 8:c4546894:::benchmark_data_smithi107_251897_object93018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e113)", 2023-10-09T06:20:43.504 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 442.62388675400001, 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.017569357, 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.505 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.506 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.506 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 93019 2023-10-09T06:20:43.506 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864417+0000", 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864416+0000", 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864440+0000", 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3807000000000001e-05 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.508 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864450+0000", 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.045e-05 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864458+0000", 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9719999999999992e-06 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864513+0000", 2023-10-09T06:20:43.509 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4622e-05 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:20.864523+0000", 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0564000000000001e-05 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.867903+0000", 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0033802949999999 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.510 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.868043+0000", 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013930299999999999 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.868202+0000", 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015874899999999999 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.511 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.879826+0000", 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011624834000000001 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881863+0000", 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00203666 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881902+0000", 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9048000000000003e-05 2023-10-09T06:20:43.512 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881945+0000", 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3384999999999999e-05 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:13:21.881987+0000", 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1214999999999998e-05 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.513 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160720 8.7 8:eb35e384:::benchmark_data_smithi107_251897_object43806:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 355.332325282, 2023-10-09T06:20:43.514 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0247990149999999, 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160720 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:20:43.515 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155979+0000", 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155977+0000", 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.516 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.155985+0000", 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4079999999999998e-06 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156014+0000", 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8671e-05 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.517 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156024+0000", 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5349999999999993e-06 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169755+0000", 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013731729999999999 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169767+0000", 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1823e-05 2023-10-09T06:20:43.518 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.180692+0000", 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0109248500000001 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.180778+0000", 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5822000000000006e-05 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.519 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160723 8.7 8:e87a05d0:::benchmark_data_smithi107_251897_object43809:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 355.332269209, 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0470548319999999, 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.520 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160723 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156035+0000", 2023-10-09T06:20:43.521 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156034+0000", 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.522 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156038+0000", 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1879999999999998e-06 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156044+0000", 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0290000000000002e-06 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156049+0000", 2023-10-09T06:20:43.523 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9200000000000003e-06 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169787+0000", 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01373889 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169794+0000", 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9269999999999999e-06 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.524 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181389+0000", 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0115949829999999 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181521+0000", 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000131736 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.525 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.202970+0000", 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021449275 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203090+0000", 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119338 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.526 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160724 8.7 8:e23977a0:::benchmark_data_smithi107_251897_object43810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 355.33225175400003, 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.047384474, 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160724 2023-10-09T06:20:43.527 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.528 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156052+0000", 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156054+0000", 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2819999999999999e-06 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.529 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156060+0000", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2480000000000004e-06 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156064+0000", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3490000000000002e-06 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169814+0000", 2023-10-09T06:20:43.530 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013749740999999999 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169820+0000", 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5390000000000004e-06 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181584+0000", 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0117643000000001 2023-10-09T06:20:43.531 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181682+0000", 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7349999999999995e-05 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.533 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203344+0000", 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021662364 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203437+0000", 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2603999999999996e-05 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.534 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160729 8.7 8:e8d3cb53:::benchmark_data_smithi107_251897_object43815:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 355.332236669, 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0430160129999999, 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160729 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.535 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.536 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156067+0000", 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156069+0000", 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9250000000000002e-06 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.537 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156074+0000", 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0320000000000003e-06 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156078+0000", 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3610000000000003e-06 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.538 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169840+0000", 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013761051999999999 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169846+0000", 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8519999999999998e-06 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.539 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181733+0000", 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0118862280000001 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.181862+0000", 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000129691 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.182446+0000", 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058357799999999996 2023-10-09T06:20:43.540 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.189884+0000", 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074385659999999998 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.195994+0000", 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0061091219999999998 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.541 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.198929+0000", 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002935797 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.198987+0000", 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8125000000000003e-05 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.542 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.199083+0000", 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5947999999999997e-05 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:160737 8.7 8:e2f37bf9:::benchmark_data_smithi107_251897_object43823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 355.33222221599999, 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0547179149999999, 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.543 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 160737 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.544 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156082+0000", 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.545 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156084+0000", 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9369999999999998e-06 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156089+0000", 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4310000000000002e-06 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.156093+0000", 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3499999999999999e-06 2023-10-09T06:20:43.546 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169866+0000", 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013772863999999999 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:48.169873+0000", 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6189999999999996e-06 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.547 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196022+0000", 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.02614893 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196120+0000", 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7827999999999999e-05 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.548 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.196205+0000", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5265000000000004e-05 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203135+0000", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006929724 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.203302+0000", 2023-10-09T06:20:43.549 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016770899999999999 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210674+0000", 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073718250000000003 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210726+0000", 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1895999999999999e-05 2023-10-09T06:20:43.550 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:49.210800+0000", 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3824000000000003e-05 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162305 8.a 8:5d4cc9a4:::benchmark_data_smithi107_251897_object45391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:43.551 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.32798969200002, 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0500951030000001, 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162305 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.552 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160314+0000", 2023-10-09T06:20:43.553 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160316+0000", 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6110000000000001e-06 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160322+0000", 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6289999999999998e-06 2023-10-09T06:20:43.554 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160327+0000", 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1930000000000001e-06 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160352+0000", 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4896e-05 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.555 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160550+0000", 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000197732 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.160658+0000", 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108039 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.556 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.166029+0000", 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053705699999999999 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.176139+0000", 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010110887000000001 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.191860+0000", 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015721044 2023-10-09T06:20:43.557 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.191870+0000", 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8439999999999993e-06 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.210328+0000", 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0184577509999999 2023-10-09T06:20:43.558 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.210409+0000", 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1397999999999996e-05 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162309 8.a 8:5870fc90:::benchmark_data_smithi107_251897_object45395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:20:43.559 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.31889727599997, 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.041964809, 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162309 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.560 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169407+0000", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169405+0000", 2023-10-09T06:20:43.561 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169412+0000", 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2180000000000002e-06 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169433+0000", 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0420000000000001e-05 2023-10-09T06:20:43.562 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.169442+0000", 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4970000000000007e-06 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171660+0000", 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022180530000000002 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.563 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171801+0000", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014037300000000001 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.171929+0000", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000127914 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.177701+0000", 2023-10-09T06:20:43.564 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057724229999999996 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.189331+0000", 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011629525999999999 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192876+0000", 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035456989999999998 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.565 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192888+0000", 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1413000000000001e-05 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.211303+0000", 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.018415525 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.566 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.211372+0000", 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8337000000000007e-05 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162315 8.1 8:89f47883:::benchmark_data_smithi107_251897_object45401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.302040288, 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0542757009999999, 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.567 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162315 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.568 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186264+0000", 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186262+0000", 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.569 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186268+0000", 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6119999999999998e-06 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186277+0000", 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0119999999999994e-06 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186283+0000", 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:20:43.570 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186447+0000", 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000164086 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186571+0000", 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000123607 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.571 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.186690+0000", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119653 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193070+0000", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0063797419999999999 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193078+0000", 2023-10-09T06:20:43.572 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0700000000000007e-06 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.222587+0000", 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0295085129999999 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.222700+0000", 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0001127 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.573 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.240453+0000", 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017753890000000001 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.240539+0000", 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6069000000000006e-05 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.574 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162320 8.1 8:8b316482:::benchmark_data_smithi107_251897_object45406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.29608440599998, 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0519342679999999, 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162320 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.575 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192220+0000", 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.576 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192218+0000", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192228+0000", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.91e-06 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192247+0000", 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8972999999999999e-05 2023-10-09T06:20:43.577 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.192257+0000", 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0061000000000001e-05 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193540+0000", 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012824869999999999 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.578 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.193547+0000", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8329999999999994e-06 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223305+0000", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0297573090000001 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223409+0000", 2023-10-09T06:20:43.579 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010399300000000001 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.223520+0000", 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000111723 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.230986+0000", 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074656439999999996 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.580 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244030+0000", 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013043602 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244077+0000", 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6981000000000003e-05 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.581 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244154+0000", 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7329000000000005e-05 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162329 8.1 8:835e27a5:::benchmark_data_smithi107_251897_object45415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.28278759099999, 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.038815201, 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.582 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162329 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.583 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205516+0000", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205515+0000", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205525+0000", 2023-10-09T06:20:43.584 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.059e-05 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205545+0000", 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9582e-05 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205556+0000", 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1056999999999999e-05 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.585 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205622+0000", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5757000000000001e-05 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.205636+0000", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.435e-05 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227102+0000", 2023-10-09T06:20:43.586 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0214658539999999 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227217+0000", 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115251 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.227340+0000", 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012321899999999999 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.587 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.235355+0000", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0080145619999999994 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244200+0000", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088452549999999998 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244254+0000", 2023-10-09T06:20:43.588 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3495000000000002e-05 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244332+0000", 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8145000000000005e-05 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:162333 8.1 8:88e0a74d:::benchmark_data_smithi107_251897_object45419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:20:43.589 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 352.27018015200002, 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0263455720000001, 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 162333 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.590 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218124+0000", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218122+0000", 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.591 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218130+0000", 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0299999999999994e-06 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218145+0000", 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4691999999999999e-05 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.592 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218153+0000", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0530000000000007e-06 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218193+0000", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9475999999999999e-05 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:51.218203+0000", 2023-10-09T06:20:43.593 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0229e-05 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231014+0000", 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0128109240000001 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231130+0000", 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116248 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.594 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0", 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.231267+0000", 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013658999999999999 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.235360+0000", 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00409377 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.595 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244375+0000", 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090142599999999996 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244414+0000", 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9273999999999997e-05 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:14:52.244469+0000", 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5528000000000001e-05 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.596 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176889 8.1 8:842966fd:::benchmark_data_smithi107_251897_object59975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.79666824200001, 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1818074730000001, 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.597 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176889 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691636+0000", 2023-10-09T06:20:43.598 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691634+0000", 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691640+0000", 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8429999999999998e-06 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.599 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691652+0000", 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1778e-05 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691662+0000", 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.600 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691810+0000", 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014854000000000001 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.691824+0000", 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3281e-05 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862557+0000", 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.170733891 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.601 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862570+0000", 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2087e-05 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859255+0000", 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9966857579999999 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.602 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859403+0000", 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014815999999999999 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.859545+0000", 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014124499999999999 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870614+0000", 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011069522 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.603 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.870629+0000", 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4932999999999999e-05 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873223+0000", 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025939309999999998 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.604 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873276+0000", 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2948999999999997e-05 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873443+0000", 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016722699999999999 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.605 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176917 8.1 8:891de888:::benchmark_data_smithi107_251897_object60003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.766563803, 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1591737929999999, 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176917 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.606 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721740+0000", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721739+0000", 2023-10-09T06:20:43.607 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721747+0000", 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.422e-06 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721761+0000", 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3781000000000001e-05 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.608 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721770+0000", 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9169999999999995e-06 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721847+0000", 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7078999999999998e-05 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.609 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.721860+0000", 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2833000000000001e-05 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862594+0000", 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.140733884 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862602+0000", 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2179999999999999e-06 2023-10-09T06:20:43.610 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863642+0000", 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.001040164 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863765+0000", 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012266100000000001 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:20:43.611 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863882+0000", 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011691 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.872363+0000", 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.008481255 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.872371+0000", 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.926e-06 2023-10-09T06:20:43.612 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880690+0000", 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083193750000000004 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880744+0000", 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3591000000000002e-05 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.613 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.880914+0000", 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000170321 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176924 8.a 8:5db5ab82:::benchmark_data_smithi107_251897_object60010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.75522496899998, 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.141292129, 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.614 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176924 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.615 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733079+0000", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733077+0000", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733084+0000", 2023-10-09T06:20:43.616 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0650000000000001e-06 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733099+0000", 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4124999999999999e-05 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733108+0000", 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.825e-06 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.617 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733179+0000", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0592000000000004e-05 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733191+0000", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1746000000000001e-05 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862654+0000", 2023-10-09T06:20:43.618 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.12946349500000001 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.862664+0000", 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5619999999999993e-06 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863267+0000", 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0006028649999998 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.619 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863408+0000", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000141013 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.863529+0000", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000121663 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.871583+0000", 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0080541659999999998 2023-10-09T06:20:43.620 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873555+0000", 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001971492 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874145+0000", 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058958700000000003 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.621 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874191+0000", 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6397999999999999e-05 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874371+0000", 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018031000000000001 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.622 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:176925 8.a 8:5360f822:::benchmark_data_smithi107_251897_object60011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.75498677899998, 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1412183040000001, 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 176925 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.623 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.624 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733317+0000", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733321+0000", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.211e-06 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733331+0000", 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0514e-05 2023-10-09T06:20:43.625 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733339+0000", 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.289e-06 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733616+0000", 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000277275 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.626 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.733622+0000", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1349999999999997e-06 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.863714+0000", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.130092441 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:11.863724+0000", 2023-10-09T06:20:43.627 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1020000000000003e-06 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867691+0000", 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0039678250000001 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867831+0000", 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000140061 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.628 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.867946+0000", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114079 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.871810+0000", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003864606 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.873593+0000", 2023-10-09T06:20:43.629 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001783228 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874416+0000", 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082254599999999995 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874452+0000", 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6037000000000001e-05 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.630 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:13.874536+0000", 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3639000000000003e-05 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:227141 8.a 8:5db9bc41:::benchmark_data_smithi107_251897_object110227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 260.13794304100003, 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0313837400000001, 2023-10-09T06:20:43.631 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 227141 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.632 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350361+0000", 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350360+0000", 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.633 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350367+0000", 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.61e-06 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350383+0000", 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6019e-05 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350392+0000", 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3300000000000005e-06 2023-10-09T06:20:43.634 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350443+0000", 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0949000000000003e-05 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:23.350457+0000", 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3511e-05 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.635 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372276+0000", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0218192319999999 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372402+0000", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012583700000000001 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.372491+0000", 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8690999999999995e-05 2023-10-09T06:20:43.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.375873+0000", 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003382005 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381596+0000", 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057236939999999997 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.637 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381676+0000", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9623000000000007e-05 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381710+0000", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4304999999999998e-05 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:24.381745+0000", 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4433e-05 2023-10-09T06:20:43.638 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231190 8.7 8:ecf39326:::benchmark_data_smithi107_251897_object114276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 253.913697216, 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.104789587, 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:20:43.639 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231190 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574607+0000", 2023-10-09T06:20:43.640 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574606+0000", 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574611+0000", 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4269999999999999e-06 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.641 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574620+0000", 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2689999999999999e-06 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.574625+0000", 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1510000000000003e-06 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579614+0000", 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0049893089999999999 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579782+0000", 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016760700000000001 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.579925+0000", 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014329999999999999 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.643 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.588552+0000", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0086273949999999995 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.593852+0000", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0052992849999999999 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.615463+0000", 2023-10-09T06:20:43.644 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021611755 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.615477+0000", 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3352e-05 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679328+0000", 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0638517119999999 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:20:43.645 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679396+0000", 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8059999999999996e-05 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:20:43.646 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:20:43.646 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:43.687 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:43.795 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:43.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:43.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:43.953 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:44.046 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:44.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:44.158 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:44.221 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:44.289 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:44 smithi107 ceph-mon[212750]: pgmap v651: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:44.626 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:44 smithi107 ceph-mon[220921]: pgmap v651: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:44.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:44 smithi150 ceph-mon[194178]: pgmap v651: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 160 KiB/s, 2 objects/s recovering 2023-10-09T06:20:45.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:20:45.246 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:45.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:45.359 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:45.455 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:45.788 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:20:45.912 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:45.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:46.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:46.069 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:46.160 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:46.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:46.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:46.301 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:46.305 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:46.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:46.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:46.393 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:46 smithi150 ceph-mon[194178]: pgmap v652: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:20:46.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:46 smithi107 ceph-mon[212750]: pgmap v652: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:20:46.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:46 smithi107 ceph-mon[220921]: pgmap v652: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:20:46.493 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:46.591 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:47.116 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:47.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:47.120 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:47.157 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:47.194 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:47.220 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:47.251 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:47.307 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:47.315 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:47.353 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:47.408 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:47.445 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:47.465 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.466 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:20:47.497 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:47.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:47.523 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:20:47.525 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:20:47.607 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:47.633 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:47.675 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:47.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:47.780 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:47.827 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:47.828 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:47.828 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1801117240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:47.834 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:47.834 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:47.834 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1801117240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:47.835 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:20:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:20:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1801117240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:48.443 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:48.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:48.709 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:48 smithi107 ceph-mon[220921]: pgmap v653: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:20:48.710 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:48 smithi107 ceph-mon[220921]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:48.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:48.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:48 smithi107 ceph-mon[212750]: pgmap v653: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:20:48.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:48 smithi107 ceph-mon[212750]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:48.812 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:48.835 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:48 smithi150 ceph-mon[194178]: pgmap v653: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-09T06:20:48.835 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:48 smithi150 ceph-mon[194178]: from='client.55426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:48.839 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:48.880 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:48.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:48.986 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:49.036 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":653,"stamp":"2023-10-09T06:20:46.991309+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3345422264,"num_objects":51063,"num_object_clones":0,"num_object_copies":153189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51063,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":144401,"num_write_kb":6246862,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8446,"num_bytes_recovered":554709108,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130017,"ondisk_log_size":130017,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13977460,"kb_used_data":9999796,"kb_used_omap":20,"kb_used_meta":3977003,"kb_avail":735983756,"statfs":{"total":767960285184,"available":753647366144,"internally_reserved":0,"allocated":10239791104,"data_stored":10233492894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4072451726},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,6,6,11,2,0,0,0,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-465829888,"num_objects":-7108,"num_object_clones":0,"num_object_copies":-21324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7108,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005464"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263039+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:20:31.263039+0000","last_peered":"2023-10-09T06:20:31.263039+0000","last_clean":"2023-10-09T06:20:31.263039+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:20:31.263039+0000","last_undegraded":"2023-10-09T06:20:31.263039+0000","last_fullsized":"2023-10-09T06:20:31.263039+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":137,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293350+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:20:31.293350+0000","last_peered":"2023-10-09T06:20:31.293350+0000","last_clean":"2023-10-09T06:20:31.293350+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:20:31.293350+0000","last_undegraded":"2023-10-09T06:20:31.293350+0000","last_fullsized":"2023-10-09T06:20:31.293350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274926+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:20:31.274926+0000","last_peered":"2023-10-09T06:20:31.274926+0000","last_clean":"2023-10-09T06:20:31.274926+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:20:31.274926+0000","last_undegraded":"2023-10-09T06:20:31.274926+0000","last_fullsized":"2023-10-09T06:20:31.274926+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261422+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:20:31.261422+0000","last_peered":"2023-10-09T06:20:31.261422+0000","last_clean":"2023-10-09T06:20:31.261422+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:20:31.261422+0000","last_undegraded":"2023-10-09T06:20:31.261422+0000","last_fullsized":"2023-10-09T06:20:31.261422+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293707+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:20:31.293707+0000","last_peered":"2023-10-09T06:20:31.293707+0000","last_clean":"2023-10-09T06:20:31.293707+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:20:31.293707+0000","last_undegraded":"2023-10-09T06:20:31.293707+0000","last_fullsized":"2023-10-09T06:20:31.293707+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":161,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.299075+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:20:31.299075+0000","last_peered":"2023-10-09T06:20:31.299075+0000","last_clean":"2023-10-09T06:20:31.299075+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:20:31.299075+0000","last_undegraded":"2023-10-09T06:20:31.299075+0000","last_fullsized":"2023-10-09T06:20:31.299075+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261884+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:20:31.261884+0000","last_peered":"2023-10-09T06:20:31.261884+0000","last_clean":"2023-10-09T06:20:31.261884+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:20:31.261884+0000","last_undegraded":"2023-10-09T06:20:31.261884+0000","last_fullsized":"2023-10-09T06:20:31.261884+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":137,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293785+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:20:31.293785+0000","last_peered":"2023-10-09T06:20:31.293785+0000","last_clean":"2023-10-09T06:20:31.293785+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:20:31.293785+0000","last_undegraded":"2023-10-09T06:20:31.293785+0000","last_fullsized":"2023-10-09T06:20:31.293785+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275380+0000","last_change":"2023-10-09T06:19:06.673434+0000","last_active":"2023-10-09T06:20:31.275380+0000","last_peered":"2023-10-09T06:20:31.275380+0000","last_clean":"2023-10-09T06:20:31.275380+0000","last_became_active":"2023-10-09T06:19:06.673042+0000","last_became_peered":"2023-10-09T06:19:06.673042+0000","last_unstale":"2023-10-09T06:20:31.275380+0000","last_undegraded":"2023-10-09T06:20:31.275380+0000","last_fullsized":"2023-10-09T06:20:31.275380+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:30.387567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289571+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:20:31.289571+0000","last_peered":"2023-10-09T06:20:31.289571+0000","last_clean":"2023-10-09T06:20:31.289571+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:20:31.289571+0000","last_undegraded":"2023-10-09T06:20:31.289571+0000","last_fullsized":"2023-10-09T06:20:31.289571+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":51,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274687+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:20:31.274687+0000","last_peered":"2023-10-09T06:20:31.274687+0000","last_clean":"2023-10-09T06:20:31.274687+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:20:31.274687+0000","last_undegraded":"2023-10-09T06:20:31.274687+0000","last_fullsized":"2023-10-09T06:20:31.274687+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.277157+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:20:31.277157+0000","last_peered":"2023-10-09T06:20:31.277157+0000","last_clean":"2023-10-09T06:20:31.277157+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:20:31.277157+0000","last_undegraded":"2023-10-09T06:20:31.277157+0000","last_fullsized":"2023-10-09T06:20:31.277157+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":208,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263422+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:20:31.263422+0000","last_peered":"2023-10-09T06:20:31.263422+0000","last_clean":"2023-10-09T06:20:31.263422+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:20:31.263422+0000","last_undegraded":"2023-10-09T06:20:31.263422+0000","last_fullsized":"2023-10-09T06:20:31.263422+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263171+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:20:31.263171+0000","last_peered":"2023-10-09T06:20:31.263171+0000","last_clean":"2023-10-09T06:20:31.263171+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:20:31.263171+0000","last_undegraded":"2023-10-09T06:20:31.263171+0000","last_fullsized":"2023-10-09T06:20:31.263171+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":51,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274097+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:20:31.274097+0000","last_peered":"2023-10-09T06:20:31.274097+0000","last_clean":"2023-10-09T06:20:31.274097+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:20:31.274097+0000","last_undegraded":"2023-10-09T06:20:31.274097+0000","last_fullsized":"2023-10-09T06:20:31.274097+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293162+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:20:31.293162+0000","last_peered":"2023-10-09T06:20:31.293162+0000","last_clean":"2023-10-09T06:20:31.293162+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:20:31.293162+0000","last_undegraded":"2023-10-09T06:20:31.293162+0000","last_fullsized":"2023-10-09T06:20:31.293162+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261012+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:20:31.261012+0000","last_peered":"2023-10-09T06:20:31.261012+0000","last_clean":"2023-10-09T06:20:31.261012+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:20:31.261012+0000","last_undegraded":"2023-10-09T06:20:31.261012+0000","last_fullsized":"2023-10-09T06:20:31.261012+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275987+0000","last_change":"2023-10-09T06:19:06.758564+0000","last_active":"2023-10-09T06:20:31.275987+0000","last_peered":"2023-10-09T06:20:31.275987+0000","last_clean":"2023-10-09T06:20:31.275987+0000","last_became_active":"2023-10-09T06:19:06.758270+0000","last_became_peered":"2023-10-09T06:19:06.758270+0000","last_unstale":"2023-10-09T06:20:31.275987+0000","last_undegraded":"2023-10-09T06:20:31.275987+0000","last_fullsized":"2023-10-09T06:20:31.275987+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.823391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.272916+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:20:31.272916+0000","last_peered":"2023-10-09T06:20:31.272916+0000","last_clean":"2023-10-09T06:20:31.272916+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:20:31.272916+0000","last_undegraded":"2023-10-09T06:20:31.272916+0000","last_fullsized":"2023-10-09T06:20:31.272916+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261554+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:20:31.261554+0000","last_peered":"2023-10-09T06:20:31.261554+0000","last_clean":"2023-10-09T06:20:31.261554+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:20:31.261554+0000","last_undegraded":"2023-10-09T06:20:31.261554+0000","last_fullsized":"2023-10-09T06:20:31.261554+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.298660+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:20:31.298660+0000","last_peered":"2023-10-09T06:20:31.298660+0000","last_clean":"2023-10-09T06:20:31.298660+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:20:31.298660+0000","last_undegraded":"2023-10-09T06:20:31.298660+0000","last_fullsized":"2023-10-09T06:20:31.298660+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293572+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:20:31.293572+0000","last_peered":"2023-10-09T06:20:31.293572+0000","last_clean":"2023-10-09T06:20:31.293572+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:20:31.293572+0000","last_undegraded":"2023-10-09T06:20:31.293572+0000","last_fullsized":"2023-10-09T06:20:31.293572+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"202'8882","reported_seq":10717,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:44.910244+0000","last_change":"2023-10-09T06:19:43.978924+0000","last_active":"2023-10-09T06:20:44.910244+0000","last_peered":"2023-10-09T06:20:44.910244+0000","last_clean":"2023-10-09T06:20:44.910244+0000","last_became_active":"2023-10-09T06:19:43.970383+0000","last_became_peered":"2023-10-09T06:19:43.970383+0000","last_unstale":"2023-10-09T06:20:44.910244+0000","last_undegraded":"2023-10-09T06:20:44.910244+0000","last_fullsized":"2023-10-09T06:20:44.910244+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:59.701538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209190912,"num_objects":3192,"num_object_clones":0,"num_object_copies":9576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8882,"num_write_kb":386368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":875,"num_bytes_recovered":57344000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":49,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274266+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:20:31.274266+0000","last_peered":"2023-10-09T06:20:31.274266+0000","last_clean":"2023-10-09T06:20:31.274266+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:20:31.274266+0000","last_undegraded":"2023-10-09T06:20:31.274266+0000","last_fullsized":"2023-10-09T06:20:31.274266+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.273941+0000","last_change":"2023-10-09T06:19:06.673411+0000","last_active":"2023-10-09T06:20:31.273941+0000","last_peered":"2023-10-09T06:20:31.273941+0000","last_clean":"2023-10-09T06:20:31.273941+0000","last_became_active":"2023-10-09T06:19:06.673113+0000","last_became_peered":"2023-10-09T06:19:06.673113+0000","last_unstale":"2023-10-09T06:20:31.273941+0000","last_undegraded":"2023-10-09T06:20:31.273941+0000","last_fullsized":"2023-10-09T06:20:31.273941+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:39.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"202'9085","reported_seq":9145,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:46.489497+0000","last_change":"2023-10-09T06:17:57.020187+0000","last_active":"2023-10-09T06:20:46.489497+0000","last_peered":"2023-10-09T06:20:46.489497+0000","last_clean":"2023-10-09T06:20:46.489497+0000","last_became_active":"2023-10-09T06:17:57.019892+0000","last_became_peered":"2023-10-09T06:17:57.019892+0000","last_unstale":"2023-10-09T06:20:46.489497+0000","last_undegraded":"2023-10-09T06:20:46.489497+0000","last_fullsized":"2023-10-09T06:20:46.489497+0000","mapping_epoch":152,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:01.403006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202047488,"num_objects":3083,"num_object_clones":0,"num_object_copies":9249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9085,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":49,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274218+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:20:31.274218+0000","last_peered":"2023-10-09T06:20:31.274218+0000","last_clean":"2023-10-09T06:20:31.274218+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:20:31.274218+0000","last_undegraded":"2023-10-09T06:20:31.274218+0000","last_fullsized":"2023-10-09T06:20:31.274218+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263576+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:20:31.263576+0000","last_peered":"2023-10-09T06:20:31.263576+0000","last_clean":"2023-10-09T06:20:31.263576+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:20:31.263576+0000","last_undegraded":"2023-10-09T06:20:31.263576+0000","last_fullsized":"2023-10-09T06:20:31.263576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"202'9115","reported_seq":10850,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.103789+0000","last_change":"2023-10-09T06:20:00.751403+0000","last_active":"2023-10-09T06:20:43.103789+0000","last_peered":"2023-10-09T06:20:43.103789+0000","last_clean":"2023-10-09T06:20:43.103789+0000","last_became_active":"2023-10-09T06:20:00.740256+0000","last_became_peered":"2023-10-09T06:20:00.740256+0000","last_unstale":"2023-10-09T06:20:43.103789+0000","last_undegraded":"2023-10-09T06:20:43.103789+0000","last_fullsized":"2023-10-09T06:20:43.103789+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9115,"log_dups_size":0,"ondisk_log_size":9115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:18.856099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211353600,"num_objects":3225,"num_object_clones":0,"num_object_copies":9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9115,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":49,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274499+0000","last_change":"2023-10-09T06:19:42.328723+0000","last_active":"2023-10-09T06:20:31.274499+0000","last_peered":"2023-10-09T06:20:31.274499+0000","last_clean":"2023-10-09T06:20:31.274499+0000","last_became_active":"2023-10-09T06:19:06.757480+0000","last_became_peered":"2023-10-09T06:19:06.757480+0000","last_unstale":"2023-10-09T06:20:31.274499+0000","last_undegraded":"2023-10-09T06:20:31.274499+0000","last_fullsized":"2023-10-09T06:20:31.274499+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:57.712635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.298933+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:20:31.298933+0000","last_peered":"2023-10-09T06:20:31.298933+0000","last_clean":"2023-10-09T06:20:31.298933+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:20:31.298933+0000","last_undegraded":"2023-10-09T06:20:31.298933+0000","last_fullsized":"2023-10-09T06:20:31.298933+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"202'9070","reported_seq":9127,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.114742+0000","last_change":"2023-10-09T06:17:57.217507+0000","last_active":"2023-10-09T06:20:43.114742+0000","last_peered":"2023-10-09T06:20:43.114742+0000","last_clean":"2023-10-09T06:20:43.114742+0000","last_became_active":"2023-10-09T06:17:57.217183+0000","last_became_peered":"2023-10-09T06:17:57.217183+0000","last_unstale":"2023-10-09T06:20:43.114742+0000","last_undegraded":"2023-10-09T06:20:43.114742+0000","last_fullsized":"2023-10-09T06:20:43.114742+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9070,"log_dups_size":0,"ondisk_log_size":9070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:56.905622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216399872,"num_objects":3302,"num_object_clones":0,"num_object_copies":9906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9070,"num_write_kb":395904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274137+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:20:31.274137+0000","last_peered":"2023-10-09T06:20:31.274137+0000","last_clean":"2023-10-09T06:20:31.274137+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:20:31.274137+0000","last_undegraded":"2023-10-09T06:20:31.274137+0000","last_fullsized":"2023-10-09T06:20:31.274137+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.262264+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:20:31.262264+0000","last_peered":"2023-10-09T06:20:31.262264+0000","last_clean":"2023-10-09T06:20:31.262264+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:20:31.262264+0000","last_undegraded":"2023-10-09T06:20:31.262264+0000","last_fullsized":"2023-10-09T06:20:31.262264+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"202'9086","reported_seq":10780,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.618418+0000","last_change":"2023-10-09T06:20:30.273502+0000","last_active":"2023-10-09T06:20:43.618418+0000","last_peered":"2023-10-09T06:20:43.618418+0000","last_clean":"2023-10-09T06:20:43.618418+0000","last_became_active":"2023-10-09T06:20:30.253922+0000","last_became_peered":"2023-10-09T06:20:30.253922+0000","last_unstale":"2023-10-09T06:20:43.618418+0000","last_undegraded":"2023-10-09T06:20:43.618418+0000","last_fullsized":"2023-10-09T06:20:43.618418+0000","mapping_epoch":199,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:32.110407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208273408,"num_objects":3178,"num_object_clones":0,"num_object_copies":9534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9086,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":163,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289427+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:20:31.289427+0000","last_peered":"2023-10-09T06:20:31.289427+0000","last_clean":"2023-10-09T06:20:31.289427+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:20:31.289427+0000","last_undegraded":"2023-10-09T06:20:31.289427+0000","last_fullsized":"2023-10-09T06:20:31.289427+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263492+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:20:31.263492+0000","last_peered":"2023-10-09T06:20:31.263492+0000","last_clean":"2023-10-09T06:20:31.263492+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:20:31.263492+0000","last_undegraded":"2023-10-09T06:20:31.263492+0000","last_fullsized":"2023-10-09T06:20:31.263492+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"202'8968","reported_seq":9028,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:46.464763+0000","last_change":"2023-10-09T06:17:57.236716+0000","last_active":"2023-10-09T06:20:46.464763+0000","last_peered":"2023-10-09T06:20:46.464763+0000","last_clean":"2023-10-09T06:20:46.464763+0000","last_became_active":"2023-10-09T06:17:57.236363+0000","last_became_peered":"2023-10-09T06:17:57.236363+0000","last_unstale":"2023-10-09T06:20:46.464763+0000","last_undegraded":"2023-10-09T06:20:46.464763+0000","last_fullsized":"2023-10-09T06:20:46.464763+0000","mapping_epoch":152,"log_start":"156'1600","ondisk_log_start":"156'1600","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:53.348003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200933376,"num_objects":3066,"num_object_clones":0,"num_object_copies":9198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8968,"num_write_kb":385088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.298901+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:20:31.298901+0000","last_peered":"2023-10-09T06:20:31.298901+0000","last_clean":"2023-10-09T06:20:31.298901+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:20:31.298901+0000","last_undegraded":"2023-10-09T06:20:31.298901+0000","last_fullsized":"2023-10-09T06:20:31.298901+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263180+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:20:31.263180+0000","last_peered":"2023-10-09T06:20:31.263180+0000","last_clean":"2023-10-09T06:20:31.263180+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:20:31.263180+0000","last_undegraded":"2023-10-09T06:20:31.263180+0000","last_fullsized":"2023-10-09T06:20:31.263180+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"202'9014","reported_seq":12586,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:42.924749+0000","last_change":"2023-10-09T06:20:21.206561+0000","last_active":"2023-10-09T06:20:42.924749+0000","last_peered":"2023-10-09T06:20:42.924749+0000","last_clean":"2023-10-09T06:20:42.924749+0000","last_became_active":"2023-10-09T06:20:21.163864+0000","last_became_peered":"2023-10-09T06:20:21.163864+0000","last_unstale":"2023-10-09T06:20:42.924749+0000","last_undegraded":"2023-10-09T06:20:42.924749+0000","last_fullsized":"2023-10-09T06:20:42.924749+0000","mapping_epoch":196,"log_start":"156'1500","ondisk_log_start":"156'1500","created":152,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:11.775536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207880192,"num_objects":3172,"num_object_clones":0,"num_object_copies":9516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9014,"num_write_kb":389952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1744,"num_bytes_recovered":114294784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263419+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:20:31.263419+0000","last_peered":"2023-10-09T06:20:31.263419+0000","last_clean":"2023-10-09T06:20:31.263419+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:20:31.263419+0000","last_undegraded":"2023-10-09T06:20:31.263419+0000","last_fullsized":"2023-10-09T06:20:31.263419+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263228+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:20:31.263228+0000","last_peered":"2023-10-09T06:20:31.263228+0000","last_clean":"2023-10-09T06:20:31.263228+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:20:31.263228+0000","last_undegraded":"2023-10-09T06:20:31.263228+0000","last_fullsized":"2023-10-09T06:20:31.263228+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"202'8823","reported_seq":8883,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.599592+0000","last_change":"2023-10-09T06:17:57.217463+0000","last_active":"2023-10-09T06:20:43.599592+0000","last_peered":"2023-10-09T06:20:43.599592+0000","last_clean":"2023-10-09T06:20:43.599592+0000","last_became_active":"2023-10-09T06:17:57.217100+0000","last_became_peered":"2023-10-09T06:17:57.217100+0000","last_unstale":"2023-10-09T06:20:43.599592+0000","last_undegraded":"2023-10-09T06:20:43.599592+0000","last_fullsized":"2023-10-09T06:20:43.599592+0000","mapping_epoch":152,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7123,"log_dups_size":0,"ondisk_log_size":7123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:23.024261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203620352,"num_objects":3107,"num_object_clones":0,"num_object_copies":9321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8823,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293862+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:20:31.293862+0000","last_peered":"2023-10-09T06:20:31.293862+0000","last_clean":"2023-10-09T06:20:31.293862+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:20:31.293862+0000","last_undegraded":"2023-10-09T06:20:31.293862+0000","last_fullsized":"2023-10-09T06:20:31.293862+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275135+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:20:31.275135+0000","last_peered":"2023-10-09T06:20:31.275135+0000","last_clean":"2023-10-09T06:20:31.275135+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:20:31.275135+0000","last_undegraded":"2023-10-09T06:20:31.275135+0000","last_fullsized":"2023-10-09T06:20:31.275135+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"202'8913","reported_seq":8971,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.084165+0000","last_change":"2023-10-09T06:17:57.022383+0000","last_active":"2023-10-09T06:20:43.084165+0000","last_peered":"2023-10-09T06:20:43.084165+0000","last_clean":"2023-10-09T06:20:43.084165+0000","last_became_active":"2023-10-09T06:17:57.022128+0000","last_became_peered":"2023-10-09T06:17:57.022128+0000","last_unstale":"2023-10-09T06:20:43.084165+0000","last_undegraded":"2023-10-09T06:20:43.084165+0000","last_fullsized":"2023-10-09T06:20:43.084165+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":8913,"log_dups_size":0,"ondisk_log_size":8913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.131013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206766080,"num_objects":3155,"num_object_clones":0,"num_object_copies":9465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8913,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":182,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275424+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:20:31.275424+0000","last_peered":"2023-10-09T06:20:31.275424+0000","last_clean":"2023-10-09T06:20:31.275424+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:20:31.275424+0000","last_undegraded":"2023-10-09T06:20:31.275424+0000","last_fullsized":"2023-10-09T06:20:31.275424+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276083+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:20:31.276083+0000","last_peered":"2023-10-09T06:20:31.276083+0000","last_clean":"2023-10-09T06:20:31.276083+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:20:31.276083+0000","last_undegraded":"2023-10-09T06:20:31.276083+0000","last_fullsized":"2023-10-09T06:20:31.276083+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276877+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:20:31.276877+0000","last_peered":"2023-10-09T06:20:31.276877+0000","last_clean":"2023-10-09T06:20:31.276877+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:20:31.276877+0000","last_undegraded":"2023-10-09T06:20:31.276877+0000","last_fullsized":"2023-10-09T06:20:31.276877+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"202'9095","reported_seq":10842,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:44.873998+0000","last_change":"2023-10-09T06:20:11.090804+0000","last_active":"2023-10-09T06:20:44.873998+0000","last_peered":"2023-10-09T06:20:44.873998+0000","last_clean":"2023-10-09T06:20:44.873998+0000","last_became_active":"2023-10-09T06:20:11.073402+0000","last_became_peered":"2023-10-09T06:20:11.073402+0000","last_unstale":"2023-10-09T06:20:44.873998+0000","last_undegraded":"2023-10-09T06:20:44.873998+0000","last_fullsized":"2023-10-09T06:20:44.873998+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9095,"log_dups_size":0,"ondisk_log_size":9095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:23.451973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9095,"num_write_kb":391936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":830,"num_bytes_recovered":54394880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274150+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:20:31.274150+0000","last_peered":"2023-10-09T06:20:31.274150+0000","last_clean":"2023-10-09T06:20:31.274150+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:20:31.274150+0000","last_undegraded":"2023-10-09T06:20:31.274150+0000","last_fullsized":"2023-10-09T06:20:31.274150+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263050+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:20:31.263050+0000","last_peered":"2023-10-09T06:20:31.263050+0000","last_clean":"2023-10-09T06:20:31.263050+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:20:31.263050+0000","last_undegraded":"2023-10-09T06:20:31.263050+0000","last_fullsized":"2023-10-09T06:20:31.263050+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":146,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275489+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:20:31.275489+0000","last_peered":"2023-10-09T06:20:31.275489+0000","last_clean":"2023-10-09T06:20:31.275489+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:20:31.275489+0000","last_undegraded":"2023-10-09T06:20:31.275489+0000","last_fullsized":"2023-10-09T06:20:31.275489+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"202'9021","reported_seq":10807,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.078894+0000","last_change":"2023-10-09T06:19:15.900457+0000","last_active":"2023-10-09T06:20:43.078894+0000","last_peered":"2023-10-09T06:20:43.078894+0000","last_clean":"2023-10-09T06:20:43.078894+0000","last_became_active":"2023-10-09T06:19:15.893387+0000","last_became_peered":"2023-10-09T06:19:15.893387+0000","last_unstale":"2023-10-09T06:20:43.078894+0000","last_undegraded":"2023-10-09T06:20:43.078894+0000","last_fullsized":"2023-10-09T06:20:43.078894+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9021,"log_dups_size":0,"ondisk_log_size":9021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:09.689997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212795392,"num_objects":3247,"num_object_clones":0,"num_object_copies":9741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9021,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":850,"num_bytes_recovered":55705600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":160,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.297624+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:20:31.297624+0000","last_peered":"2023-10-09T06:20:31.297624+0000","last_clean":"2023-10-09T06:20:31.297624+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:20:31.297624+0000","last_undegraded":"2023-10-09T06:20:31.297624+0000","last_fullsized":"2023-10-09T06:20:31.297624+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275044+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:20:31.275044+0000","last_peered":"2023-10-09T06:20:31.275044+0000","last_clean":"2023-10-09T06:20:31.275044+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:20:31.275044+0000","last_undegraded":"2023-10-09T06:20:31.275044+0000","last_fullsized":"2023-10-09T06:20:31.275044+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":159,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276113+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:20:31.276113+0000","last_peered":"2023-10-09T06:20:31.276113+0000","last_clean":"2023-10-09T06:20:31.276113+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:20:31.276113+0000","last_undegraded":"2023-10-09T06:20:31.276113+0000","last_fullsized":"2023-10-09T06:20:31.276113+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"202'8965","reported_seq":9025,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.620121+0000","last_change":"2023-10-09T06:17:57.016464+0000","last_active":"2023-10-09T06:20:43.620121+0000","last_peered":"2023-10-09T06:20:43.620121+0000","last_clean":"2023-10-09T06:20:43.620121+0000","last_became_active":"2023-10-09T06:17:57.016166+0000","last_became_peered":"2023-10-09T06:17:57.016166+0000","last_unstale":"2023-10-09T06:20:43.620121+0000","last_undegraded":"2023-10-09T06:20:43.620121+0000","last_fullsized":"2023-10-09T06:20:43.620121+0000","mapping_epoch":152,"log_start":"156'1800","ondisk_log_start":"156'1800","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7165,"log_dups_size":0,"ondisk_log_size":7165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:53.029953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218562560,"num_objects":3335,"num_object_clones":0,"num_object_copies":10005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8965,"num_write_kb":393600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":186,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274719+0000","last_change":"2023-10-09T06:19:06.672499+0000","last_active":"2023-10-09T06:20:31.274719+0000","last_peered":"2023-10-09T06:20:31.274719+0000","last_clean":"2023-10-09T06:20:31.274719+0000","last_became_active":"2023-10-09T06:19:06.672182+0000","last_became_peered":"2023-10-09T06:19:06.672182+0000","last_unstale":"2023-10-09T06:20:31.274719+0000","last_undegraded":"2023-10-09T06:20:31.274719+0000","last_fullsized":"2023-10-09T06:20:31.274719+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:07.246328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":454,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.270462+0000","last_change":"2023-10-09T06:20:11.075324+0000","last_active":"2023-10-09T06:20:31.270462+0000","last_peered":"2023-10-09T06:20:31.270462+0000","last_clean":"2023-10-09T06:20:31.270462+0000","last_became_active":"2023-10-09T06:20:11.074888+0000","last_became_peered":"2023-10-09T06:20:11.074888+0000","last_unstale":"2023-10-09T06:20:31.270462+0000","last_undegraded":"2023-10-09T06:20:31.270462+0000","last_fullsized":"2023-10-09T06:20:31.270462+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:03.264166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289455+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:20:31.289455+0000","last_peered":"2023-10-09T06:20:31.289455+0000","last_clean":"2023-10-09T06:20:31.289455+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:20:31.289455+0000","last_undegraded":"2023-10-09T06:20:31.289455+0000","last_fullsized":"2023-10-09T06:20:31.289455+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.298919+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:20:31.298919+0000","last_peered":"2023-10-09T06:20:31.298919+0000","last_clean":"2023-10-09T06:20:31.298919+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:20:31.298919+0000","last_undegraded":"2023-10-09T06:20:31.298919+0000","last_fullsized":"2023-10-09T06:20:31.298919+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"202'9318","reported_seq":9378,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:44.920066+0000","last_change":"2023-10-09T06:17:57.236464+0000","last_active":"2023-10-09T06:20:44.920066+0000","last_peered":"2023-10-09T06:20:44.920066+0000","last_clean":"2023-10-09T06:20:44.920066+0000","last_became_active":"2023-10-09T06:17:57.236124+0000","last_became_peered":"2023-10-09T06:17:57.236124+0000","last_unstale":"2023-10-09T06:20:44.920066+0000","last_undegraded":"2023-10-09T06:20:44.920066+0000","last_fullsized":"2023-10-09T06:20:44.920066+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9318,"log_dups_size":0,"ondisk_log_size":9318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:33.469044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204472320,"num_objects":3120,"num_object_clones":0,"num_object_copies":9360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9318,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":45,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.270306+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:20:31.270306+0000","last_peered":"2023-10-09T06:20:31.270306+0000","last_clean":"2023-10-09T06:20:31.270306+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:20:31.270306+0000","last_undegraded":"2023-10-09T06:20:31.270306+0000","last_fullsized":"2023-10-09T06:20:31.270306+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.244617+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:20:41.244617+0000","last_peered":"2023-10-09T06:20:41.244617+0000","last_clean":"2023-10-09T06:20:41.244617+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:20:41.244617+0000","last_undegraded":"2023-10-09T06:20:41.244617+0000","last_fullsized":"2023-10-09T06:20:41.244617+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261487+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:20:31.261487+0000","last_peered":"2023-10-09T06:20:31.261487+0000","last_clean":"2023-10-09T06:20:31.261487+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:20:31.261487+0000","last_undegraded":"2023-10-09T06:20:31.261487+0000","last_fullsized":"2023-10-09T06:20:31.261487+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"202'8999","reported_seq":10791,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:42.948129+0000","last_change":"2023-10-09T06:19:52.709796+0000","last_active":"2023-10-09T06:20:42.948129+0000","last_peered":"2023-10-09T06:20:42.948129+0000","last_clean":"2023-10-09T06:20:42.948129+0000","last_became_active":"2023-10-09T06:19:52.651903+0000","last_became_peered":"2023-10-09T06:19:52.651903+0000","last_unstale":"2023-10-09T06:20:42.948129+0000","last_undegraded":"2023-10-09T06:20:42.948129+0000","last_fullsized":"2023-10-09T06:20:42.948129+0000","mapping_epoch":184,"log_start":"156'1300","ondisk_log_start":"156'1300","created":152,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:57.164182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216989696,"num_objects":3311,"num_object_clones":0,"num_object_copies":9933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8999,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":45,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274613+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:20:31.274613+0000","last_peered":"2023-10-09T06:20:31.274613+0000","last_clean":"2023-10-09T06:20:31.274613+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:20:31.274613+0000","last_undegraded":"2023-10-09T06:20:31.274613+0000","last_fullsized":"2023-10-09T06:20:31.274613+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275566+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:20:31.275566+0000","last_peered":"2023-10-09T06:20:31.275566+0000","last_clean":"2023-10-09T06:20:31.275566+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:20:31.275566+0000","last_undegraded":"2023-10-09T06:20:31.275566+0000","last_fullsized":"2023-10-09T06:20:31.275566+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":156,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245818+0000","last_change":"2023-10-09T06:19:06.667489+0000","last_active":"2023-10-09T06:20:41.245818+0000","last_peered":"2023-10-09T06:20:41.245818+0000","last_clean":"2023-10-09T06:20:41.245818+0000","last_became_active":"2023-10-09T06:19:06.667029+0000","last_became_peered":"2023-10-09T06:19:06.667029+0000","last_unstale":"2023-10-09T06:20:41.245818+0000","last_undegraded":"2023-10-09T06:20:41.245818+0000","last_fullsized":"2023-10-09T06:20:41.245818+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:35.989088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"202'8975","reported_seq":10770,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:42.931154+0000","last_change":"2023-10-09T06:19:26.121898+0000","last_active":"2023-10-09T06:20:42.931154+0000","last_peered":"2023-10-09T06:20:42.931154+0000","last_clean":"2023-10-09T06:20:42.931154+0000","last_became_active":"2023-10-09T06:19:26.079797+0000","last_became_peered":"2023-10-09T06:19:26.079797+0000","last_unstale":"2023-10-09T06:20:42.931154+0000","last_undegraded":"2023-10-09T06:20:42.931154+0000","last_fullsized":"2023-10-09T06:20:42.931154+0000","mapping_epoch":173,"log_start":"156'1200","ondisk_log_start":"156'1200","created":152,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:05.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207749144,"num_objects":3171,"num_object_clones":0,"num_object_copies":9513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3171,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8975,"num_write_kb":388609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":189,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293659+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:20:31.293659+0000","last_peered":"2023-10-09T06:20:31.293659+0000","last_clean":"2023-10-09T06:20:31.293659+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:20:31.293659+0000","last_undegraded":"2023-10-09T06:20:31.293659+0000","last_fullsized":"2023-10-09T06:20:31.293659+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":158,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275792+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:20:31.275792+0000","last_peered":"2023-10-09T06:20:31.275792+0000","last_clean":"2023-10-09T06:20:31.275792+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:20:31.275792+0000","last_undegraded":"2023-10-09T06:20:31.275792+0000","last_fullsized":"2023-10-09T06:20:31.275792+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245602+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:20:41.245602+0000","last_peered":"2023-10-09T06:20:41.245602+0000","last_clean":"2023-10-09T06:20:41.245602+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:20:41.245602+0000","last_undegraded":"2023-10-09T06:20:41.245602+0000","last_fullsized":"2023-10-09T06:20:41.245602+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"202'8772","reported_seq":10426,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.491056+0000","last_change":"2023-10-09T06:19:34.580478+0000","last_active":"2023-10-09T06:20:43.491056+0000","last_peered":"2023-10-09T06:20:43.491056+0000","last_clean":"2023-10-09T06:20:43.491056+0000","last_became_active":"2023-10-09T06:19:34.573414+0000","last_became_peered":"2023-10-09T06:19:34.573414+0000","last_unstale":"2023-10-09T06:20:43.491056+0000","last_undegraded":"2023-10-09T06:20:43.491056+0000","last_fullsized":"2023-10-09T06:20:43.491056+0000","mapping_epoch":176,"log_start":"156'1600","ondisk_log_start":"156'1600","created":152,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7172,"log_dups_size":0,"ondisk_log_size":7172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:08.324993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8772,"num_write_kb":383808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":51380224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":186,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.299089+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:20:31.299089+0000","last_peered":"2023-10-09T06:20:31.299089+0000","last_clean":"2023-10-09T06:20:31.299089+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:20:31.299089+0000","last_undegraded":"2023-10-09T06:20:31.299089+0000","last_fullsized":"2023-10-09T06:20:31.299089+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":156,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261351+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:20:31.261351+0000","last_peered":"2023-10-09T06:20:31.261351+0000","last_clean":"2023-10-09T06:20:31.261351+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:20:31.261351+0000","last_undegraded":"2023-10-09T06:20:31.261351+0000","last_fullsized":"2023-10-09T06:20:31.261351+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.263533+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:20:31.263533+0000","last_peered":"2023-10-09T06:20:31.263533+0000","last_clean":"2023-10-09T06:20:31.263533+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:20:31.263533+0000","last_undegraded":"2023-10-09T06:20:31.263533+0000","last_fullsized":"2023-10-09T06:20:31.263533+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246142+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:20:41.246142+0000","last_peered":"2023-10-09T06:20:41.246142+0000","last_clean":"2023-10-09T06:20:41.246142+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:20:41.246142+0000","last_undegraded":"2023-10-09T06:20:41.246142+0000","last_fullsized":"2023-10-09T06:20:41.246142+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":168,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.300762+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:20:31.300762+0000","last_peered":"2023-10-09T06:20:31.300762+0000","last_clean":"2023-10-09T06:20:31.300762+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:20:31.300762+0000","last_undegraded":"2023-10-09T06:20:31.300762+0000","last_fullsized":"2023-10-09T06:20:31.300762+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274279+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:20:31.274279+0000","last_peered":"2023-10-09T06:20:31.274279+0000","last_clean":"2023-10-09T06:20:31.274279+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:20:31.274279+0000","last_undegraded":"2023-10-09T06:20:31.274279+0000","last_fullsized":"2023-10-09T06:20:31.274279+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.262973+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:20:31.262973+0000","last_peered":"2023-10-09T06:20:31.262973+0000","last_clean":"2023-10-09T06:20:31.262973+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:20:31.262973+0000","last_undegraded":"2023-10-09T06:20:31.262973+0000","last_fullsized":"2023-10-09T06:20:31.262973+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":147,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276135+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:20:31.276135+0000","last_peered":"2023-10-09T06:20:31.276135+0000","last_clean":"2023-10-09T06:20:31.276135+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:20:31.276135+0000","last_undegraded":"2023-10-09T06:20:31.276135+0000","last_fullsized":"2023-10-09T06:20:31.276135+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.262940+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:20:31.262940+0000","last_peered":"2023-10-09T06:20:31.262940+0000","last_clean":"2023-10-09T06:20:31.262940+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:20:31.262940+0000","last_undegraded":"2023-10-09T06:20:31.262940+0000","last_fullsized":"2023-10-09T06:20:31.262940+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":158,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.244776+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:20:41.244776+0000","last_peered":"2023-10-09T06:20:41.244776+0000","last_clean":"2023-10-09T06:20:41.244776+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:20:41.244776+0000","last_undegraded":"2023-10-09T06:20:41.244776+0000","last_fullsized":"2023-10-09T06:20:41.244776+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289784+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:20:31.289784+0000","last_peered":"2023-10-09T06:20:31.289784+0000","last_clean":"2023-10-09T06:20:31.289784+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:20:31.289784+0000","last_undegraded":"2023-10-09T06:20:31.289784+0000","last_fullsized":"2023-10-09T06:20:31.289784+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":157,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.248400+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:20:41.248400+0000","last_peered":"2023-10-09T06:20:41.248400+0000","last_clean":"2023-10-09T06:20:41.248400+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:20:41.248400+0000","last_undegraded":"2023-10-09T06:20:41.248400+0000","last_fullsized":"2023-10-09T06:20:41.248400+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.262731+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:20:31.262731+0000","last_peered":"2023-10-09T06:20:31.262731+0000","last_clean":"2023-10-09T06:20:31.262731+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:20:31.262731+0000","last_undegraded":"2023-10-09T06:20:31.262731+0000","last_fullsized":"2023-10-09T06:20:31.262731+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3344826392,"num_objects":51039,"num_object_clones":0,"num_object_copies":153117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51039,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144101,"num_write_kb":6244417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8419,"num_bytes_recovered":551747584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10170216448,"data_stored":10170204232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129801,"ondisk_log_size":129801,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234281,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2781936,"kb_used_data":1873512,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":90963216,"statfs":{"total":95995035648,"available":93146333184,"internally_reserved":0,"allocated":1918476288,"data_stored":1917797093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.071,"15min":1.278},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.3600000000000003,"5min":10.388,"15min":11.785},"last":0.376},{"interface":"front","average":{"1min":1.494,"5min":1.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.4500000000000002,"5min":10.396000000000001,"15min":10.784000000000001},"last":0.32100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.9569999999999999,"15min":2.093},"min":{"1min":0.28999999999999998,"5min":0.25,"15min":0.25},"max":{"1min":17.774000000000001,"5min":31.869,"15min":31.869},"last":0.33700000000000002},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.181,"15min":1.7569999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.747,"5min":29.486999999999998,"15min":29.486999999999998},"last":0.50600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.51,"15min":1.8700000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.3559999999999999,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.6020000000000001,"15min":1.9970000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.4189999999999996,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":1.05,"15min":0.74099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.1240000000000006,"5min":8.1240000000000006,"15min":8.1240000000000006},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.0349999999999999,"15min":0.76100000000000001},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.312},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":3.2719999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.575,"15min":1.339},"min":{"1min":0.32200000000000001,"5min":0.28299999999999997,"15min":0.251},"max":{"1min":25.678999999999998,"5min":25.678999999999998,"15min":25.678999999999998},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.018,"15min":1.2450000000000001},"min":{"1min":0.32800000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":8.4359999999999999,"5min":8.4359999999999999,"15min":9.234},"last":0.434}]},{"osd":5,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":5.6619999999999999,"5min":2.3180000000000001,"15min":1.6259999999999999},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":50.304000000000002,"5min":50.304000000000002,"15min":50.304000000000002},"last":3.2509999999999999},{"interface":"front","average":{"1min":4.8899999999999997,"5min":2.0219999999999998,"15min":1.413},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":48.774999999999999,"5min":48.774999999999999,"15min":48.774999999999999},"last":0.754}]},{"osd":6,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.3420000000000001,"15min":1.2969999999999999},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":5.2830000000000004,"5min":11.878,"15min":11.878},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.081,"15min":1.2929999999999999},"min":{"1min":0.38800000000000001,"5min":0.374,"15min":0.28599999999999998},"max":{"1min":5.7229999999999999,"5min":7.0380000000000003,"15min":9.2050000000000001},"last":1.544}]}]},{"osd":6,"up_from":46,"seq":197568495918,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":931352,"kb_used_data":629968,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":92813800,"statfs":{"total":95995035648,"available":95041331200,"internally_reserved":0,"allocated":645087232,"data_stored":644431855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:41 2023","interfaces":[{"interface":"back","average":{"1min":13.701000000000001,"5min":3.6930000000000001,"15min":2.335},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.214},"max":{"1min":109.092,"5min":109.092,"15min":109.092},"last":0.40200000000000002},{"interface":"front","average":{"1min":14.138999999999999,"5min":4.048,"15min":2.4660000000000002},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":109.369,"5min":109.369,"15min":109.369},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:20:11 2023","interfaces":[{"interface":"back","average":{"1min":16.698,"5min":5.1689999999999996,"15min":3.2919999999999998},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":114.202,"5min":114.202,"15min":114.202},"last":7.0789999999999997},{"interface":"front","average":{"1min":15.648999999999999,"5min":5.0220000000000002,"15min":3.2719999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.246},"max":{"1min":114.179,"5min":114.179,"15min":114.179},"last":7.0899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:20:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.7410000000000001,"15min":2.2410000000000001},"min":{"1min":0.317,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.016,"5min":1.3939999999999999,"15min":2.2610000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.734,"5min":8.9870000000000001,"15min":8.9870000000000001},"last":0.56499999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:19:46 2023","interfaces":[{"interface":"back","average":{"1min":6.9539999999999997,"5min":1.9259999999999999,"15min":1.002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":64.012,"5min":64.012,"15min":64.012},"last":8.3059999999999992},{"interface":"front","average":{"1min":6.3280000000000003,"5min":1.8109999999999999,"15min":0.97799999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":64.105999999999995,"5min":64.105999999999995,"15min":64.105999999999995},"last":15.507999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":1.474,"15min":1.1659999999999999},"min":{"1min":0.33400000000000002,"5min":0.252,"15min":0.245},"max":{"1min":17.643999999999998,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.159,"5min":1.075,"15min":1.2030000000000001},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.26400000000000001},"max":{"1min":3.6549999999999998,"5min":8.9239999999999995,"15min":9.4100000000000001},"last":0.746}]},{"osd":5,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.514,"15min":1.29},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.2410000000000001,"15min":1.827},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":8.0630000000000006,"5min":9.0169999999999995,"15min":159.06800000000001},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0169999999999999,"15min":1.071},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":4.2640000000000002,"5min":10.962,"15min":14.837},"last":0.372},{"interface":"front","average":{"1min":1.008,"5min":1.177,"15min":1.393},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.9939999999999998,"5min":8.9920000000000009,"15min":10.891},"last":0.59099999999999997}]}]},{"osd":1,"up_from":110,"seq":472446402655,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128684,"kb_used_data":1767268,"kb_used_omap":4,"kb_used_meta":361019,"kb_avail":91616468,"statfs":{"total":95995035648,"available":93815263232,"internally_reserved":0,"allocated":1809682432,"data_stored":1808772841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":369684366},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":7.2290000000000001,"5min":2.5230000000000001,"15min":2.661},"min":{"1min":0.20799999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":148.98400000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.4500000000000002,"15min":2.948},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20100000000000001},"max":{"1min":149.02699999999999,"5min":149.02699999999999,"15min":149.02699999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.9219999999999999,"15min":1.796},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":7.3399999999999999,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.309},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.9350000000000001,"15min":2.5619999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":4.9690000000000003,"5min":49.783000000000001,"15min":49.783000000000001},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:54 2023","interfaces":[{"interface":"back","average":{"1min":10.465,"5min":2.6920000000000002,"15min":1.1519999999999999},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":153.18199999999999,"5min":153.18199999999999,"15min":153.18199999999999},"last":2.0710000000000002},{"interface":"front","average":{"1min":10.079000000000001,"5min":2.5569999999999999,"15min":1.1240000000000001},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":152.52000000000001,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.48499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":10.198,"5min":3.0299999999999998,"15min":3.0430000000000001},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":199.43799999999999,"5min":199.43799999999999,"15min":199.43799999999999},"last":1.9430000000000001},{"interface":"front","average":{"1min":8.7400000000000002,"5min":2.956,"15min":2.9940000000000002},"min":{"1min":0.39000000000000001,"5min":0.315,"15min":0.311},"max":{"1min":199.32900000000001,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.58799999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.6619999999999999,"15min":2.9060000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":61.930999999999997,"5min":61.930999999999997,"15min":61.930999999999997},"last":2.5449999999999999},{"interface":"front","average":{"1min":3.2069999999999999,"5min":1.958,"15min":2.879},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":61.947000000000003,"5min":61.947000000000003,"15min":61.947000000000003},"last":2.5579999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.8380000000000001,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":4.274,"5min":12.91,"15min":12.91},"last":4.4290000000000003},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.7629999999999999,"15min":2.3769999999999998},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":4.4420000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.772,"15min":2.5139999999999998},"min":{"1min":0.378,"5min":0.376,"15min":0.376},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":3.169},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.3819999999999999,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":6.7270000000000003,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":3.1579999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024088,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1272852,"kb_used_data":823500,"kb_used_omap":5,"kb_used_meta":449338,"kb_avail":92472300,"statfs":{"total":95995035648,"available":94691635200,"internally_reserved":0,"allocated":843264000,"data_stored":842204290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":460122746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:20:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5099999999999998,"5min":2.133,"15min":2.0139999999999998},"min":{"1min":0.26600000000000001,"5min":0.19,"15min":0.19},"max":{"1min":29.021999999999998,"5min":30.733000000000001,"15min":30.733000000000001},"last":0.629},{"interface":"front","average":{"1min":3.3849999999999998,"5min":2.6389999999999998,"15min":2.7040000000000002},"min":{"1min":0.33700000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":29.077999999999999,"5min":30.279,"15min":30.279},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.762,"15min":2.3730000000000002},"min":{"1min":0.248,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":9.9909999999999997,"5min":17.588999999999999,"15min":17.588999999999999},"last":9.1110000000000007},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.3889999999999998},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":11.162000000000001,"5min":21.198,"15min":21.198},"last":0.45300000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:50 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":1.4750000000000001,"15min":0.747},"min":{"1min":0.33200000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":30.123000000000001,"5min":30.123000000000001,"15min":30.123000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":4.851,"5min":1.522,"15min":0.77400000000000002},"min":{"1min":0.34300000000000003,"5min":0.219,"15min":0.219},"max":{"1min":29.221,"5min":29.221,"15min":29.221},"last":0.46400000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.1819999999999999,"15min":1.647},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":4.3739999999999997,"5min":11.162000000000001,"15min":15.835000000000001},"last":3.0259999999999998},{"interface":"front","average":{"1min":2.137,"5min":1.8129999999999999,"15min":2.133},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":7.6210000000000004,"5min":12.752000000000001,"15min":15.023},"last":0.36599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.6060000000000001,"15min":1.895},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.567,"5min":11.222,"15min":15.858000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.589,"5min":2.0350000000000001,"15min":2.226},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":9.6400000000000006,"5min":21.167000000000002,"15min":31.984999999999999},"last":0.32800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":2.1509999999999998,"15min":2.2069999999999999},"min":{"1min":0.45700000000000002,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":9.6880000000000006,"5min":21.088999999999999,"15min":32.079000000000001},"last":0.432},{"interface":"front","average":{"1min":2.3210000000000002,"5min":1.542,"15min":1.978},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":10.606,"5min":11.236000000000001,"15min":15.893000000000001},"last":3.431}]},{"osd":7,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.796,"15min":2.2080000000000002},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":9.9269999999999996,"5min":12.798999999999999,"15min":15.050000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.974,"15min":2.2549999999999999},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":8.2219999999999995,"5min":21.135999999999999,"15min":32.055999999999997},"last":3.2429999999999999}]}]},{"osd":2,"up_from":128,"seq":549755813957,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833168,"kb_used_data":597832,"kb_used_omap":7,"kb_used_meta":235320,"kb_avail":92911984,"statfs":{"total":95995035648,"available":95141871616,"internally_reserved":0,"allocated":612179968,"data_stored":611295045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":240967761},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":5.3449999999999998,"15min":3.0710000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":88.225999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":4.1680000000000001},{"interface":"front","average":{"1min":5.8730000000000002,"5min":5.4880000000000004,"15min":3.6520000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":85.781000000000006,"5min":163.363,"15min":163.363},"last":0.25}]},{"osd":1,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.29,"15min":2.6669999999999998},"min":{"1min":0.24299999999999999,"5min":0.188,"15min":0.188},"max":{"1min":4.5679999999999996,"5min":222.50299999999999,"15min":222.50299999999999},"last":4.3120000000000003},{"interface":"front","average":{"1min":0.86099999999999999,"5min":4.3620000000000001,"15min":3.2959999999999998},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8069999999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":3.7850000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":6.6619999999999999,"5min":2.0470000000000002,"15min":0.91700000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":81.183999999999997,"5min":81.183999999999997,"15min":81.183999999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":6.4820000000000002,"5min":1.8520000000000001,"15min":0.89800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":81.061999999999998,"5min":81.061999999999998,"15min":81.061999999999998},"last":10.042999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.532,"15min":1.97},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.106999999999999,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.7599999999999998,"5min":1.6870000000000001,"15min":2.2309999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.994999999999999,"5min":15.183,"15min":15.183},"last":0.33000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":1.3089999999999999,"15min":1.6499999999999999},"min":{"1min":0.36399999999999999,"5min":0.309,"15min":0.309},"max":{"1min":4.9989999999999997,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9079999999999999,"15min":2.3919999999999999},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.6840000000000002,"5min":10.215999999999999,"15min":10.215999999999999},"last":0.36499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.5109999999999999,"15min":2.1259999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.4939999999999998,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.4119999999999999,"15min":1.6140000000000001},"min":{"1min":0.39300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.883,"5min":7.327,"15min":7.327},"last":4.2809999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.583,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.6660000000000004,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":4.3360000000000003},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.7310000000000001,"15min":2.7549999999999999},"min":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.7779999999999996,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":0.38100000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127154,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1924392,"kb_used_data":1694240,"kb_used_omap":2,"kb_used_meta":230077,"kb_avail":91820760,"statfs":{"total":95995035648,"available":94024458240,"internally_reserved":0,"allocated":1734901760,"data_stored":1734111776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":235598901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":0.88300000000000001,"15min":0.70699999999999996},"min":{"1min":0.28399999999999997,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.2729999999999997,"5min":9.2729999999999997,"15min":9.2729999999999997},"last":3.9300000000000002},{"interface":"front","average":{"1min":2.839,"5min":1.415,"15min":1.103},"min":{"1min":0.21099999999999999,"5min":0.191,"15min":0.191},"max":{"1min":9.8070000000000004,"5min":11.696,"15min":11.696},"last":0.61299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9580000000000002,"5min":1.5269999999999999,"15min":1.141},"min":{"1min":0.26700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":12.387,"5min":12.387,"15min":12.387},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.2349999999999999,"5min":1.2310000000000001,"15min":0.81699999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.757999999999999,"5min":12.757999999999999,"15min":12.757999999999999},"last":4.5869999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":1.385,"15min":1.091},"min":{"1min":0.30199999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.9930000000000003,"5min":11.667999999999999,"15min":11.667999999999999},"last":4.1760000000000002},{"interface":"front","average":{"1min":1.8120000000000001,"5min":0.75800000000000001,"15min":0.53300000000000003},"min":{"1min":0.316,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.2230000000000008,"5min":9.2230000000000008,"15min":9.2230000000000008},"last":3.9929999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":0.86899999999999999,"15min":0.70099999999999996},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.1760000000000002,"5min":9.1760000000000002,"15min":9.1760000000000002},"last":3.9550000000000001},{"interface":"front","average":{"1min":1.306,"5min":1.2050000000000001,"15min":1.196},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.5919999999999996,"5min":11.742000000000001,"15min":11.742000000000001},"last":0.88300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":1.397,"15min":1.2809999999999999},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.5519999999999996,"5min":11.772,"15min":11.772},"last":0.93300000000000005},{"interface":"front","average":{"1min":2.7799999999999998,"5min":1.524,"15min":1.3260000000000001},"min":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.8989999999999991,"5min":11.598000000000001,"15min":11.598000000000001},"last":4.1139999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":1.3520000000000001,"15min":1.2470000000000001},"min":{"1min":0.38800000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.6550000000000002,"5min":11.798,"15min":11.798},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.452,"15min":1.282},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":9.9640000000000004,"5min":11.552,"15min":11.552},"last":4.1529999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":0.99199999999999999,"15min":0.75800000000000001},"min":{"1min":0.41799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.0899999999999999,"5min":9.0899999999999999,"15min":9.0899999999999999},"last":3.883},{"interface":"front","average":{"1min":1.5,"5min":1.3049999999999999,"15min":1.2450000000000001},"min":{"1min":0.44400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":7.6260000000000003,"5min":11.823,"15min":11.823},"last":0.90900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019190,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266464,"kb_used_data":668504,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":92478688,"statfs":{"total":95995035648,"available":94698176512,"internally_reserved":0,"allocated":684548096,"data_stored":683884382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.4370000000000001,"15min":1.4670000000000001},"min":{"1min":0.26000000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":19.577000000000002,"5min":19.577000000000002,"15min":19.577000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.0830000000000002,"5min":1.7430000000000001,"15min":1.837},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":19.594999999999999,"5min":19.594999999999999,"15min":19.594999999999999},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.5369999999999999,"15min":1.9470000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":34.412999999999997,"5min":91.849999999999994,"15min":91.849999999999994},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.71,"5min":2.7210000000000001,"15min":2},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":34.604999999999997,"5min":91.885000000000005,"15min":91.885000000000005},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0960000000000001,"5min":1.524,"15min":1.778},"min":{"1min":0.316,"5min":0.255,"15min":0.255},"max":{"1min":6.8650000000000002,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.831,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.898,"5min":15.894,"15min":15.894},"last":0.498}]},{"osd":3,"last update":"Mon Oct 9 06:19:48 2023","interfaces":[{"interface":"back","average":{"1min":3.8330000000000002,"5min":2.2269999999999999,"15min":1.117},"min":{"1min":0.45000000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":12.598000000000001,"5min":91.697000000000003,"15min":91.697000000000003},"last":57.826999999999998},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.9530000000000001,"15min":1.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.6920000000000002,"5min":94.356999999999999,"15min":94.356999999999999},"last":57.853000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.1639999999999999,"15min":1.0629999999999999},"min":{"1min":0.26300000000000001,"5min":0.254,"15min":0.24299999999999999},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":11.130000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.2929999999999999,"15min":1.353},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":12.552,"5min":12.552,"15min":12.552},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.70399999999999996,"15min":0.89200000000000002},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.58999999999999997,"5min":4.907,"15min":9.7680000000000007},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.4099999999999999,"15min":1.212},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":7.5380000000000003,"5min":14.202999999999999,"15min":14.202999999999999},"last":0.41099999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:19:56 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":1.548,"15min":1.2270000000000001},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":49.648000000000003,"5min":49.648000000000003,"15min":49.648000000000003},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.956,"5min":1.7509999999999999,"15min":1.2829999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":49.750999999999998,"5min":49.750999999999998,"15min":49.750999999999998},"last":0.42799999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757555,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2838612,"kb_used_data":1944972,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":90906540,"statfs":{"total":95995035648,"available":93088296960,"internally_reserved":0,"allocated":1991651328,"data_stored":1990995612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.6599999999999999,"15min":1.675},"min":{"1min":0.216,"5min":0.216,"15min":0.215},"max":{"1min":6.8159999999999998,"5min":8.5139999999999993,"15min":18.637},"last":4.1900000000000004},{"interface":"front","average":{"1min":1.909,"5min":1.4690000000000001,"15min":1.6799999999999999},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.1790000000000003,"5min":8.4019999999999992,"15min":9.9939999999999998},"last":3.7930000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.423,"15min":2.125},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.8520000000000003,"5min":8.8170000000000002,"15min":11.292},"last":4.2270000000000003},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.645,"15min":2.0859999999999999},"min":{"1min":0.437,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.467000000000001,"5min":10.467000000000001,"15min":10.467000000000001},"last":3.2160000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6339999999999999,"15min":2.2029999999999998},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.307},"max":{"1min":10.42,"5min":10.42,"15min":10.42},"last":3.2469999999999999},{"interface":"front","average":{"1min":1.946,"5min":1.4259999999999999,"15min":2.044},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.5990000000000002,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":5.5430000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":4.444,"15min":1.8979999999999999},"min":{"1min":0.376,"5min":0.314,"15min":0.314},"max":{"1min":6.4450000000000003,"5min":190.411,"15min":190.411},"last":4.444},{"interface":"front","average":{"1min":2.0169999999999999,"5min":4.4160000000000004,"15min":1.8919999999999999},"min":{"1min":0.34699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9980000000000002,"5min":190.70699999999999,"15min":190.70699999999999},"last":3.601}]},{"osd":4,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.4490000000000001,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":8.6389999999999993,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":4.3840000000000003},{"interface":"front","average":{"1min":1.704,"5min":1.117,"15min":1.0660000000000001},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.25900000000000001},"max":{"1min":5.0570000000000004,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":2.4119999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5590000000000002,"5min":1.593,"15min":1.304},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.28899999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":2.4470000000000001},{"interface":"front","average":{"1min":3.2589999999999999,"5min":1.5109999999999999,"15min":1.2190000000000001},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":3.3940000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:19:54 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.4570000000000001,"15min":1.1399999999999999},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.6029999999999998,"5min":8.8409999999999993,"15min":8.8409999999999993},"last":4.4119999999999999},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.2150000000000001,"15min":1.103},"min":{"1min":0.40100000000000002,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":5.4290000000000003,"5min":7.7809999999999997,"15min":9.4580000000000002},"last":4.3310000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":834404352,"data_stored":834404352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1805258752,"data_stored":1805254680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":607059968,"data_stored":607059968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1724776448,"data_stored":1724776448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":671547392,"data_stored":671547392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1983713280,"data_stored":1983709208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":634388480,"data_stored":634388480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1909067776,"data_stored":1909063704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:20:49.043 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:20:49.082 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:49.386 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:49.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:49.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:49.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:49.558 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:49.635 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:20:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:49.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:20:49.748 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:49.832 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.073 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:50.181 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:50.249 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.354 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:50.463 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:50.516 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.609 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:20:50.623 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:50.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:50.711 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:50.732 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:50.804 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:50.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:50 smithi107 ceph-mon[212750]: pgmap v654: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:20:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:50 smithi107 ceph-mon[220921]: pgmap v654: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:20:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:50.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:50 smithi150 ceph-mon[194178]: pgmap v654: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-09T06:20:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:20:51.273 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:51.331 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:20:51.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:51.464 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[212750]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.694 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.695 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:51 smithi107 ceph-mon[220921]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.736 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:51.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:51.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1973422754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: osdmap e203: 8 total, 8 up, 8 in 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:51 smithi150 ceph-mon[194178]: failed to encode map e203 with expected crc 2023-10-09T06:20:51.909 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.040 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:20:52.041 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:20:52.284 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:20:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:20:52] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:20:52.292 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:52.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:52.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:52.424 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.467 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:52.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:52.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:52.612 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:52 smithi107 ceph-mon[212750]: pgmap v656: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:20:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:52 smithi107 ceph-mon[220921]: pgmap v656: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:20:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:52 smithi150 ceph-mon[194178]: pgmap v656: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-09T06:20:53.188 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:53.296 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:53.335 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:53.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:20:53.416 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:53.476 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:20:53.489 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:53.524 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:53.640 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:53.969 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:53.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:54.085 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:54.136 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:54.226 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:54.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:54.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:20:54.427 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:54.442 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:20:54.457 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:20:54.679 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:54 smithi107 ceph-mon[212750]: pgmap v657: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:20:54.924 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:20:54.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:54.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:20:55.030 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:55.044 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:54 smithi107 ceph-mon[220921]: pgmap v657: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:20:55.091 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:54 smithi150 ceph-mon[194178]: pgmap v657: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-09T06:20:55.098 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.170 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:55.276 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:55.344 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.436 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:55.553 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:20:55.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.612 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":657,"stamp":"2023-10-09T06:20:52.996460+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3076790200,"num_objects":46964,"num_object_clones":0,"num_object_copies":140892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46964,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":148500,"num_write_kb":6246862,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8446,"num_bytes_recovered":554709108,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132616,"ondisk_log_size":132616,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13047948,"kb_used_data":9091340,"kb_used_omap":20,"kb_used_meta":3956523,"kb_avail":736913268,"statfs":{"total":767960285184,"available":754599186432,"internally_reserved":0,"allocated":9309532160,"data_stored":9303233584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4051480206},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,2,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-457768960,"num_objects":-6985,"num_object_clones":0,"num_object_copies":-20955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6985,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009244"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246093+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:20:41.246093+0000","last_peered":"2023-10-09T06:20:41.246093+0000","last_clean":"2023-10-09T06:20:41.246093+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:20:41.246093+0000","last_undegraded":"2023-10-09T06:20:41.246093+0000","last_fullsized":"2023-10-09T06:20:41.246093+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":137,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293350+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:20:31.293350+0000","last_peered":"2023-10-09T06:20:31.293350+0000","last_clean":"2023-10-09T06:20:31.293350+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:20:31.293350+0000","last_undegraded":"2023-10-09T06:20:31.293350+0000","last_fullsized":"2023-10-09T06:20:31.293350+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269009+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:20:41.269009+0000","last_peered":"2023-10-09T06:20:41.269009+0000","last_clean":"2023-10-09T06:20:41.269009+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:20:41.269009+0000","last_undegraded":"2023-10-09T06:20:41.269009+0000","last_fullsized":"2023-10-09T06:20:41.269009+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261422+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:20:31.261422+0000","last_peered":"2023-10-09T06:20:31.261422+0000","last_clean":"2023-10-09T06:20:31.261422+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:20:31.261422+0000","last_undegraded":"2023-10-09T06:20:31.261422+0000","last_fullsized":"2023-10-09T06:20:31.261422+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293707+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:20:31.293707+0000","last_peered":"2023-10-09T06:20:31.293707+0000","last_clean":"2023-10-09T06:20:31.293707+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:20:31.293707+0000","last_undegraded":"2023-10-09T06:20:31.293707+0000","last_fullsized":"2023-10-09T06:20:31.293707+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252991+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:20:41.252991+0000","last_peered":"2023-10-09T06:20:41.252991+0000","last_clean":"2023-10-09T06:20:41.252991+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:20:41.252991+0000","last_undegraded":"2023-10-09T06:20:41.252991+0000","last_fullsized":"2023-10-09T06:20:41.252991+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261884+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:20:31.261884+0000","last_peered":"2023-10-09T06:20:31.261884+0000","last_clean":"2023-10-09T06:20:31.261884+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:20:31.261884+0000","last_undegraded":"2023-10-09T06:20:31.261884+0000","last_fullsized":"2023-10-09T06:20:31.261884+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":137,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293785+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:20:31.293785+0000","last_peered":"2023-10-09T06:20:31.293785+0000","last_clean":"2023-10-09T06:20:31.293785+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:20:31.293785+0000","last_undegraded":"2023-10-09T06:20:31.293785+0000","last_fullsized":"2023-10-09T06:20:31.293785+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275380+0000","last_change":"2023-10-09T06:19:06.673434+0000","last_active":"2023-10-09T06:20:31.275380+0000","last_peered":"2023-10-09T06:20:31.275380+0000","last_clean":"2023-10-09T06:20:31.275380+0000","last_became_active":"2023-10-09T06:19:06.673042+0000","last_became_peered":"2023-10-09T06:19:06.673042+0000","last_unstale":"2023-10-09T06:20:31.275380+0000","last_undegraded":"2023-10-09T06:20:31.275380+0000","last_fullsized":"2023-10-09T06:20:31.275380+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:30.387567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289571+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:20:31.289571+0000","last_peered":"2023-10-09T06:20:31.289571+0000","last_clean":"2023-10-09T06:20:31.289571+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:20:31.289571+0000","last_undegraded":"2023-10-09T06:20:31.289571+0000","last_fullsized":"2023-10-09T06:20:31.289571+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263033+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:20:41.263033+0000","last_peered":"2023-10-09T06:20:41.263033+0000","last_clean":"2023-10-09T06:20:41.263033+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:20:41.263033+0000","last_undegraded":"2023-10-09T06:20:41.263033+0000","last_fullsized":"2023-10-09T06:20:41.263033+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.270425+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:20:41.270425+0000","last_peered":"2023-10-09T06:20:41.270425+0000","last_clean":"2023-10-09T06:20:41.270425+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:20:41.270425+0000","last_undegraded":"2023-10-09T06:20:41.270425+0000","last_fullsized":"2023-10-09T06:20:41.270425+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":209,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246393+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:20:41.246393+0000","last_peered":"2023-10-09T06:20:41.246393+0000","last_clean":"2023-10-09T06:20:41.246393+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:20:41.246393+0000","last_undegraded":"2023-10-09T06:20:41.246393+0000","last_fullsized":"2023-10-09T06:20:41.246393+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247289+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:20:41.247289+0000","last_peered":"2023-10-09T06:20:41.247289+0000","last_clean":"2023-10-09T06:20:41.247289+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:20:41.247289+0000","last_undegraded":"2023-10-09T06:20:41.247289+0000","last_fullsized":"2023-10-09T06:20:41.247289+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262452+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:20:41.262452+0000","last_peered":"2023-10-09T06:20:41.262452+0000","last_clean":"2023-10-09T06:20:41.262452+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:20:41.262452+0000","last_undegraded":"2023-10-09T06:20:41.262452+0000","last_fullsized":"2023-10-09T06:20:41.262452+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293162+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:20:31.293162+0000","last_peered":"2023-10-09T06:20:31.293162+0000","last_clean":"2023-10-09T06:20:31.293162+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:20:31.293162+0000","last_undegraded":"2023-10-09T06:20:31.293162+0000","last_fullsized":"2023-10-09T06:20:31.293162+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261012+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:20:31.261012+0000","last_peered":"2023-10-09T06:20:31.261012+0000","last_clean":"2023-10-09T06:20:31.261012+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:20:31.261012+0000","last_undegraded":"2023-10-09T06:20:31.261012+0000","last_fullsized":"2023-10-09T06:20:31.261012+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.275987+0000","last_change":"2023-10-09T06:19:06.758564+0000","last_active":"2023-10-09T06:20:31.275987+0000","last_peered":"2023-10-09T06:20:31.275987+0000","last_clean":"2023-10-09T06:20:31.275987+0000","last_became_active":"2023-10-09T06:19:06.758270+0000","last_became_peered":"2023-10-09T06:19:06.758270+0000","last_unstale":"2023-10-09T06:20:31.275987+0000","last_undegraded":"2023-10-09T06:20:31.275987+0000","last_fullsized":"2023-10-09T06:20:31.275987+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.823391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262191+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:20:41.262191+0000","last_peered":"2023-10-09T06:20:41.262191+0000","last_clean":"2023-10-09T06:20:41.262191+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:20:41.262191+0000","last_undegraded":"2023-10-09T06:20:41.262191+0000","last_fullsized":"2023-10-09T06:20:41.262191+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261554+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:20:31.261554+0000","last_peered":"2023-10-09T06:20:31.261554+0000","last_clean":"2023-10-09T06:20:31.261554+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:20:31.261554+0000","last_undegraded":"2023-10-09T06:20:31.261554+0000","last_fullsized":"2023-10-09T06:20:31.261554+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":171,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.255295+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:20:41.255295+0000","last_peered":"2023-10-09T06:20:41.255295+0000","last_clean":"2023-10-09T06:20:41.255295+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:20:41.255295+0000","last_undegraded":"2023-10-09T06:20:41.255295+0000","last_fullsized":"2023-10-09T06:20:41.255295+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293572+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:20:31.293572+0000","last_peered":"2023-10-09T06:20:31.293572+0000","last_clean":"2023-10-09T06:20:31.293572+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:20:31.293572+0000","last_undegraded":"2023-10-09T06:20:31.293572+0000","last_fullsized":"2023-10-09T06:20:31.293572+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"202'9086","reported_seq":10921,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:49.928096+0000","last_change":"2023-10-09T06:19:43.978924+0000","last_active":"2023-10-09T06:20:49.928096+0000","last_peered":"2023-10-09T06:20:49.928096+0000","last_clean":"2023-10-09T06:20:49.928096+0000","last_became_active":"2023-10-09T06:19:43.970383+0000","last_became_peered":"2023-10-09T06:19:43.970383+0000","last_unstale":"2023-10-09T06:20:49.928096+0000","last_undegraded":"2023-10-09T06:20:49.928096+0000","last_fullsized":"2023-10-09T06:20:49.928096+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9086,"log_dups_size":0,"ondisk_log_size":9086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:59.701538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195821568,"num_objects":2988,"num_object_clones":0,"num_object_copies":8964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9086,"num_write_kb":386368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":875,"num_bytes_recovered":57344000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263321+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:20:41.263321+0000","last_peered":"2023-10-09T06:20:41.263321+0000","last_clean":"2023-10-09T06:20:41.263321+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:20:41.263321+0000","last_undegraded":"2023-10-09T06:20:41.263321+0000","last_fullsized":"2023-10-09T06:20:41.263321+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.273941+0000","last_change":"2023-10-09T06:19:06.673411+0000","last_active":"2023-10-09T06:20:31.273941+0000","last_peered":"2023-10-09T06:20:31.273941+0000","last_clean":"2023-10-09T06:20:31.273941+0000","last_became_active":"2023-10-09T06:19:06.673113+0000","last_became_peered":"2023-10-09T06:19:06.673113+0000","last_unstale":"2023-10-09T06:20:31.273941+0000","last_undegraded":"2023-10-09T06:20:31.273941+0000","last_fullsized":"2023-10-09T06:20:31.273941+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:39.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"203'9307","reported_seq":9368,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:51.414377+0000","last_change":"2023-10-09T06:17:57.020187+0000","last_active":"2023-10-09T06:20:51.414377+0000","last_peered":"2023-10-09T06:20:51.414377+0000","last_clean":"2023-10-09T06:20:51.414377+0000","last_became_active":"2023-10-09T06:17:57.019892+0000","last_became_peered":"2023-10-09T06:17:57.019892+0000","last_unstale":"2023-10-09T06:20:51.414377+0000","last_undegraded":"2023-10-09T06:20:51.414377+0000","last_fullsized":"2023-10-09T06:20:51.414377+0000","mapping_epoch":152,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:01.403006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187498496,"num_objects":2861,"num_object_clones":0,"num_object_copies":8583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9307,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262711+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:20:41.262711+0000","last_peered":"2023-10-09T06:20:41.262711+0000","last_clean":"2023-10-09T06:20:41.262711+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:20:41.262711+0000","last_undegraded":"2023-10-09T06:20:41.262711+0000","last_fullsized":"2023-10-09T06:20:41.262711+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246196+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:20:41.246196+0000","last_peered":"2023-10-09T06:20:41.246196+0000","last_clean":"2023-10-09T06:20:41.246196+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:20:41.246196+0000","last_undegraded":"2023-10-09T06:20:41.246196+0000","last_fullsized":"2023-10-09T06:20:41.246196+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"202'9322","reported_seq":11057,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.129952+0000","last_change":"2023-10-09T06:20:00.751403+0000","last_active":"2023-10-09T06:20:48.129952+0000","last_peered":"2023-10-09T06:20:48.129952+0000","last_clean":"2023-10-09T06:20:48.129952+0000","last_became_active":"2023-10-09T06:20:00.740256+0000","last_became_peered":"2023-10-09T06:20:00.740256+0000","last_unstale":"2023-10-09T06:20:48.129952+0000","last_undegraded":"2023-10-09T06:20:48.129952+0000","last_fullsized":"2023-10-09T06:20:48.129952+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9322,"log_dups_size":0,"ondisk_log_size":9322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:18.856099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197787648,"num_objects":3018,"num_object_clones":0,"num_object_copies":9054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9322,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262850+0000","last_change":"2023-10-09T06:19:42.328723+0000","last_active":"2023-10-09T06:20:41.262850+0000","last_peered":"2023-10-09T06:20:41.262850+0000","last_clean":"2023-10-09T06:20:41.262850+0000","last_became_active":"2023-10-09T06:19:06.757480+0000","last_became_peered":"2023-10-09T06:19:06.757480+0000","last_unstale":"2023-10-09T06:20:41.262850+0000","last_undegraded":"2023-10-09T06:20:41.262850+0000","last_fullsized":"2023-10-09T06:20:41.262850+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:57.712635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252252+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:20:41.252252+0000","last_peered":"2023-10-09T06:20:41.252252+0000","last_clean":"2023-10-09T06:20:41.252252+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:20:41.252252+0000","last_undegraded":"2023-10-09T06:20:41.252252+0000","last_fullsized":"2023-10-09T06:20:41.252252+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"202'9314","reported_seq":9372,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.116716+0000","last_change":"2023-10-09T06:17:57.217507+0000","last_active":"2023-10-09T06:20:48.116716+0000","last_peered":"2023-10-09T06:20:48.116716+0000","last_clean":"2023-10-09T06:20:48.116716+0000","last_became_active":"2023-10-09T06:17:57.217183+0000","last_became_peered":"2023-10-09T06:17:57.217183+0000","last_unstale":"2023-10-09T06:20:48.116716+0000","last_undegraded":"2023-10-09T06:20:48.116716+0000","last_fullsized":"2023-10-09T06:20:48.116716+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9314,"log_dups_size":0,"ondisk_log_size":9314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:56.905622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200409088,"num_objects":3058,"num_object_clones":0,"num_object_copies":9174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":395904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":162,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274137+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:20:31.274137+0000","last_peered":"2023-10-09T06:20:31.274137+0000","last_clean":"2023-10-09T06:20:31.274137+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:20:31.274137+0000","last_undegraded":"2023-10-09T06:20:31.274137+0000","last_fullsized":"2023-10-09T06:20:31.274137+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.262264+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:20:31.262264+0000","last_peered":"2023-10-09T06:20:31.262264+0000","last_clean":"2023-10-09T06:20:31.262264+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:20:31.262264+0000","last_undegraded":"2023-10-09T06:20:31.262264+0000","last_fullsized":"2023-10-09T06:20:31.262264+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"202'9310","reported_seq":11004,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.609772+0000","last_change":"2023-10-09T06:20:30.273502+0000","last_active":"2023-10-09T06:20:48.609772+0000","last_peered":"2023-10-09T06:20:48.609772+0000","last_clean":"2023-10-09T06:20:48.609772+0000","last_became_active":"2023-10-09T06:20:30.253922+0000","last_became_peered":"2023-10-09T06:20:30.253922+0000","last_unstale":"2023-10-09T06:20:48.609772+0000","last_undegraded":"2023-10-09T06:20:48.609772+0000","last_fullsized":"2023-10-09T06:20:48.609772+0000","mapping_epoch":199,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:32.110407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193593344,"num_objects":2954,"num_object_clones":0,"num_object_copies":8862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9310,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":163,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289427+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:20:31.289427+0000","last_peered":"2023-10-09T06:20:31.289427+0000","last_clean":"2023-10-09T06:20:31.289427+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:20:31.289427+0000","last_undegraded":"2023-10-09T06:20:31.289427+0000","last_fullsized":"2023-10-09T06:20:31.289427+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246475+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:20:41.246475+0000","last_peered":"2023-10-09T06:20:41.246475+0000","last_clean":"2023-10-09T06:20:41.246475+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:20:41.246475+0000","last_undegraded":"2023-10-09T06:20:41.246475+0000","last_fullsized":"2023-10-09T06:20:41.246475+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"203'9161","reported_seq":9222,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:51.426116+0000","last_change":"2023-10-09T06:17:57.236716+0000","last_active":"2023-10-09T06:20:51.426116+0000","last_peered":"2023-10-09T06:20:51.426116+0000","last_clean":"2023-10-09T06:20:51.426116+0000","last_became_active":"2023-10-09T06:17:57.236363+0000","last_became_peered":"2023-10-09T06:17:57.236363+0000","last_unstale":"2023-10-09T06:20:51.426116+0000","last_undegraded":"2023-10-09T06:20:51.426116+0000","last_fullsized":"2023-10-09T06:20:51.426116+0000","mapping_epoch":152,"log_start":"156'1800","ondisk_log_start":"156'1800","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:53.348003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9161,"num_write_kb":385088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252368+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:20:41.252368+0000","last_peered":"2023-10-09T06:20:41.252368+0000","last_clean":"2023-10-09T06:20:41.252368+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:20:41.252368+0000","last_undegraded":"2023-10-09T06:20:41.252368+0000","last_fullsized":"2023-10-09T06:20:41.252368+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246078+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:20:41.246078+0000","last_peered":"2023-10-09T06:20:41.246078+0000","last_clean":"2023-10-09T06:20:41.246078+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:20:41.246078+0000","last_undegraded":"2023-10-09T06:20:41.246078+0000","last_fullsized":"2023-10-09T06:20:41.246078+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"203'9421","reported_seq":12995,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:52.949904+0000","last_change":"2023-10-09T06:20:21.206561+0000","last_active":"2023-10-09T06:20:52.949904+0000","last_peered":"2023-10-09T06:20:52.949904+0000","last_clean":"2023-10-09T06:20:52.949904+0000","last_became_active":"2023-10-09T06:20:21.163864+0000","last_became_peered":"2023-10-09T06:20:21.163864+0000","last_unstale":"2023-10-09T06:20:52.949904+0000","last_undegraded":"2023-10-09T06:20:52.949904+0000","last_fullsized":"2023-10-09T06:20:52.949904+0000","mapping_epoch":196,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7721,"log_dups_size":0,"ondisk_log_size":7721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:11.775536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9421,"num_write_kb":389952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1744,"num_bytes_recovered":114294784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":184,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246060+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:20:41.246060+0000","last_peered":"2023-10-09T06:20:41.246060+0000","last_clean":"2023-10-09T06:20:41.246060+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:20:41.246060+0000","last_undegraded":"2023-10-09T06:20:41.246060+0000","last_fullsized":"2023-10-09T06:20:41.246060+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247107+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:20:41.247107+0000","last_peered":"2023-10-09T06:20:41.247107+0000","last_clean":"2023-10-09T06:20:41.247107+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:20:41.247107+0000","last_undegraded":"2023-10-09T06:20:41.247107+0000","last_fullsized":"2023-10-09T06:20:41.247107+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"202'9044","reported_seq":9104,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.608406+0000","last_change":"2023-10-09T06:17:57.217463+0000","last_active":"2023-10-09T06:20:48.608406+0000","last_peered":"2023-10-09T06:20:48.608406+0000","last_clean":"2023-10-09T06:20:48.608406+0000","last_became_active":"2023-10-09T06:17:57.217100+0000","last_became_peered":"2023-10-09T06:17:57.217100+0000","last_unstale":"2023-10-09T06:20:48.608406+0000","last_undegraded":"2023-10-09T06:20:48.608406+0000","last_fullsized":"2023-10-09T06:20:48.608406+0000","mapping_epoch":152,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:23.024261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189136896,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9044,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293862+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:20:31.293862+0000","last_peered":"2023-10-09T06:20:31.293862+0000","last_clean":"2023-10-09T06:20:31.293862+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:20:31.293862+0000","last_undegraded":"2023-10-09T06:20:31.293862+0000","last_fullsized":"2023-10-09T06:20:31.293862+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":132,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274064+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:20:41.274064+0000","last_peered":"2023-10-09T06:20:41.274064+0000","last_clean":"2023-10-09T06:20:41.274064+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:20:41.274064+0000","last_undegraded":"2023-10-09T06:20:41.274064+0000","last_fullsized":"2023-10-09T06:20:41.274064+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"202'9135","reported_seq":9193,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.091702+0000","last_change":"2023-10-09T06:17:57.022383+0000","last_active":"2023-10-09T06:20:48.091702+0000","last_peered":"2023-10-09T06:20:48.091702+0000","last_clean":"2023-10-09T06:20:48.091702+0000","last_became_active":"2023-10-09T06:17:57.022128+0000","last_became_peered":"2023-10-09T06:17:57.022128+0000","last_unstale":"2023-10-09T06:20:48.091702+0000","last_undegraded":"2023-10-09T06:20:48.091702+0000","last_fullsized":"2023-10-09T06:20:48.091702+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9135,"log_dups_size":0,"ondisk_log_size":9135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.131013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192217088,"num_objects":2933,"num_object_clones":0,"num_object_copies":8799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9135,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":183,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274364+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:20:41.274364+0000","last_peered":"2023-10-09T06:20:41.274364+0000","last_clean":"2023-10-09T06:20:41.274364+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:20:41.274364+0000","last_undegraded":"2023-10-09T06:20:41.274364+0000","last_fullsized":"2023-10-09T06:20:41.274364+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276083+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:20:31.276083+0000","last_peered":"2023-10-09T06:20:31.276083+0000","last_clean":"2023-10-09T06:20:31.276083+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:20:31.276083+0000","last_undegraded":"2023-10-09T06:20:31.276083+0000","last_fullsized":"2023-10-09T06:20:31.276083+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269971+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:20:41.269971+0000","last_peered":"2023-10-09T06:20:41.269971+0000","last_clean":"2023-10-09T06:20:41.269971+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:20:41.269971+0000","last_undegraded":"2023-10-09T06:20:41.269971+0000","last_fullsized":"2023-10-09T06:20:41.269971+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"202'9303","reported_seq":11050,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:49.875641+0000","last_change":"2023-10-09T06:20:11.090804+0000","last_active":"2023-10-09T06:20:49.875641+0000","last_peered":"2023-10-09T06:20:49.875641+0000","last_clean":"2023-10-09T06:20:49.875641+0000","last_became_active":"2023-10-09T06:20:11.073402+0000","last_became_peered":"2023-10-09T06:20:11.073402+0000","last_unstale":"2023-10-09T06:20:49.875641+0000","last_undegraded":"2023-10-09T06:20:49.875641+0000","last_fullsized":"2023-10-09T06:20:49.875641+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9303,"log_dups_size":0,"ondisk_log_size":9303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:23.451973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193003520,"num_objects":2945,"num_object_clones":0,"num_object_copies":8835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9303,"num_write_kb":391936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":830,"num_bytes_recovered":54394880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274150+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:20:31.274150+0000","last_peered":"2023-10-09T06:20:31.274150+0000","last_clean":"2023-10-09T06:20:31.274150+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:20:31.274150+0000","last_undegraded":"2023-10-09T06:20:31.274150+0000","last_fullsized":"2023-10-09T06:20:31.274150+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247158+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:20:41.247158+0000","last_peered":"2023-10-09T06:20:41.247158+0000","last_clean":"2023-10-09T06:20:41.247158+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:20:41.247158+0000","last_undegraded":"2023-10-09T06:20:41.247158+0000","last_fullsized":"2023-10-09T06:20:41.247158+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274500+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:20:41.274500+0000","last_peered":"2023-10-09T06:20:41.274500+0000","last_clean":"2023-10-09T06:20:41.274500+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:20:41.274500+0000","last_undegraded":"2023-10-09T06:20:41.274500+0000","last_fullsized":"2023-10-09T06:20:41.274500+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"202'9247","reported_seq":11033,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.095312+0000","last_change":"2023-10-09T06:19:15.900457+0000","last_active":"2023-10-09T06:20:48.095312+0000","last_peered":"2023-10-09T06:20:48.095312+0000","last_clean":"2023-10-09T06:20:48.095312+0000","last_became_active":"2023-10-09T06:19:15.893387+0000","last_became_peered":"2023-10-09T06:19:15.893387+0000","last_unstale":"2023-10-09T06:20:48.095312+0000","last_undegraded":"2023-10-09T06:20:48.095312+0000","last_fullsized":"2023-10-09T06:20:48.095312+0000","mapping_epoch":170,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9247,"log_dups_size":0,"ondisk_log_size":9247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:09.689997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197984256,"num_objects":3021,"num_object_clones":0,"num_object_copies":9063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9247,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":850,"num_bytes_recovered":55705600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":161,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269585+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:20:41.269585+0000","last_peered":"2023-10-09T06:20:41.269585+0000","last_clean":"2023-10-09T06:20:41.269585+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:20:41.269585+0000","last_undegraded":"2023-10-09T06:20:41.269585+0000","last_fullsized":"2023-10-09T06:20:41.269585+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269513+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:20:41.269513+0000","last_peered":"2023-10-09T06:20:41.269513+0000","last_clean":"2023-10-09T06:20:41.269513+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:20:41.269513+0000","last_undegraded":"2023-10-09T06:20:41.269513+0000","last_fullsized":"2023-10-09T06:20:41.269513+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274233+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:20:41.274233+0000","last_peered":"2023-10-09T06:20:41.274233+0000","last_clean":"2023-10-09T06:20:41.274233+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:20:41.274233+0000","last_undegraded":"2023-10-09T06:20:41.274233+0000","last_fullsized":"2023-10-09T06:20:41.274233+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"202'9182","reported_seq":9242,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.594669+0000","last_change":"2023-10-09T06:17:57.016464+0000","last_active":"2023-10-09T06:20:48.594669+0000","last_peered":"2023-10-09T06:20:48.594669+0000","last_clean":"2023-10-09T06:20:48.594669+0000","last_became_active":"2023-10-09T06:17:57.016166+0000","last_became_peered":"2023-10-09T06:17:57.016166+0000","last_unstale":"2023-10-09T06:20:48.594669+0000","last_undegraded":"2023-10-09T06:20:48.594669+0000","last_fullsized":"2023-10-09T06:20:48.594669+0000","mapping_epoch":152,"log_start":"156'1800","ondisk_log_start":"156'1800","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:53.029953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204341248,"num_objects":3118,"num_object_clones":0,"num_object_copies":9354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9182,"num_write_kb":393600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":186,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274719+0000","last_change":"2023-10-09T06:19:06.672499+0000","last_active":"2023-10-09T06:20:31.274719+0000","last_peered":"2023-10-09T06:20:31.274719+0000","last_clean":"2023-10-09T06:20:31.274719+0000","last_became_active":"2023-10-09T06:19:06.672182+0000","last_became_peered":"2023-10-09T06:19:06.672182+0000","last_unstale":"2023-10-09T06:20:31.274719+0000","last_undegraded":"2023-10-09T06:20:31.274719+0000","last_fullsized":"2023-10-09T06:20:31.274719+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:07.246328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":455,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.258659+0000","last_change":"2023-10-09T06:20:11.075324+0000","last_active":"2023-10-09T06:20:41.258659+0000","last_peered":"2023-10-09T06:20:41.258659+0000","last_clean":"2023-10-09T06:20:41.258659+0000","last_became_active":"2023-10-09T06:20:11.074888+0000","last_became_peered":"2023-10-09T06:20:11.074888+0000","last_unstale":"2023-10-09T06:20:41.258659+0000","last_undegraded":"2023-10-09T06:20:41.258659+0000","last_fullsized":"2023-10-09T06:20:41.258659+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:03.264166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289455+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:20:31.289455+0000","last_peered":"2023-10-09T06:20:31.289455+0000","last_clean":"2023-10-09T06:20:31.289455+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:20:31.289455+0000","last_undegraded":"2023-10-09T06:20:31.289455+0000","last_fullsized":"2023-10-09T06:20:31.289455+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252594+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:20:41.252594+0000","last_peered":"2023-10-09T06:20:41.252594+0000","last_clean":"2023-10-09T06:20:41.252594+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:20:41.252594+0000","last_undegraded":"2023-10-09T06:20:41.252594+0000","last_fullsized":"2023-10-09T06:20:41.252594+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"202'9533","reported_seq":9593,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:49.914368+0000","last_change":"2023-10-09T06:17:57.236464+0000","last_active":"2023-10-09T06:20:49.914368+0000","last_peered":"2023-10-09T06:20:49.914368+0000","last_clean":"2023-10-09T06:20:49.914368+0000","last_became_active":"2023-10-09T06:17:57.236124+0000","last_became_peered":"2023-10-09T06:17:57.236124+0000","last_unstale":"2023-10-09T06:20:49.914368+0000","last_undegraded":"2023-10-09T06:20:49.914368+0000","last_fullsized":"2023-10-09T06:20:49.914368+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9533,"log_dups_size":0,"ondisk_log_size":9533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:33.469044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190382080,"num_objects":2905,"num_object_clones":0,"num_object_copies":8715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9533,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.258471+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:20:41.258471+0000","last_peered":"2023-10-09T06:20:41.258471+0000","last_clean":"2023-10-09T06:20:41.258471+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:20:41.258471+0000","last_undegraded":"2023-10-09T06:20:41.258471+0000","last_fullsized":"2023-10-09T06:20:41.258471+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.244617+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:20:41.244617+0000","last_peered":"2023-10-09T06:20:41.244617+0000","last_clean":"2023-10-09T06:20:41.244617+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:20:41.244617+0000","last_undegraded":"2023-10-09T06:20:41.244617+0000","last_fullsized":"2023-10-09T06:20:41.244617+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261487+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:20:31.261487+0000","last_peered":"2023-10-09T06:20:31.261487+0000","last_clean":"2023-10-09T06:20:31.261487+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:20:31.261487+0000","last_undegraded":"2023-10-09T06:20:31.261487+0000","last_fullsized":"2023-10-09T06:20:31.261487+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"203'9419","reported_seq":11212,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:52.865740+0000","last_change":"2023-10-09T06:19:52.709796+0000","last_active":"2023-10-09T06:20:52.865740+0000","last_peered":"2023-10-09T06:20:52.865740+0000","last_clean":"2023-10-09T06:20:52.865740+0000","last_became_active":"2023-10-09T06:19:52.651903+0000","last_became_peered":"2023-10-09T06:19:52.651903+0000","last_unstale":"2023-10-09T06:20:52.865740+0000","last_undegraded":"2023-10-09T06:20:52.865740+0000","last_fullsized":"2023-10-09T06:20:52.865740+0000","mapping_epoch":184,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:57.164182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189464576,"num_objects":2891,"num_object_clones":0,"num_object_copies":8673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9419,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262961+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:20:41.262961+0000","last_peered":"2023-10-09T06:20:41.262961+0000","last_clean":"2023-10-09T06:20:41.262961+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:20:41.262961+0000","last_undegraded":"2023-10-09T06:20:41.262961+0000","last_fullsized":"2023-10-09T06:20:41.262961+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274450+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:20:41.274450+0000","last_peered":"2023-10-09T06:20:41.274450+0000","last_clean":"2023-10-09T06:20:41.274450+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:20:41.274450+0000","last_undegraded":"2023-10-09T06:20:41.274450+0000","last_fullsized":"2023-10-09T06:20:41.274450+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":156,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245818+0000","last_change":"2023-10-09T06:19:06.667489+0000","last_active":"2023-10-09T06:20:41.245818+0000","last_peered":"2023-10-09T06:20:41.245818+0000","last_clean":"2023-10-09T06:20:41.245818+0000","last_became_active":"2023-10-09T06:19:06.667029+0000","last_became_peered":"2023-10-09T06:19:06.667029+0000","last_unstale":"2023-10-09T06:20:41.245818+0000","last_undegraded":"2023-10-09T06:20:41.245818+0000","last_fullsized":"2023-10-09T06:20:41.245818+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:35.989088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"203'9399","reported_seq":11195,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:52.885600+0000","last_change":"2023-10-09T06:19:26.121898+0000","last_active":"2023-10-09T06:20:52.885600+0000","last_peered":"2023-10-09T06:20:52.885600+0000","last_clean":"2023-10-09T06:20:52.885600+0000","last_became_active":"2023-10-09T06:19:26.079797+0000","last_became_peered":"2023-10-09T06:19:26.079797+0000","last_unstale":"2023-10-09T06:20:52.885600+0000","last_undegraded":"2023-10-09T06:20:52.885600+0000","last_fullsized":"2023-10-09T06:20:52.885600+0000","mapping_epoch":173,"log_start":"156'1700","ondisk_log_start":"156'1700","created":152,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:05.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179961880,"num_objects":2747,"num_object_clones":0,"num_object_copies":8241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2747,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9399,"num_write_kb":388609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":189,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.293659+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:20:31.293659+0000","last_peered":"2023-10-09T06:20:31.293659+0000","last_clean":"2023-10-09T06:20:31.293659+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:20:31.293659+0000","last_undegraded":"2023-10-09T06:20:31.293659+0000","last_fullsized":"2023-10-09T06:20:31.293659+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.276065+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:20:41.276065+0000","last_peered":"2023-10-09T06:20:41.276065+0000","last_clean":"2023-10-09T06:20:41.276065+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:20:41.276065+0000","last_undegraded":"2023-10-09T06:20:41.276065+0000","last_fullsized":"2023-10-09T06:20:41.276065+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245602+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:20:41.245602+0000","last_peered":"2023-10-09T06:20:41.245602+0000","last_clean":"2023-10-09T06:20:41.245602+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:20:41.245602+0000","last_undegraded":"2023-10-09T06:20:41.245602+0000","last_fullsized":"2023-10-09T06:20:41.245602+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"202'9017","reported_seq":10671,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:48.607743+0000","last_change":"2023-10-09T06:19:34.580478+0000","last_active":"2023-10-09T06:20:48.607743+0000","last_peered":"2023-10-09T06:20:48.607743+0000","last_clean":"2023-10-09T06:20:48.607743+0000","last_became_active":"2023-10-09T06:19:34.573414+0000","last_became_peered":"2023-10-09T06:19:34.573414+0000","last_unstale":"2023-10-09T06:20:48.607743+0000","last_undegraded":"2023-10-09T06:20:48.607743+0000","last_fullsized":"2023-10-09T06:20:48.607743+0000","mapping_epoch":176,"log_start":"156'1600","ondisk_log_start":"156'1600","created":152,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:08.324993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9017,"num_write_kb":383808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":51380224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":187,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252927+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:20:41.252927+0000","last_peered":"2023-10-09T06:20:41.252927+0000","last_clean":"2023-10-09T06:20:41.252927+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:20:41.252927+0000","last_undegraded":"2023-10-09T06:20:41.252927+0000","last_fullsized":"2023-10-09T06:20:41.252927+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":156,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.261351+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:20:31.261351+0000","last_peered":"2023-10-09T06:20:31.261351+0000","last_clean":"2023-10-09T06:20:31.261351+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:20:31.261351+0000","last_undegraded":"2023-10-09T06:20:31.261351+0000","last_fullsized":"2023-10-09T06:20:31.261351+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246136+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:20:41.246136+0000","last_peered":"2023-10-09T06:20:41.246136+0000","last_clean":"2023-10-09T06:20:41.246136+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:20:41.246136+0000","last_undegraded":"2023-10-09T06:20:41.246136+0000","last_fullsized":"2023-10-09T06:20:41.246136+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246142+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:20:41.246142+0000","last_peered":"2023-10-09T06:20:41.246142+0000","last_clean":"2023-10-09T06:20:41.246142+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:20:41.246142+0000","last_undegraded":"2023-10-09T06:20:41.246142+0000","last_fullsized":"2023-10-09T06:20:41.246142+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":169,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.268910+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:20:41.268910+0000","last_peered":"2023-10-09T06:20:41.268910+0000","last_clean":"2023-10-09T06:20:41.268910+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:20:41.268910+0000","last_undegraded":"2023-10-09T06:20:41.268910+0000","last_fullsized":"2023-10-09T06:20:41.268910+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.274279+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:20:31.274279+0000","last_peered":"2023-10-09T06:20:31.274279+0000","last_clean":"2023-10-09T06:20:31.274279+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:20:31.274279+0000","last_undegraded":"2023-10-09T06:20:31.274279+0000","last_fullsized":"2023-10-09T06:20:31.274279+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247082+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:20:41.247082+0000","last_peered":"2023-10-09T06:20:41.247082+0000","last_clean":"2023-10-09T06:20:41.247082+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:20:41.247082+0000","last_undegraded":"2023-10-09T06:20:41.247082+0000","last_fullsized":"2023-10-09T06:20:41.247082+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":147,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.276135+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:20:31.276135+0000","last_peered":"2023-10-09T06:20:31.276135+0000","last_clean":"2023-10-09T06:20:31.276135+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:20:31.276135+0000","last_undegraded":"2023-10-09T06:20:31.276135+0000","last_fullsized":"2023-10-09T06:20:31.276135+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245997+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:20:41.245997+0000","last_peered":"2023-10-09T06:20:41.245997+0000","last_clean":"2023-10-09T06:20:41.245997+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:20:41.245997+0000","last_undegraded":"2023-10-09T06:20:41.245997+0000","last_fullsized":"2023-10-09T06:20:41.245997+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":158,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.244776+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:20:41.244776+0000","last_peered":"2023-10-09T06:20:41.244776+0000","last_clean":"2023-10-09T06:20:41.244776+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:20:41.244776+0000","last_undegraded":"2023-10-09T06:20:41.244776+0000","last_fullsized":"2023-10-09T06:20:41.244776+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-10-09T06:20:31.289784+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:20:31.289784+0000","last_peered":"2023-10-09T06:20:31.289784+0000","last_clean":"2023-10-09T06:20:31.289784+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:20:31.289784+0000","last_undegraded":"2023-10-09T06:20:31.289784+0000","last_fullsized":"2023-10-09T06:20:31.289784+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":157,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.248400+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:20:41.248400+0000","last_peered":"2023-10-09T06:20:41.248400+0000","last_clean":"2023-10-09T06:20:41.248400+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:20:41.248400+0000","last_undegraded":"2023-10-09T06:20:41.248400+0000","last_fullsized":"2023-10-09T06:20:41.248400+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245884+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:20:41.245884+0000","last_peered":"2023-10-09T06:20:41.245884+0000","last_clean":"2023-10-09T06:20:41.245884+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:20:41.245884+0000","last_undegraded":"2023-10-09T06:20:41.245884+0000","last_fullsized":"2023-10-09T06:20:41.245884+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3076194328,"num_objects":46940,"num_object_clones":0,"num_object_copies":140820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46940,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148200,"num_write_kb":6244417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8419,"num_bytes_recovered":551747584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9223811072,"data_stored":9223798856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132400,"ondisk_log_size":132400,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234283,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2526200,"kb_used_data":1621872,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":91218952,"statfs":{"total":95995035648,"available":93408206848,"internally_reserved":0,"allocated":1660796928,"data_stored":1660117611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.071,"15min":1.278},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.3600000000000003,"5min":10.388,"15min":11.785},"last":1.4710000000000001},{"interface":"front","average":{"1min":1.494,"5min":1.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.4500000000000002,"5min":10.396000000000001,"15min":10.784000000000001},"last":2.806}]},{"osd":1,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.9569999999999999,"15min":2.093},"min":{"1min":0.28999999999999998,"5min":0.25,"15min":0.25},"max":{"1min":17.774000000000001,"5min":31.869,"15min":31.869},"last":3.7999999999999998},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.181,"15min":1.7569999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.747,"5min":29.486999999999998,"15min":29.486999999999998},"last":2.5990000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.51,"15min":1.8700000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.3559999999999999,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":5.8520000000000003},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.6020000000000001,"15min":1.9970000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.4189999999999996,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":1.4470000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":1.05,"15min":0.74099999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.1240000000000006,"5min":8.1240000000000006,"15min":8.1240000000000006},"last":2.5350000000000001},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.0349999999999999,"15min":0.76100000000000001},"min":{"1min":0.40699999999999997,"5min":0.312,"15min":0.312},"max":{"1min":4.657,"5min":4.657,"15min":4.657},"last":5.915}]},{"osd":4,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.575,"15min":1.339},"min":{"1min":0.32200000000000001,"5min":0.28299999999999997,"15min":0.251},"max":{"1min":25.678999999999998,"5min":25.678999999999998,"15min":25.678999999999998},"last":1.415},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.018,"15min":1.2450000000000001},"min":{"1min":0.32800000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":8.4359999999999999,"5min":8.4359999999999999,"15min":9.234},"last":2.4990000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":5.6619999999999999,"5min":2.3180000000000001,"15min":1.6259999999999999},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.27400000000000002},"max":{"1min":50.304000000000002,"5min":50.304000000000002,"15min":50.304000000000002},"last":3.8239999999999998},{"interface":"front","average":{"1min":4.8899999999999997,"5min":2.0219999999999998,"15min":1.413},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":48.774999999999999,"5min":48.774999999999999,"15min":48.774999999999999},"last":1.5229999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.3420000000000001,"15min":1.2969999999999999},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":5.2830000000000004,"5min":11.878,"15min":11.878},"last":1.496},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.081,"15min":1.2929999999999999},"min":{"1min":0.38800000000000001,"5min":0.374,"15min":0.28599999999999998},"max":{"1min":5.7229999999999999,"5min":7.0380000000000003,"15min":9.2050000000000001},"last":2.5600000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495920,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":849824,"kb_used_data":544344,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":92895328,"statfs":{"total":95995035648,"available":95124815872,"internally_reserved":0,"allocated":557408256,"data_stored":556752757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:44 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":3.7429999999999999,"15min":2.3620000000000001},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.214},"max":{"1min":7.5919999999999996,"5min":109.092,"15min":109.092},"last":0.251},{"interface":"front","average":{"1min":2.6219999999999999,"5min":4.2089999999999996,"15min":2.5470000000000002},"min":{"1min":0.30199999999999999,"5min":0.28299999999999997,"15min":0.24399999999999999},"max":{"1min":9.7609999999999992,"5min":109.369,"15min":109.369},"last":0.318}]},{"osd":1,"last update":"Mon Oct 9 06:20:11 2023","interfaces":[{"interface":"back","average":{"1min":16.698,"5min":5.1689999999999996,"15min":3.2919999999999998},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":114.202,"5min":114.202,"15min":114.202},"last":2.242},{"interface":"front","average":{"1min":15.648999999999999,"5min":5.0220000000000002,"15min":3.2719999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.246},"max":{"1min":114.179,"5min":114.179,"15min":114.179},"last":2.2530000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:20:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.7410000000000001,"15min":2.2410000000000001},"min":{"1min":0.317,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.441},{"interface":"front","average":{"1min":1.016,"5min":1.3939999999999999,"15min":2.2610000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.734,"5min":8.9870000000000001,"15min":8.9870000000000001},"last":0.61199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":2.2690000000000001,"15min":1.117},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.3059999999999992,"5min":64.012,"15min":64.012},"last":0.33900000000000002},{"interface":"front","average":{"1min":3.3919999999999999,"5min":2.3769999999999998,"15min":1.167},"min":{"1min":0.35499999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":15.507999999999999,"5min":64.105999999999995,"15min":64.105999999999995},"last":3.9409999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":1.474,"15min":1.1659999999999999},"min":{"1min":0.33400000000000002,"5min":0.252,"15min":0.245},"max":{"1min":17.643999999999998,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.159,"5min":1.075,"15min":1.2030000000000001},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.26400000000000001},"max":{"1min":3.6549999999999998,"5min":8.9239999999999995,"15min":9.4100000000000001},"last":0.92100000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.514,"15min":1.29},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.2410000000000001,"15min":1.827},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":8.0630000000000006,"5min":9.0169999999999995,"15min":159.06800000000001},"last":0.71799999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0169999999999999,"15min":1.071},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":4.2640000000000002,"5min":10.962,"15min":14.837},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.008,"5min":1.177,"15min":1.393},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.9939999999999998,"5min":8.9920000000000009,"15min":10.891},"last":0.77500000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402656,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2032684,"kb_used_data":1667556,"kb_used_omap":4,"kb_used_meta":365115,"kb_avail":91712468,"statfs":{"total":95995035648,"available":93913567232,"internally_reserved":0,"allocated":1707577344,"data_stored":1706667753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":373878670},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":7.2290000000000001,"5min":2.5230000000000001,"15min":2.661},"min":{"1min":0.20799999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":148.98400000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.4500000000000002,"15min":2.948},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20100000000000001},"max":{"1min":149.02699999999999,"5min":149.02699999999999,"15min":149.02699999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.9219999999999999,"15min":1.796},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":7.3399999999999999,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.309},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.9350000000000001,"15min":2.5619999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":4.9690000000000003,"5min":49.783000000000001,"15min":49.783000000000001},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:54 2023","interfaces":[{"interface":"back","average":{"1min":10.465,"5min":2.6920000000000002,"15min":1.1519999999999999},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":153.18199999999999,"5min":153.18199999999999,"15min":153.18199999999999},"last":2.0710000000000002},{"interface":"front","average":{"1min":10.079000000000001,"5min":2.5569999999999999,"15min":1.1240000000000001},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":152.52000000000001,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.48499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":10.198,"5min":3.0299999999999998,"15min":3.0430000000000001},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":199.43799999999999,"5min":199.43799999999999,"15min":199.43799999999999},"last":1.9430000000000001},{"interface":"front","average":{"1min":8.7400000000000002,"5min":2.956,"15min":2.9940000000000002},"min":{"1min":0.39000000000000001,"5min":0.315,"15min":0.311},"max":{"1min":199.32900000000001,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.58799999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.6619999999999999,"15min":2.9060000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":61.930999999999997,"5min":61.930999999999997,"15min":61.930999999999997},"last":2.5449999999999999},{"interface":"front","average":{"1min":3.2069999999999999,"5min":1.958,"15min":2.879},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":61.947000000000003,"5min":61.947000000000003,"15min":61.947000000000003},"last":2.5579999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.8380000000000001,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":4.274,"5min":12.91,"15min":12.91},"last":4.4290000000000003},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.7629999999999999,"15min":2.3769999999999998},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":4.4420000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.772,"15min":2.5139999999999998},"min":{"1min":0.378,"5min":0.376,"15min":0.376},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":3.169},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.3819999999999999,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":6.7270000000000003,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":3.1579999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024089,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1223572,"kb_used_data":770124,"kb_used_omap":5,"kb_used_meta":453434,"kb_avail":92521580,"statfs":{"total":95995035648,"available":94742097920,"internally_reserved":0,"allocated":788606976,"data_stored":787547266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":464317050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:20:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5099999999999998,"5min":2.133,"15min":2.0139999999999998},"min":{"1min":0.26600000000000001,"5min":0.19,"15min":0.19},"max":{"1min":29.021999999999998,"5min":30.733000000000001,"15min":30.733000000000001},"last":0.20300000000000001},{"interface":"front","average":{"1min":3.3849999999999998,"5min":2.6389999999999998,"15min":2.7040000000000002},"min":{"1min":0.33700000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":29.077999999999999,"5min":30.279,"15min":30.279},"last":5.1529999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.762,"15min":2.3730000000000002},"min":{"1min":0.248,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":9.9909999999999997,"5min":17.588999999999999,"15min":17.588999999999999},"last":5.3280000000000003},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.3889999999999998},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":11.162000000000001,"5min":21.198,"15min":21.198},"last":5.1870000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:19:50 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":1.4750000000000001,"15min":0.747},"min":{"1min":0.33200000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":30.123000000000001,"5min":30.123000000000001,"15min":30.123000000000001},"last":4.5060000000000002},{"interface":"front","average":{"1min":4.851,"5min":1.522,"15min":0.77400000000000002},"min":{"1min":0.34300000000000003,"5min":0.219,"15min":0.219},"max":{"1min":29.221,"5min":29.221,"15min":29.221},"last":4.8470000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.1819999999999999,"15min":1.647},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":4.3739999999999997,"5min":11.162000000000001,"15min":15.835000000000001},"last":0.25900000000000001},{"interface":"front","average":{"1min":2.137,"5min":1.8129999999999999,"15min":2.133},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":7.6210000000000004,"5min":12.752000000000001,"15min":15.023},"last":5.0910000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.6060000000000001,"15min":1.895},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.567,"5min":11.222,"15min":15.858000000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":2.589,"5min":2.0350000000000001,"15min":2.226},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":9.6400000000000006,"5min":21.167000000000002,"15min":31.984999999999999},"last":4.5430000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":2.1509999999999998,"15min":2.2069999999999999},"min":{"1min":0.45700000000000002,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":9.6880000000000006,"5min":21.088999999999999,"15min":32.079000000000001},"last":4.6020000000000003},{"interface":"front","average":{"1min":2.3210000000000002,"5min":1.542,"15min":1.978},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":10.606,"5min":11.236000000000001,"15min":15.893000000000001},"last":0.36599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.796,"15min":2.2080000000000002},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":9.9269999999999996,"5min":12.798999999999999,"15min":15.050000000000001},"last":5.1269999999999998},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.974,"15min":2.2549999999999999},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":8.2219999999999995,"5min":21.135999999999999,"15min":32.055999999999997},"last":4.5780000000000003}]}]},{"osd":2,"up_from":128,"seq":549755813958,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796752,"kb_used_data":557320,"kb_used_omap":7,"kb_used_meta":239416,"kb_avail":92948400,"statfs":{"total":95995035648,"available":95179161600,"internally_reserved":0,"allocated":570695680,"data_stored":569810757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":245162065},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":5.3449999999999998,"15min":3.0710000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":88.225999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":5.8730000000000002,"5min":5.4880000000000004,"15min":3.6520000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":85.781000000000006,"5min":163.363,"15min":163.363},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.29,"15min":2.6669999999999998},"min":{"1min":0.24299999999999999,"5min":0.188,"15min":0.188},"max":{"1min":4.5679999999999996,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.86099999999999999,"5min":4.3620000000000001,"15min":3.2959999999999998},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8069999999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.57399999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":6.6619999999999999,"5min":2.0470000000000002,"15min":0.91700000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":81.183999999999997,"5min":81.183999999999997,"15min":81.183999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":6.4820000000000002,"5min":1.8520000000000001,"15min":0.89800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":81.061999999999998,"5min":81.061999999999998,"15min":81.061999999999998},"last":1.8280000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.532,"15min":1.97},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.106999999999999,"5min":12.141999999999999,"15min":12.141999999999999},"last":3.964},{"interface":"front","average":{"1min":2.7599999999999998,"5min":1.6870000000000001,"15min":2.2309999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.994999999999999,"5min":15.183,"15min":15.183},"last":3.9060000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":1.3089999999999999,"15min":1.6499999999999999},"min":{"1min":0.36399999999999999,"5min":0.309,"15min":0.309},"max":{"1min":4.9989999999999997,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9079999999999999,"15min":2.3919999999999999},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.6840000000000002,"5min":10.215999999999999,"15min":10.215999999999999},"last":3.5779999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.5109999999999999,"15min":2.1259999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.4939999999999998,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":2.7320000000000002},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.4119999999999999,"15min":1.6140000000000001},"min":{"1min":0.39300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.883,"5min":7.327,"15min":7.327},"last":2.1299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.583,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.6660000000000004,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":0.436},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.7310000000000001,"15min":2.7549999999999999},"min":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.7779999999999996,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":2.4790000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127155,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1780072,"kb_used_data":1562272,"kb_used_omap":2,"kb_used_meta":217789,"kb_avail":91965080,"statfs":{"total":95995035648,"available":94172241920,"internally_reserved":0,"allocated":1599766528,"data_stored":1598976544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":223015989},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":1.3,"15min":0.84599999999999997},"min":{"1min":0.23300000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.9160000000000004,"5min":9.2729999999999997,"15min":9.2729999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.5680000000000001,"5min":1.7390000000000001,"15min":1.2110000000000001},"min":{"1min":0.223,"5min":0.191,"15min":0.191},"max":{"1min":11.946,"5min":11.946,"15min":11.946},"last":1.234}]},{"osd":1,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.8580000000000001,"15min":1.2509999999999999},"min":{"1min":0.26400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":16.260000000000002,"5min":16.260000000000002,"15min":16.260000000000002},"last":1.121},{"interface":"front","average":{"1min":2.1179999999999999,"5min":1.5329999999999999,"15min":0.91700000000000004},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.254},"max":{"1min":8.6140000000000008,"5min":12.757999999999999,"15min":12.757999999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":1.768,"15min":1.218},"min":{"1min":0.28599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.871,"5min":11.871,"15min":11.871},"last":1.645},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.1080000000000001,"15min":0.65000000000000002},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.6430000000000007,"5min":9.2230000000000008,"15min":9.2230000000000008},"last":0.37}]},{"osd":4,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.139,"15min":0.79100000000000004},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.4740000000000002,"5min":1.462,"15min":1.282},"min":{"1min":0.39900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.6899999999999995,"5min":11.742000000000001,"15min":11.742000000000001},"last":0.82499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4740000000000002,"5min":1.847,"15min":1.431},"min":{"1min":0.39000000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.6930000000000001,"5min":1.8169999999999999,"15min":1.4239999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":15.109999999999999,"5min":15.109999999999999,"15min":15.109999999999999},"last":1.2090000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.665,"15min":1.351},"min":{"1min":0.38900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.7710000000000008,"5min":11.798,"15min":11.798},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.875,"5min":1.587,"15min":1.327},"min":{"1min":0.36299999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.3559999999999999,"5min":11.552,"15min":11.552},"last":1.1679999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.3400000000000001,"15min":0.874},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.843,"5min":9.843,"15min":9.843},"last":0.85699999999999998},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.6060000000000001,"15min":1.3460000000000001},"min":{"1min":0.44600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.7590000000000003,"5min":11.823,"15min":11.823},"last":0.56599999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019192,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1147240,"kb_used_data":561568,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":92597912,"statfs":{"total":95995035648,"available":94820261888,"internally_reserved":0,"allocated":575045632,"data_stored":574381796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.4370000000000001,"15min":1.4670000000000001},"min":{"1min":0.26000000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":19.577000000000002,"5min":19.577000000000002,"15min":19.577000000000002},"last":0.255},{"interface":"front","average":{"1min":3.0830000000000002,"5min":1.7430000000000001,"15min":1.837},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":19.594999999999999,"5min":19.594999999999999,"15min":19.594999999999999},"last":1.0640000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.5369999999999999,"15min":1.9470000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":34.412999999999997,"5min":91.849999999999994,"15min":91.849999999999994},"last":4.548},{"interface":"front","average":{"1min":3.71,"5min":2.7210000000000001,"15min":2},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":34.604999999999997,"5min":91.885000000000005,"15min":91.885000000000005},"last":4.4589999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0960000000000001,"5min":1.524,"15min":1.778},"min":{"1min":0.316,"5min":0.255,"15min":0.255},"max":{"1min":6.8650000000000002,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":3.0550000000000002},{"interface":"front","average":{"1min":2.831,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.898,"5min":15.894,"15min":15.894},"last":4.4249999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:20:51 2023","interfaces":[{"interface":"back","average":{"1min":7.4139999999999997,"5min":3.597,"15min":1.5740000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":57.826999999999998,"5min":91.697000000000003,"15min":91.697000000000003},"last":1.0940000000000001},{"interface":"front","average":{"1min":8.1120000000000001,"5min":3.4510000000000001,"15min":1.5649999999999999},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.853000000000002,"5min":94.356999999999999,"15min":94.356999999999999},"last":3.2650000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.1639999999999999,"15min":1.0629999999999999},"min":{"1min":0.26300000000000001,"5min":0.254,"15min":0.24299999999999999},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":11.130000000000001},"last":2.9540000000000002},{"interface":"front","average":{"1min":2.149,"5min":1.2929999999999999,"15min":1.353},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":12.552,"5min":12.552,"15min":12.552},"last":1.0229999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.70399999999999996,"15min":0.89200000000000002},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.58999999999999997,"5min":4.907,"15min":9.7680000000000007},"last":1.617},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.4099999999999999,"15min":1.212},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":7.5380000000000003,"5min":14.202999999999999,"15min":14.202999999999999},"last":3.0019999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:19:56 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":1.548,"15min":1.2270000000000001},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":49.648000000000003,"5min":49.648000000000003,"15min":49.648000000000003},"last":3.0350000000000001},{"interface":"front","average":{"1min":2.956,"5min":1.7509999999999999,"15min":1.2829999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":49.750999999999998,"5min":49.750999999999998,"15min":49.750999999999998},"last":0.44}]}]},{"osd":5,"up_from":38,"seq":163208757556,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2691604,"kb_used_data":1806284,"kb_used_omap":0,"kb_used_meta":885312,"kb_avail":91053548,"statfs":{"total":95995035648,"available":93238833152,"internally_reserved":0,"allocated":1849634816,"data_stored":1848979100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906559488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.6599999999999999,"15min":1.675},"min":{"1min":0.216,"5min":0.216,"15min":0.215},"max":{"1min":6.8159999999999998,"5min":8.5139999999999993,"15min":18.637},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.909,"5min":1.4690000000000001,"15min":1.6799999999999999},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.1790000000000003,"5min":8.4019999999999992,"15min":9.9939999999999998},"last":3.1899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.423,"15min":2.125},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.8520000000000003,"5min":8.8170000000000002,"15min":11.292},"last":5.2999999999999998},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.645,"15min":2.0859999999999999},"min":{"1min":0.437,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.467000000000001,"5min":10.467000000000001,"15min":10.467000000000001},"last":1.0009999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6339999999999999,"15min":2.2029999999999998},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.307},"max":{"1min":10.42,"5min":10.42,"15min":10.42},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.946,"5min":1.4259999999999999,"15min":2.044},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.5990000000000002,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":1.0109999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2770000000000001,"5min":4.444,"15min":1.8979999999999999},"min":{"1min":0.376,"5min":0.314,"15min":0.314},"max":{"1min":6.4450000000000003,"5min":190.411,"15min":190.411},"last":5.3310000000000004},{"interface":"front","average":{"1min":2.0169999999999999,"5min":4.4160000000000004,"15min":1.8919999999999999},"min":{"1min":0.34699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.9980000000000002,"5min":190.70699999999999,"15min":190.70699999999999},"last":5.2290000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.4490000000000001,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":8.6389999999999993,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.704,"5min":1.117,"15min":1.0660000000000001},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.25900000000000001},"max":{"1min":5.0570000000000004,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5590000000000002,"5min":1.593,"15min":1.304},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.28899999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":5.2709999999999999},{"interface":"front","average":{"1min":3.2589999999999999,"5min":1.5109999999999999,"15min":1.2190000000000001},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.621}]},{"osd":7,"last update":"Mon Oct 9 06:19:54 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.4570000000000001,"15min":1.1399999999999999},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":8.6029999999999998,"5min":8.8409999999999993,"15min":8.8409999999999993},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.2150000000000001,"15min":1.103},"min":{"1min":0.40100000000000002,"5min":0.36099999999999999,"15min":0.29299999999999998},"max":{"1min":5.4290000000000003,"5min":7.7809999999999997,"15min":9.4580000000000002},"last":2.496}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":778829824,"data_stored":778829824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1680281600,"data_stored":1680277528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":565051392,"data_stored":565051392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1590296576,"data_stored":1590296576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":565444608,"data_stored":565444608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1841696768,"data_stored":1841692696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":548077568,"data_stored":548077568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1654132736,"data_stored":1654128664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:20:55.619 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:20:55.619 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:20:55.619 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:20:55.620 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:20:55.632 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:55.664 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:20:55.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:55.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:20:55.761 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:55.772 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:55.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:55 smithi107 ceph-mon[212750]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:55.841 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:55 smithi107 ceph-mon[220921]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:55.856 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:55.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:55 smithi150 ceph-mon[194178]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:20:56.015 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:56.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:56.127 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:20:56.234 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:56.327 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:20:56.349 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:20:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:56.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:20:56.370 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:20:56.413 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:20:56.458 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:20:56.468 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:20:56.493 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:20:56.527 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:20:56.530 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:56.565 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:20:56.610 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:20:56.616 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.616 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.616 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:20:56.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:56 smithi107 ceph-mon[212750]: pgmap v658: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:20:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3958472782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:56 smithi107 ceph-mon[220921]: pgmap v658: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:20:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3958472782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:57.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:56 smithi150 ceph-mon[194178]: pgmap v658: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T06:20:57.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3958472782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:20:58.273 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:20:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:20:58.390 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:58.438 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.524 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:58.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:58.633 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:20:58.711 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:20:58.724 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:58.871 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:58 smithi107 ceph-mon[212750]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:20:58.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:58 smithi107 ceph-mon[220921]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:20:59.095 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:20:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:59.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:20:59.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:58 smithi150 ceph-mon[194178]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T06:20:59.210 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:20:59.313 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:20:59.313 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:20:59.314 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:20:59.314 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:20:59.314 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:20:59.361 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:59.448 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:20:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:59.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:20:59.570 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:20:59.666 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:20:59.782 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:20:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:20:59.783 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:20:59.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:20:59.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:20:59.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/961020641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:20:59.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:20:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:00.003 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/961020641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:20:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/961020641' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:00.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:20:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:00.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:00.426 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:00.527 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:00.598 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:00.662 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:21:00.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: pgmap v660: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:21:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[220921]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: pgmap v660: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:00.933 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T06:21:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:00 smithi107 ceph-mon[212750]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: pgmap v660: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: osdmap e204: 8 total, 8 up, 8 in 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:00 smithi150 ceph-mon[194178]: failed to encode map e204 with expected crc 2023-10-09T06:21:01.192 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:01.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:01.300 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:01.380 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:01.535 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:01.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:01.648 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:01.693 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:01.777 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:01.785 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:01.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:01.834 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:01.893 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:01.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.023 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:02.139 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:02.183 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.244 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:02.260 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:21:02.275 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:02] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T06:21:02.281 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:21:02.357 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:02.415 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.506 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:02.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:02.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[212750]: pgmap v662: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:21:02.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:02.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:02.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[220921]: pgmap v662: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:21:02.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:02.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:02.649 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:02.729 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:02 smithi150 ceph-mon[194178]: pgmap v662: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T06:21:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:03.144 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:21:03.708 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:03.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:03.817 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:03.881 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":662,"stamp":"2023-10-09T06:21:01.003582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2692421560,"num_objects":41099,"num_object_clones":0,"num_object_copies":123297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41099,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":154365,"num_write_kb":6246862,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8446,"num_bytes_recovered":554709108,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134981,"ondisk_log_size":134981,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11991056,"kb_used_data":8005456,"kb_used_omap":20,"kb_used_meta":3985195,"kb_avail":737970160,"statfs":{"total":767960285184,"available":755681443840,"internally_reserved":0,"allocated":8197586944,"data_stored":8191268811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4080840334},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,6,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-454164480,"num_objects":-6930,"num_object_clones":0,"num_object_copies":-20790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6930,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010504"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246093+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:20:41.246093+0000","last_peered":"2023-10-09T06:20:41.246093+0000","last_clean":"2023-10-09T06:20:41.246093+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:20:41.246093+0000","last_undegraded":"2023-10-09T06:20:41.246093+0000","last_fullsized":"2023-10-09T06:20:41.246093+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246363+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:20:41.246363+0000","last_peered":"2023-10-09T06:20:41.246363+0000","last_clean":"2023-10-09T06:20:41.246363+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:20:41.246363+0000","last_undegraded":"2023-10-09T06:20:41.246363+0000","last_fullsized":"2023-10-09T06:20:41.246363+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269009+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:20:41.269009+0000","last_peered":"2023-10-09T06:20:41.269009+0000","last_clean":"2023-10-09T06:20:41.269009+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:20:41.269009+0000","last_undegraded":"2023-10-09T06:20:41.269009+0000","last_fullsized":"2023-10-09T06:20:41.269009+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601620+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:20:50.601620+0000","last_peered":"2023-10-09T06:20:50.601620+0000","last_clean":"2023-10-09T06:20:50.601620+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:20:50.601620+0000","last_undegraded":"2023-10-09T06:20:50.601620+0000","last_fullsized":"2023-10-09T06:20:50.601620+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":156,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245390+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:20:41.245390+0000","last_peered":"2023-10-09T06:20:41.245390+0000","last_clean":"2023-10-09T06:20:41.245390+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:20:41.245390+0000","last_undegraded":"2023-10-09T06:20:41.245390+0000","last_fullsized":"2023-10-09T06:20:41.245390+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252991+0000","last_change":"2023-10-09T06:13:02.840441+0000","last_active":"2023-10-09T06:20:41.252991+0000","last_peered":"2023-10-09T06:20:41.252991+0000","last_clean":"2023-10-09T06:20:41.252991+0000","last_became_active":"2023-10-09T06:13:02.840078+0000","last_became_peered":"2023-10-09T06:13:02.840078+0000","last_unstale":"2023-10-09T06:20:41.252991+0000","last_undegraded":"2023-10-09T06:20:41.252991+0000","last_fullsized":"2023-10-09T06:20:41.252991+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:58.571898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":154,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601964+0000","last_change":"2023-10-09T06:16:44.958636+0000","last_active":"2023-10-09T06:20:50.601964+0000","last_peered":"2023-10-09T06:20:50.601964+0000","last_clean":"2023-10-09T06:20:50.601964+0000","last_became_active":"2023-10-09T06:16:44.958270+0000","last_became_peered":"2023-10-09T06:16:44.958270+0000","last_unstale":"2023-10-09T06:20:50.601964+0000","last_undegraded":"2023-10-09T06:20:50.601964+0000","last_fullsized":"2023-10-09T06:20:50.601964+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:36.789320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245459+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:20:41.245459+0000","last_peered":"2023-10-09T06:20:41.245459+0000","last_clean":"2023-10-09T06:20:41.245459+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:20:41.245459+0000","last_undegraded":"2023-10-09T06:20:41.245459+0000","last_fullsized":"2023-10-09T06:20:41.245459+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":183,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.259955+0000","last_change":"2023-10-09T06:19:06.673434+0000","last_active":"2023-10-09T06:20:41.259955+0000","last_peered":"2023-10-09T06:20:41.259955+0000","last_clean":"2023-10-09T06:20:41.259955+0000","last_became_active":"2023-10-09T06:19:06.673042+0000","last_became_peered":"2023-10-09T06:19:06.673042+0000","last_unstale":"2023-10-09T06:20:41.259955+0000","last_undegraded":"2023-10-09T06:20:41.259955+0000","last_fullsized":"2023-10-09T06:20:41.259955+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:30.387567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":136,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245836+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:20:41.245836+0000","last_peered":"2023-10-09T06:20:41.245836+0000","last_clean":"2023-10-09T06:20:41.245836+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:20:41.245836+0000","last_undegraded":"2023-10-09T06:20:41.245836+0000","last_fullsized":"2023-10-09T06:20:41.245836+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263033+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:20:41.263033+0000","last_peered":"2023-10-09T06:20:41.263033+0000","last_clean":"2023-10-09T06:20:41.263033+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:20:41.263033+0000","last_undegraded":"2023-10-09T06:20:41.263033+0000","last_fullsized":"2023-10-09T06:20:41.263033+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":132,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.270425+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:20:41.270425+0000","last_peered":"2023-10-09T06:20:41.270425+0000","last_clean":"2023-10-09T06:20:41.270425+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:20:41.270425+0000","last_undegraded":"2023-10-09T06:20:41.270425+0000","last_fullsized":"2023-10-09T06:20:41.270425+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":209,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246393+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:20:41.246393+0000","last_peered":"2023-10-09T06:20:41.246393+0000","last_clean":"2023-10-09T06:20:41.246393+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:20:41.246393+0000","last_undegraded":"2023-10-09T06:20:41.246393+0000","last_fullsized":"2023-10-09T06:20:41.246393+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247289+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:20:41.247289+0000","last_peered":"2023-10-09T06:20:41.247289+0000","last_clean":"2023-10-09T06:20:41.247289+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:20:41.247289+0000","last_undegraded":"2023-10-09T06:20:41.247289+0000","last_fullsized":"2023-10-09T06:20:41.247289+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":52,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262452+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:20:41.262452+0000","last_peered":"2023-10-09T06:20:41.262452+0000","last_clean":"2023-10-09T06:20:41.262452+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:20:41.262452+0000","last_undegraded":"2023-10-09T06:20:41.262452+0000","last_fullsized":"2023-10-09T06:20:41.262452+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246547+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:20:41.246547+0000","last_peered":"2023-10-09T06:20:41.246547+0000","last_clean":"2023-10-09T06:20:41.246547+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:20:41.246547+0000","last_undegraded":"2023-10-09T06:20:41.246547+0000","last_fullsized":"2023-10-09T06:20:41.246547+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601693+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:20:50.601693+0000","last_peered":"2023-10-09T06:20:50.601693+0000","last_clean":"2023-10-09T06:20:50.601693+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:20:50.601693+0000","last_undegraded":"2023-10-09T06:20:50.601693+0000","last_fullsized":"2023-10-09T06:20:50.601693+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263403+0000","last_change":"2023-10-09T06:19:06.758564+0000","last_active":"2023-10-09T06:20:41.263403+0000","last_peered":"2023-10-09T06:20:41.263403+0000","last_clean":"2023-10-09T06:20:41.263403+0000","last_became_active":"2023-10-09T06:19:06.758270+0000","last_became_peered":"2023-10-09T06:19:06.758270+0000","last_unstale":"2023-10-09T06:20:41.263403+0000","last_undegraded":"2023-10-09T06:20:41.263403+0000","last_fullsized":"2023-10-09T06:20:41.263403+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:16.823391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262191+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:20:41.262191+0000","last_peered":"2023-10-09T06:20:41.262191+0000","last_clean":"2023-10-09T06:20:41.262191+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:20:41.262191+0000","last_undegraded":"2023-10-09T06:20:41.262191+0000","last_fullsized":"2023-10-09T06:20:41.262191+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601762+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:20:50.601762+0000","last_peered":"2023-10-09T06:20:50.601762+0000","last_clean":"2023-10-09T06:20:50.601762+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:20:50.601762+0000","last_undegraded":"2023-10-09T06:20:50.601762+0000","last_fullsized":"2023-10-09T06:20:50.601762+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":171,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.255295+0000","last_change":"2023-10-09T06:08:16.097273+0000","last_active":"2023-10-09T06:20:41.255295+0000","last_peered":"2023-10-09T06:20:41.255295+0000","last_clean":"2023-10-09T06:20:41.255295+0000","last_became_active":"2023-10-09T06:08:16.096732+0000","last_became_peered":"2023-10-09T06:08:16.096732+0000","last_unstale":"2023-10-09T06:20:41.255295+0000","last_undegraded":"2023-10-09T06:20:41.255295+0000","last_fullsized":"2023-10-09T06:20:41.255295+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:15.759519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246042+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:20:41.246042+0000","last_peered":"2023-10-09T06:20:41.246042+0000","last_clean":"2023-10-09T06:20:41.246042+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:20:41.246042+0000","last_undegraded":"2023-10-09T06:20:41.246042+0000","last_fullsized":"2023-10-09T06:20:41.246042+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"204'9533","reported_seq":11370,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:20:59.947499+0000","last_change":"2023-10-09T06:19:43.978924+0000","last_active":"2023-10-09T06:20:59.947499+0000","last_peered":"2023-10-09T06:20:59.947499+0000","last_clean":"2023-10-09T06:20:59.947499+0000","last_became_active":"2023-10-09T06:19:43.970383+0000","last_became_peered":"2023-10-09T06:19:43.970383+0000","last_unstale":"2023-10-09T06:20:59.947499+0000","last_undegraded":"2023-10-09T06:20:59.947499+0000","last_fullsized":"2023-10-09T06:20:59.947499+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9533,"log_dups_size":0,"ondisk_log_size":9533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:59.701538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9533,"num_write_kb":386368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":875,"num_bytes_recovered":57344000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263321+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:20:41.263321+0000","last_peered":"2023-10-09T06:20:41.263321+0000","last_clean":"2023-10-09T06:20:41.263321+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:20:41.263321+0000","last_undegraded":"2023-10-09T06:20:41.263321+0000","last_fullsized":"2023-10-09T06:20:41.263321+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.259093+0000","last_change":"2023-10-09T06:19:06.673411+0000","last_active":"2023-10-09T06:20:41.259093+0000","last_peered":"2023-10-09T06:20:41.259093+0000","last_clean":"2023-10-09T06:20:41.259093+0000","last_became_active":"2023-10-09T06:19:06.673113+0000","last_became_peered":"2023-10-09T06:19:06.673113+0000","last_unstale":"2023-10-09T06:20:41.259093+0000","last_undegraded":"2023-10-09T06:20:41.259093+0000","last_fullsized":"2023-10-09T06:20:41.259093+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:39.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"203'9507","reported_seq":9568,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:56.475457+0000","last_change":"2023-10-09T06:17:57.020187+0000","last_active":"2023-10-09T06:20:56.475457+0000","last_peered":"2023-10-09T06:20:56.475457+0000","last_clean":"2023-10-09T06:20:56.475457+0000","last_became_active":"2023-10-09T06:17:57.019892+0000","last_became_peered":"2023-10-09T06:17:57.019892+0000","last_unstale":"2023-10-09T06:20:56.475457+0000","last_undegraded":"2023-10-09T06:20:56.475457+0000","last_fullsized":"2023-10-09T06:20:56.475457+0000","mapping_epoch":152,"log_start":"156'2100","ondisk_log_start":"156'2100","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:01.403006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174391296,"num_objects":2661,"num_object_clones":0,"num_object_copies":7983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9507,"num_write_kb":389376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262711+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:20:41.262711+0000","last_peered":"2023-10-09T06:20:41.262711+0000","last_clean":"2023-10-09T06:20:41.262711+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:20:41.262711+0000","last_undegraded":"2023-10-09T06:20:41.262711+0000","last_fullsized":"2023-10-09T06:20:41.262711+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246196+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:20:41.246196+0000","last_peered":"2023-10-09T06:20:41.246196+0000","last_clean":"2023-10-09T06:20:41.246196+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:20:41.246196+0000","last_undegraded":"2023-10-09T06:20:41.246196+0000","last_fullsized":"2023-10-09T06:20:41.246196+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"203'9761","reported_seq":11497,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.079633+0000","last_change":"2023-10-09T06:20:00.751403+0000","last_active":"2023-10-09T06:20:58.079633+0000","last_peered":"2023-10-09T06:20:58.079633+0000","last_clean":"2023-10-09T06:20:58.079633+0000","last_became_active":"2023-10-09T06:20:00.740256+0000","last_became_peered":"2023-10-09T06:20:00.740256+0000","last_unstale":"2023-10-09T06:20:58.079633+0000","last_undegraded":"2023-10-09T06:20:58.079633+0000","last_fullsized":"2023-10-09T06:20:58.079633+0000","mapping_epoch":188,"log_start":"155'300","ondisk_log_start":"155'300","created":152,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9461,"log_dups_size":0,"ondisk_log_size":9461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:18.856099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169017344,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9761,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":824,"num_bytes_recovered":54001664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":50,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262850+0000","last_change":"2023-10-09T06:19:42.328723+0000","last_active":"2023-10-09T06:20:41.262850+0000","last_peered":"2023-10-09T06:20:41.262850+0000","last_clean":"2023-10-09T06:20:41.262850+0000","last_became_active":"2023-10-09T06:19:06.757480+0000","last_became_peered":"2023-10-09T06:19:06.757480+0000","last_unstale":"2023-10-09T06:20:41.262850+0000","last_undegraded":"2023-10-09T06:20:41.262850+0000","last_fullsized":"2023-10-09T06:20:41.262850+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:57.712635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252252+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:20:41.252252+0000","last_peered":"2023-10-09T06:20:41.252252+0000","last_clean":"2023-10-09T06:20:41.252252+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:20:41.252252+0000","last_undegraded":"2023-10-09T06:20:41.252252+0000","last_fullsized":"2023-10-09T06:20:41.252252+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"203'9747","reported_seq":9806,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.144968+0000","last_change":"2023-10-09T06:17:57.217507+0000","last_active":"2023-10-09T06:20:58.144968+0000","last_peered":"2023-10-09T06:20:58.144968+0000","last_clean":"2023-10-09T06:20:58.144968+0000","last_became_active":"2023-10-09T06:17:57.217183+0000","last_became_peered":"2023-10-09T06:17:57.217183+0000","last_unstale":"2023-10-09T06:20:58.144968+0000","last_undegraded":"2023-10-09T06:20:58.144968+0000","last_fullsized":"2023-10-09T06:20:58.144968+0000","mapping_epoch":152,"log_start":"155'300","ondisk_log_start":"155'300","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9447,"log_dups_size":0,"ondisk_log_size":9447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:47:56.905622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172032000,"num_objects":2625,"num_object_clones":0,"num_object_copies":7875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9747,"num_write_kb":395904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.259515+0000","last_change":"2023-10-09T06:16:44.963142+0000","last_active":"2023-10-09T06:20:41.259515+0000","last_peered":"2023-10-09T06:20:41.259515+0000","last_clean":"2023-10-09T06:20:41.259515+0000","last_became_active":"2023-10-09T06:16:44.962821+0000","last_became_peered":"2023-10-09T06:16:44.962821+0000","last_unstale":"2023-10-09T06:20:41.259515+0000","last_undegraded":"2023-10-09T06:20:41.259515+0000","last_fullsized":"2023-10-09T06:20:41.259515+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:17:30.632022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601754+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:20:50.601754+0000","last_peered":"2023-10-09T06:20:50.601754+0000","last_clean":"2023-10-09T06:20:50.601754+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:20:50.601754+0000","last_undegraded":"2023-10-09T06:20:50.601754+0000","last_fullsized":"2023-10-09T06:20:50.601754+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"203'9744","reported_seq":11440,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.634688+0000","last_change":"2023-10-09T06:20:30.273502+0000","last_active":"2023-10-09T06:20:58.634688+0000","last_peered":"2023-10-09T06:20:58.634688+0000","last_clean":"2023-10-09T06:20:58.634688+0000","last_became_active":"2023-10-09T06:20:30.253922+0000","last_became_peered":"2023-10-09T06:20:30.253922+0000","last_unstale":"2023-10-09T06:20:58.634688+0000","last_undegraded":"2023-10-09T06:20:58.634688+0000","last_fullsized":"2023-10-09T06:20:58.634688+0000","mapping_epoch":199,"log_start":"157'2300","ondisk_log_start":"157'2300","created":152,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7444,"log_dups_size":0,"ondisk_log_size":7444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:32.110407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165150720,"num_objects":2520,"num_object_clones":0,"num_object_copies":7560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9744,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245313+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:20:41.245313+0000","last_peered":"2023-10-09T06:20:41.245313+0000","last_clean":"2023-10-09T06:20:41.245313+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:20:41.245313+0000","last_undegraded":"2023-10-09T06:20:41.245313+0000","last_fullsized":"2023-10-09T06:20:41.245313+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246475+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:20:41.246475+0000","last_peered":"2023-10-09T06:20:41.246475+0000","last_clean":"2023-10-09T06:20:41.246475+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:20:41.246475+0000","last_undegraded":"2023-10-09T06:20:41.246475+0000","last_fullsized":"2023-10-09T06:20:41.246475+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"203'9377","reported_seq":9438,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:56.478204+0000","last_change":"2023-10-09T06:17:57.236716+0000","last_active":"2023-10-09T06:20:56.478204+0000","last_peered":"2023-10-09T06:20:56.478204+0000","last_clean":"2023-10-09T06:20:56.478204+0000","last_became_active":"2023-10-09T06:17:57.236363+0000","last_became_peered":"2023-10-09T06:17:57.236363+0000","last_unstale":"2023-10-09T06:20:56.478204+0000","last_undegraded":"2023-10-09T06:20:56.478204+0000","last_fullsized":"2023-10-09T06:20:56.478204+0000","mapping_epoch":152,"log_start":"156'2000","ondisk_log_start":"156'2000","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7377,"log_dups_size":0,"ondisk_log_size":7377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:01:53.348003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174129152,"num_objects":2657,"num_object_clones":0,"num_object_copies":7971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9377,"num_write_kb":385088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252368+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:20:41.252368+0000","last_peered":"2023-10-09T06:20:41.252368+0000","last_clean":"2023-10-09T06:20:41.252368+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:20:41.252368+0000","last_undegraded":"2023-10-09T06:20:41.252368+0000","last_fullsized":"2023-10-09T06:20:41.252368+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246078+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:20:41.246078+0000","last_peered":"2023-10-09T06:20:41.246078+0000","last_clean":"2023-10-09T06:20:41.246078+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:20:41.246078+0000","last_undegraded":"2023-10-09T06:20:41.246078+0000","last_fullsized":"2023-10-09T06:20:41.246078+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"203'9645","reported_seq":13219,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:57.844426+0000","last_change":"2023-10-09T06:20:21.206561+0000","last_active":"2023-10-09T06:20:57.844426+0000","last_peered":"2023-10-09T06:20:57.844426+0000","last_clean":"2023-10-09T06:20:57.844426+0000","last_became_active":"2023-10-09T06:20:21.163864+0000","last_became_peered":"2023-10-09T06:20:21.163864+0000","last_unstale":"2023-10-09T06:20:57.844426+0000","last_undegraded":"2023-10-09T06:20:57.844426+0000","last_fullsized":"2023-10-09T06:20:57.844426+0000","mapping_epoch":196,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:11.775536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9645,"num_write_kb":389952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1744,"num_bytes_recovered":114294784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":184,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246060+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:20:41.246060+0000","last_peered":"2023-10-09T06:20:41.246060+0000","last_clean":"2023-10-09T06:20:41.246060+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:20:41.246060+0000","last_undegraded":"2023-10-09T06:20:41.246060+0000","last_fullsized":"2023-10-09T06:20:41.246060+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247107+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:20:41.247107+0000","last_peered":"2023-10-09T06:20:41.247107+0000","last_clean":"2023-10-09T06:20:41.247107+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:20:41.247107+0000","last_undegraded":"2023-10-09T06:20:41.247107+0000","last_fullsized":"2023-10-09T06:20:41.247107+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"203'9456","reported_seq":9517,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.620958+0000","last_change":"2023-10-09T06:17:57.217463+0000","last_active":"2023-10-09T06:20:58.620958+0000","last_peered":"2023-10-09T06:20:58.620958+0000","last_clean":"2023-10-09T06:20:58.620958+0000","last_became_active":"2023-10-09T06:17:57.217100+0000","last_became_peered":"2023-10-09T06:17:57.217100+0000","last_unstale":"2023-10-09T06:20:58.620958+0000","last_undegraded":"2023-10-09T06:20:58.620958+0000","last_fullsized":"2023-10-09T06:20:58.620958+0000","mapping_epoch":152,"log_start":"156'2000","ondisk_log_start":"156'2000","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7456,"log_dups_size":0,"ondisk_log_size":7456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:23.024261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162136064,"num_objects":2474,"num_object_clones":0,"num_object_copies":7422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9456,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":157,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245533+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:20:41.245533+0000","last_peered":"2023-10-09T06:20:41.245533+0000","last_clean":"2023-10-09T06:20:41.245533+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:20:41.245533+0000","last_undegraded":"2023-10-09T06:20:41.245533+0000","last_fullsized":"2023-10-09T06:20:41.245533+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":132,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274064+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:20:41.274064+0000","last_peered":"2023-10-09T06:20:41.274064+0000","last_clean":"2023-10-09T06:20:41.274064+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:20:41.274064+0000","last_undegraded":"2023-10-09T06:20:41.274064+0000","last_fullsized":"2023-10-09T06:20:41.274064+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"203'9548","reported_seq":9607,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.103671+0000","last_change":"2023-10-09T06:17:57.022383+0000","last_active":"2023-10-09T06:20:58.103671+0000","last_peered":"2023-10-09T06:20:58.103671+0000","last_clean":"2023-10-09T06:20:58.103671+0000","last_became_active":"2023-10-09T06:17:57.022128+0000","last_became_peered":"2023-10-09T06:17:57.022128+0000","last_unstale":"2023-10-09T06:20:58.103671+0000","last_undegraded":"2023-10-09T06:20:58.103671+0000","last_fullsized":"2023-10-09T06:20:58.103671+0000","mapping_epoch":152,"log_start":"155'100","ondisk_log_start":"155'100","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9448,"log_dups_size":0,"ondisk_log_size":9448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:03.131013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165150720,"num_objects":2520,"num_object_clones":0,"num_object_copies":7560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":183,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274364+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:20:41.274364+0000","last_peered":"2023-10-09T06:20:41.274364+0000","last_clean":"2023-10-09T06:20:41.274364+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:20:41.274364+0000","last_undegraded":"2023-10-09T06:20:41.274364+0000","last_fullsized":"2023-10-09T06:20:41.274364+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263556+0000","last_change":"2023-10-09T06:16:44.962615+0000","last_active":"2023-10-09T06:20:41.263556+0000","last_peered":"2023-10-09T06:20:41.263556+0000","last_clean":"2023-10-09T06:20:41.263556+0000","last_became_active":"2023-10-09T06:16:44.962327+0000","last_became_peered":"2023-10-09T06:16:44.962327+0000","last_unstale":"2023-10-09T06:20:41.263556+0000","last_undegraded":"2023-10-09T06:20:41.263556+0000","last_fullsized":"2023-10-09T06:20:41.263556+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:41:14.831957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269971+0000","last_change":"2023-10-09T06:13:02.830803+0000","last_active":"2023-10-09T06:20:41.269971+0000","last_peered":"2023-10-09T06:20:41.269971+0000","last_clean":"2023-10-09T06:20:41.269971+0000","last_became_active":"2023-10-09T06:13:02.830500+0000","last_became_peered":"2023-10-09T06:13:02.830500+0000","last_unstale":"2023-10-09T06:20:41.269971+0000","last_undegraded":"2023-10-09T06:20:41.269971+0000","last_fullsized":"2023-10-09T06:20:41.269971+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:00.749647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"204'9736","reported_seq":11484,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:20:59.925985+0000","last_change":"2023-10-09T06:20:11.090804+0000","last_active":"2023-10-09T06:20:59.925985+0000","last_peered":"2023-10-09T06:20:59.925985+0000","last_clean":"2023-10-09T06:20:59.925985+0000","last_became_active":"2023-10-09T06:20:11.073402+0000","last_became_peered":"2023-10-09T06:20:11.073402+0000","last_unstale":"2023-10-09T06:20:59.925985+0000","last_undegraded":"2023-10-09T06:20:59.925985+0000","last_fullsized":"2023-10-09T06:20:59.925985+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9736,"log_dups_size":0,"ondisk_log_size":9736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:23.451973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164626432,"num_objects":2512,"num_object_clones":0,"num_object_copies":7536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9736,"num_write_kb":391936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":830,"num_bytes_recovered":54394880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":167,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.260912+0000","last_change":"2023-10-09T06:13:02.836137+0000","last_active":"2023-10-09T06:20:41.260912+0000","last_peered":"2023-10-09T06:20:41.260912+0000","last_clean":"2023-10-09T06:20:41.260912+0000","last_became_active":"2023-10-09T06:13:02.835345+0000","last_became_peered":"2023-10-09T06:13:02.835345+0000","last_unstale":"2023-10-09T06:20:41.260912+0000","last_undegraded":"2023-10-09T06:20:41.260912+0000","last_fullsized":"2023-10-09T06:20:41.260912+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:59:04.847975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247158+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:20:41.247158+0000","last_peered":"2023-10-09T06:20:41.247158+0000","last_clean":"2023-10-09T06:20:41.247158+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:20:41.247158+0000","last_undegraded":"2023-10-09T06:20:41.247158+0000","last_fullsized":"2023-10-09T06:20:41.247158+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274500+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:20:41.274500+0000","last_peered":"2023-10-09T06:20:41.274500+0000","last_clean":"2023-10-09T06:20:41.274500+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:20:41.274500+0000","last_undegraded":"2023-10-09T06:20:41.274500+0000","last_fullsized":"2023-10-09T06:20:41.274500+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"203'9698","reported_seq":11484,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.138641+0000","last_change":"2023-10-09T06:19:15.900457+0000","last_active":"2023-10-09T06:20:58.138641+0000","last_peered":"2023-10-09T06:20:58.138641+0000","last_clean":"2023-10-09T06:20:58.138641+0000","last_became_active":"2023-10-09T06:19:15.893387+0000","last_became_peered":"2023-10-09T06:19:15.893387+0000","last_unstale":"2023-10-09T06:20:58.138641+0000","last_undegraded":"2023-10-09T06:20:58.138641+0000","last_fullsized":"2023-10-09T06:20:58.138641+0000","mapping_epoch":170,"log_start":"155'300","ondisk_log_start":"155'300","created":152,"last_epoch_clean":171,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9398,"log_dups_size":0,"ondisk_log_size":9398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:09.689997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168427520,"num_objects":2570,"num_object_clones":0,"num_object_copies":7710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9698,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":850,"num_bytes_recovered":55705600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":162,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.600031+0000","last_change":"2023-10-09T06:16:44.951366+0000","last_active":"2023-10-09T06:20:50.600031+0000","last_peered":"2023-10-09T06:20:50.600031+0000","last_clean":"2023-10-09T06:20:50.600031+0000","last_became_active":"2023-10-09T06:16:44.951049+0000","last_became_peered":"2023-10-09T06:16:44.951049+0000","last_unstale":"2023-10-09T06:20:50.600031+0000","last_undegraded":"2023-10-09T06:20:50.600031+0000","last_fullsized":"2023-10-09T06:20:50.600031+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:51.815126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.269513+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:20:41.269513+0000","last_peered":"2023-10-09T06:20:41.269513+0000","last_clean":"2023-10-09T06:20:41.269513+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:20:41.269513+0000","last_undegraded":"2023-10-09T06:20:41.269513+0000","last_fullsized":"2023-10-09T06:20:41.269513+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274233+0000","last_change":"2023-10-09T06:13:02.839601+0000","last_active":"2023-10-09T06:20:41.274233+0000","last_peered":"2023-10-09T06:20:41.274233+0000","last_clean":"2023-10-09T06:20:41.274233+0000","last_became_active":"2023-10-09T06:13:02.839279+0000","last_became_peered":"2023-10-09T06:13:02.839279+0000","last_unstale":"2023-10-09T06:20:41.274233+0000","last_undegraded":"2023-10-09T06:20:41.274233+0000","last_fullsized":"2023-10-09T06:20:41.274233+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:17.978542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"203'9637","reported_seq":9698,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.578603+0000","last_change":"2023-10-09T06:17:57.016464+0000","last_active":"2023-10-09T06:20:58.578603+0000","last_peered":"2023-10-09T06:20:58.578603+0000","last_clean":"2023-10-09T06:20:58.578603+0000","last_became_active":"2023-10-09T06:17:57.016166+0000","last_became_peered":"2023-10-09T06:17:57.016166+0000","last_unstale":"2023-10-09T06:20:58.578603+0000","last_undegraded":"2023-10-09T06:20:58.578603+0000","last_fullsized":"2023-10-09T06:20:58.578603+0000","mapping_epoch":152,"log_start":"157'2200","ondisk_log_start":"157'2200","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7437,"log_dups_size":0,"ondisk_log_size":7437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:48:53.029953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174522368,"num_objects":2663,"num_object_clones":0,"num_object_copies":7989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9637,"num_write_kb":393600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":187,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.259339+0000","last_change":"2023-10-09T06:19:06.672499+0000","last_active":"2023-10-09T06:20:41.259339+0000","last_peered":"2023-10-09T06:20:41.259339+0000","last_clean":"2023-10-09T06:20:41.259339+0000","last_became_active":"2023-10-09T06:19:06.672182+0000","last_became_peered":"2023-10-09T06:19:06.672182+0000","last_unstale":"2023-10-09T06:20:41.259339+0000","last_undegraded":"2023-10-09T06:20:41.259339+0000","last_fullsized":"2023-10-09T06:20:41.259339+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:07.246328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":455,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.258659+0000","last_change":"2023-10-09T06:20:11.075324+0000","last_active":"2023-10-09T06:20:41.258659+0000","last_peered":"2023-10-09T06:20:41.258659+0000","last_clean":"2023-10-09T06:20:41.258659+0000","last_became_active":"2023-10-09T06:20:11.074888+0000","last_became_peered":"2023-10-09T06:20:11.074888+0000","last_unstale":"2023-10-09T06:20:41.258659+0000","last_undegraded":"2023-10-09T06:20:41.258659+0000","last_fullsized":"2023-10-09T06:20:41.258659+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T05:54:19.242593+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:03.264166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.607429+0000","last_change":"2023-10-09T06:16:44.961643+0000","last_active":"2023-10-09T06:20:50.607429+0000","last_peered":"2023-10-09T06:20:50.607429+0000","last_clean":"2023-10-09T06:20:50.607429+0000","last_became_active":"2023-10-09T06:16:44.961346+0000","last_became_peered":"2023-10-09T06:16:44.961346+0000","last_unstale":"2023-10-09T06:20:50.607429+0000","last_undegraded":"2023-10-09T06:20:50.607429+0000","last_fullsized":"2023-10-09T06:20:50.607429+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:45.585045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252594+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:20:41.252594+0000","last_peered":"2023-10-09T06:20:41.252594+0000","last_clean":"2023-10-09T06:20:41.252594+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:20:41.252594+0000","last_undegraded":"2023-10-09T06:20:41.252594+0000","last_fullsized":"2023-10-09T06:20:41.252594+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"204'9968","reported_seq":10030,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T06:20:59.961336+0000","last_change":"2023-10-09T06:17:57.236464+0000","last_active":"2023-10-09T06:20:59.961336+0000","last_peered":"2023-10-09T06:20:59.961336+0000","last_clean":"2023-10-09T06:20:59.961336+0000","last_became_active":"2023-10-09T06:17:57.236124+0000","last_became_peered":"2023-10-09T06:17:57.236124+0000","last_unstale":"2023-10-09T06:20:59.961336+0000","last_undegraded":"2023-10-09T06:20:59.961336+0000","last_fullsized":"2023-10-09T06:20:59.961336+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":9968,"log_dups_size":0,"ondisk_log_size":9968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:19:33.469044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161873920,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9968,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.258471+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:20:41.258471+0000","last_peered":"2023-10-09T06:20:41.258471+0000","last_clean":"2023-10-09T06:20:41.258471+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:20:41.258471+0000","last_undegraded":"2023-10-09T06:20:41.258471+0000","last_fullsized":"2023-10-09T06:20:41.258471+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":155,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601380+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:20:50.601380+0000","last_peered":"2023-10-09T06:20:50.601380+0000","last_clean":"2023-10-09T06:20:50.601380+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:20:50.601380+0000","last_undegraded":"2023-10-09T06:20:50.601380+0000","last_fullsized":"2023-10-09T06:20:50.601380+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601689+0000","last_change":"2023-10-09T06:15:11.873710+0000","last_active":"2023-10-09T06:20:50.601689+0000","last_peered":"2023-10-09T06:20:50.601689+0000","last_clean":"2023-10-09T06:20:50.601689+0000","last_became_active":"2023-10-09T06:15:11.873395+0000","last_became_peered":"2023-10-09T06:15:11.873395+0000","last_unstale":"2023-10-09T06:20:50.601689+0000","last_undegraded":"2023-10-09T06:20:50.601689+0000","last_fullsized":"2023-10-09T06:20:50.601689+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:46.619003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"203'9645","reported_seq":11438,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:57.899936+0000","last_change":"2023-10-09T06:19:52.709796+0000","last_active":"2023-10-09T06:20:57.899936+0000","last_peered":"2023-10-09T06:20:57.899936+0000","last_clean":"2023-10-09T06:20:57.899936+0000","last_became_active":"2023-10-09T06:19:52.651903+0000","last_became_peered":"2023-10-09T06:19:52.651903+0000","last_unstale":"2023-10-09T06:20:57.899936+0000","last_undegraded":"2023-10-09T06:20:57.899936+0000","last_fullsized":"2023-10-09T06:20:57.899936+0000","mapping_epoch":184,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:15:57.164182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174653440,"num_objects":2665,"num_object_clones":0,"num_object_copies":7995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9645,"num_write_kb":393920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":46,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.262961+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:20:41.262961+0000","last_peered":"2023-10-09T06:20:41.262961+0000","last_clean":"2023-10-09T06:20:41.262961+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:20:41.262961+0000","last_undegraded":"2023-10-09T06:20:41.262961+0000","last_fullsized":"2023-10-09T06:20:41.262961+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.274450+0000","last_change":"2023-10-09T06:15:11.873771+0000","last_active":"2023-10-09T06:20:41.274450+0000","last_peered":"2023-10-09T06:20:41.274450+0000","last_clean":"2023-10-09T06:20:41.274450+0000","last_became_active":"2023-10-09T06:15:11.873481+0000","last_became_peered":"2023-10-09T06:15:11.873481+0000","last_unstale":"2023-10-09T06:20:41.274450+0000","last_undegraded":"2023-10-09T06:20:41.274450+0000","last_fullsized":"2023-10-09T06:20:41.274450+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.089628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601618+0000","last_change":"2023-10-09T06:19:06.667489+0000","last_active":"2023-10-09T06:20:50.601618+0000","last_peered":"2023-10-09T06:20:50.601618+0000","last_clean":"2023-10-09T06:20:50.601618+0000","last_became_active":"2023-10-09T06:19:06.667029+0000","last_became_peered":"2023-10-09T06:19:06.667029+0000","last_unstale":"2023-10-09T06:20:50.601618+0000","last_undegraded":"2023-10-09T06:20:50.601618+0000","last_fullsized":"2023-10-09T06:20:50.601618+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:35.989088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"203'9605","reported_seq":11401,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:57.881544+0000","last_change":"2023-10-09T06:19:26.121898+0000","last_active":"2023-10-09T06:20:57.881544+0000","last_peered":"2023-10-09T06:20:57.881544+0000","last_clean":"2023-10-09T06:20:57.881544+0000","last_became_active":"2023-10-09T06:19:26.079797+0000","last_became_peered":"2023-10-09T06:19:26.079797+0000","last_unstale":"2023-10-09T06:20:57.881544+0000","last_undegraded":"2023-10-09T06:20:57.881544+0000","last_fullsized":"2023-10-09T06:20:57.881544+0000","mapping_epoch":173,"log_start":"156'1900","ondisk_log_start":"156'1900","created":152,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7705,"log_dups_size":0,"ondisk_log_size":7705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:05.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461464,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9605,"num_write_kb":388609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246133+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:20:41.246133+0000","last_peered":"2023-10-09T06:20:41.246133+0000","last_clean":"2023-10-09T06:20:41.246133+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:20:41.246133+0000","last_undegraded":"2023-10-09T06:20:41.246133+0000","last_fullsized":"2023-10-09T06:20:41.246133+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.276065+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:20:41.276065+0000","last_peered":"2023-10-09T06:20:41.276065+0000","last_clean":"2023-10-09T06:20:41.276065+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:20:41.276065+0000","last_undegraded":"2023-10-09T06:20:41.276065+0000","last_fullsized":"2023-10-09T06:20:41.276065+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.634819+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:20:43.634819+0000","last_peered":"2023-10-09T06:20:43.634819+0000","last_clean":"2023-10-09T06:20:43.634819+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:20:43.634819+0000","last_undegraded":"2023-10-09T06:20:43.634819+0000","last_fullsized":"2023-10-09T06:20:43.634819+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"203'9458","reported_seq":11113,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:58.612249+0000","last_change":"2023-10-09T06:19:34.580478+0000","last_active":"2023-10-09T06:20:58.612249+0000","last_peered":"2023-10-09T06:20:58.612249+0000","last_clean":"2023-10-09T06:20:58.612249+0000","last_became_active":"2023-10-09T06:19:34.573414+0000","last_became_peered":"2023-10-09T06:19:34.573414+0000","last_unstale":"2023-10-09T06:20:58.612249+0000","last_undegraded":"2023-10-09T06:20:58.612249+0000","last_fullsized":"2023-10-09T06:20:58.612249+0000","mapping_epoch":176,"log_start":"156'2000","ondisk_log_start":"156'2000","created":152,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:17:55.985102+0000","last_clean_scrub_stamp":"2023-10-09T06:17:55.985102+0000","objects_scrubbed":0,"log_size":7458,"log_dups_size":0,"ondisk_log_size":7458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:08.324993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9458,"num_write_kb":383808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":784,"num_bytes_recovered":51380224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":187,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.252927+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:20:41.252927+0000","last_peered":"2023-10-09T06:20:41.252927+0000","last_clean":"2023-10-09T06:20:41.252927+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:20:41.252927+0000","last_undegraded":"2023-10-09T06:20:41.252927+0000","last_fullsized":"2023-10-09T06:20:41.252927+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":158,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601548+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:20:50.601548+0000","last_peered":"2023-10-09T06:20:50.601548+0000","last_clean":"2023-10-09T06:20:50.601548+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:20:50.601548+0000","last_undegraded":"2023-10-09T06:20:50.601548+0000","last_fullsized":"2023-10-09T06:20:50.601548+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246136+0000","last_change":"2023-10-09T05:57:53.537401+0000","last_active":"2023-10-09T06:20:41.246136+0000","last_peered":"2023-10-09T06:20:41.246136+0000","last_clean":"2023-10-09T06:20:41.246136+0000","last_became_active":"2023-10-09T05:57:53.537172+0000","last_became_peered":"2023-10-09T05:57:53.537172+0000","last_unstale":"2023-10-09T06:20:41.246136+0000","last_undegraded":"2023-10-09T06:20:41.246136+0000","last_fullsized":"2023-10-09T06:20:41.246136+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:03.619283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":161,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601839+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:20:50.601839+0000","last_peered":"2023-10-09T06:20:50.601839+0000","last_clean":"2023-10-09T06:20:50.601839+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:20:50.601839+0000","last_undegraded":"2023-10-09T06:20:50.601839+0000","last_fullsized":"2023-10-09T06:20:50.601839+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.599622+0000","last_change":"2023-10-09T06:16:44.958199+0000","last_active":"2023-10-09T06:20:50.599622+0000","last_peered":"2023-10-09T06:20:50.599622+0000","last_clean":"2023-10-09T06:20:50.599622+0000","last_became_active":"2023-10-09T06:16:44.957668+0000","last_became_peered":"2023-10-09T06:16:44.957668+0000","last_unstale":"2023-10-09T06:20:50.599622+0000","last_undegraded":"2023-10-09T06:20:50.599622+0000","last_fullsized":"2023-10-09T06:20:50.599622+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:25.870084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.259621+0000","last_change":"2023-10-09T06:13:02.835890+0000","last_active":"2023-10-09T06:20:41.259621+0000","last_peered":"2023-10-09T06:20:41.259621+0000","last_clean":"2023-10-09T06:20:41.259621+0000","last_became_active":"2023-10-09T06:13:02.835517+0000","last_became_peered":"2023-10-09T06:13:02.835517+0000","last_unstale":"2023-10-09T06:20:41.259621+0000","last_undegraded":"2023-10-09T06:20:41.259621+0000","last_fullsized":"2023-10-09T06:20:41.259621+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:38.648184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.247082+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:20:41.247082+0000","last_peered":"2023-10-09T06:20:41.247082+0000","last_clean":"2023-10-09T06:20:41.247082+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:20:41.247082+0000","last_undegraded":"2023-10-09T06:20:41.247082+0000","last_fullsized":"2023-10-09T06:20:41.247082+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":148,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.263646+0000","last_change":"2023-10-09T05:57:53.254450+0000","last_active":"2023-10-09T06:20:41.263646+0000","last_peered":"2023-10-09T06:20:41.263646+0000","last_clean":"2023-10-09T06:20:41.263646+0000","last_became_active":"2023-10-09T05:57:53.254138+0000","last_became_peered":"2023-10-09T05:57:53.254138+0000","last_unstale":"2023-10-09T06:20:41.263646+0000","last_undegraded":"2023-10-09T06:20:41.263646+0000","last_fullsized":"2023-10-09T06:20:41.263646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:04.317030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245997+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:20:41.245997+0000","last_peered":"2023-10-09T06:20:41.245997+0000","last_clean":"2023-10-09T06:20:41.245997+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:20:41.245997+0000","last_undegraded":"2023-10-09T06:20:41.245997+0000","last_fullsized":"2023-10-09T06:20:41.245997+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":159,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:43.638563+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:20:43.638563+0000","last_peered":"2023-10-09T06:20:43.638563+0000","last_clean":"2023-10-09T06:20:43.638563+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:20:43.638563+0000","last_undegraded":"2023-10-09T06:20:43.638563+0000","last_fullsized":"2023-10-09T06:20:43.638563+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.246076+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:20:41.246076+0000","last_peered":"2023-10-09T06:20:41.246076+0000","last_clean":"2023-10-09T06:20:41.246076+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:20:41.246076+0000","last_undegraded":"2023-10-09T06:20:41.246076+0000","last_fullsized":"2023-10-09T06:20:41.246076+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":159,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-10-09T06:20:50.601879+0000","last_change":"2023-10-09T06:15:11.873845+0000","last_active":"2023-10-09T06:20:50.601879+0000","last_peered":"2023-10-09T06:20:50.601879+0000","last_clean":"2023-10-09T06:20:50.601879+0000","last_became_active":"2023-10-09T06:15:11.873504+0000","last_became_peered":"2023-10-09T06:15:11.873504+0000","last_unstale":"2023-10-09T06:20:50.601879+0000","last_undegraded":"2023-10-09T06:20:50.601879+0000","last_fullsized":"2023-10-09T06:20:50.601879+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:40.957044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-10-09T06:20:41.245884+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:20:41.245884+0000","last_peered":"2023-10-09T06:20:41.245884+0000","last_clean":"2023-10-09T06:20:41.245884+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:20:41.245884+0000","last_undegraded":"2023-10-09T06:20:41.245884+0000","last_fullsized":"2023-10-09T06:20:41.245884+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":9684,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2691825688,"num_objects":41075,"num_object_clones":0,"num_object_copies":123225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41075,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154065,"num_write_kb":6244417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8419,"num_bytes_recovered":551747584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8148168704,"data_stored":8148156488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134765,"ondisk_log_size":134765,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234284,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2387640,"kb_used_data":1491504,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":91357512,"statfs":{"total":95995035648,"available":93550092288,"internally_reserved":0,"allocated":1527300096,"data_stored":1526620779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.071,"15min":1.278},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.3600000000000003,"5min":10.388,"15min":11.785},"last":3.0430000000000001},{"interface":"front","average":{"1min":1.494,"5min":1.0329999999999999,"15min":1.3149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":4.4500000000000002,"5min":10.396000000000001,"15min":10.784000000000001},"last":11.819000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.9569999999999999,"15min":2.093},"min":{"1min":0.28999999999999998,"5min":0.25,"15min":0.25},"max":{"1min":17.774000000000001,"5min":31.869,"15min":31.869},"last":0.28499999999999998},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.181,"15min":1.7569999999999999},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.747,"5min":29.486999999999998,"15min":29.486999999999998},"last":2.089}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.51,"15min":1.8700000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.3559999999999999,"5min":8.6950000000000003,"15min":8.6950000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.6020000000000001,"15min":1.9970000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":7.4189999999999996,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":2.3660000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:20:54 2023","interfaces":[{"interface":"back","average":{"1min":5.0220000000000002,"5min":1.9370000000000001,"15min":1.0369999999999999},"min":{"1min":0.39600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":24.478000000000002,"5min":24.478000000000002,"15min":24.478000000000002},"last":11.862},{"interface":"front","average":{"1min":5.1660000000000004,"5min":1.9430000000000001,"15min":1.0629999999999999},"min":{"1min":0.36199999999999999,"5min":0.312,"15min":0.312},"max":{"1min":24.611999999999998,"5min":24.611999999999998,"15min":24.611999999999998},"last":11.919}]},{"osd":4,"last update":"Mon Oct 9 06:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.4810000000000001,"15min":1.361},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.251},"max":{"1min":7.4749999999999996,"5min":25.678999999999998,"15min":25.678999999999998},"last":2.0190000000000001},{"interface":"front","average":{"1min":1.8009999999999999,"5min":1.1879999999999999,"15min":1.2609999999999999},"min":{"1min":0.32600000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":13.305,"5min":13.305,"15min":13.305},"last":13.305}]},{"osd":5,"last update":"Mon Oct 9 06:20:54 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":2.298,"15min":1.681},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.0910000000000002,"5min":50.304000000000002,"15min":50.304000000000002},"last":7.0110000000000001},{"interface":"front","average":{"1min":1.7609999999999999,"5min":2.004,"15min":1.4370000000000001},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.452,"5min":48.774999999999999,"15min":48.774999999999999},"last":2.056}]},{"osd":6,"last update":"Mon Oct 9 06:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":1.22,"15min":1.3100000000000001},"min":{"1min":0.41199999999999998,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":7.3949999999999996,"5min":8.2249999999999996,"15min":11.878},"last":2.1190000000000002},{"interface":"front","average":{"1min":1.9399999999999999,"5min":1.2709999999999999,"15min":1.3180000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":12.141999999999999,"5min":12.141999999999999,"15min":12.141999999999999},"last":12.141999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495921,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":806496,"kb_used_data":496920,"kb_used_omap":0,"kb_used_meta":309568,"kb_avail":92938656,"statfs":{"total":95995035648,"available":95169183744,"internally_reserved":0,"allocated":508846080,"data_stored":508190581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316997632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:44 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":3.7429999999999999,"15min":2.3620000000000001},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.214},"max":{"1min":7.5919999999999996,"5min":109.092,"15min":109.092},"last":110.926},{"interface":"front","average":{"1min":2.6219999999999999,"5min":4.2089999999999996,"15min":2.5470000000000002},"min":{"1min":0.30199999999999999,"5min":0.28299999999999997,"15min":0.24399999999999999},"max":{"1min":9.7609999999999992,"5min":109.369,"15min":109.369},"last":110.911}]},{"osd":1,"last update":"Mon Oct 9 06:20:11 2023","interfaces":[{"interface":"back","average":{"1min":16.698,"5min":5.1689999999999996,"15min":3.2919999999999998},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":114.202,"5min":114.202,"15min":114.202},"last":0.46100000000000002},{"interface":"front","average":{"1min":15.648999999999999,"5min":5.0220000000000002,"15min":3.2719999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.246},"max":{"1min":114.179,"5min":114.179,"15min":114.179},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:20:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.7410000000000001,"15min":2.2410000000000001},"min":{"1min":0.317,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.207000000000001,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.502},{"interface":"front","average":{"1min":1.016,"5min":1.3939999999999999,"15min":2.2610000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.734,"5min":8.9870000000000001,"15min":8.9870000000000001},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 06:20:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":2.2690000000000001,"15min":1.117},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.3059999999999992,"5min":64.012,"15min":64.012},"last":110.358},{"interface":"front","average":{"1min":3.3919999999999999,"5min":2.3769999999999998,"15min":1.167},"min":{"1min":0.35499999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":15.507999999999999,"5min":64.105999999999995,"15min":64.105999999999995},"last":110.43899999999999}]},{"osd":4,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":1.474,"15min":1.1659999999999999},"min":{"1min":0.33400000000000002,"5min":0.252,"15min":0.245},"max":{"1min":17.643999999999998,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.159,"5min":1.075,"15min":1.2030000000000001},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.26400000000000001},"max":{"1min":3.6549999999999998,"5min":8.9239999999999995,"15min":9.4100000000000001},"last":0.434}]},{"osd":5,"last update":"Mon Oct 9 06:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.514,"15min":1.29},"min":{"1min":0.34599999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.2410000000000001,"15min":1.827},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":8.0630000000000006,"5min":9.0169999999999995,"15min":159.06800000000001},"last":0.39400000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0169999999999999,"15min":1.071},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":4.2640000000000002,"5min":10.962,"15min":14.837},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.008,"5min":1.177,"15min":1.393},"min":{"1min":0.377,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":4.9939999999999998,"5min":8.9920000000000009,"15min":10.891},"last":0.47999999999999998}]}]},{"osd":1,"up_from":110,"seq":472446402657,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1898360,"kb_used_data":1525040,"kb_used_omap":4,"kb_used_meta":373307,"kb_avail":91846792,"statfs":{"total":95995035648,"available":94051115008,"internally_reserved":0,"allocated":1561640960,"data_stored":1560727482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":382267278},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":7.2290000000000001,"5min":2.5230000000000001,"15min":2.661},"min":{"1min":0.20799999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":148.98400000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.4500000000000002,"15min":2.948},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20100000000000001},"max":{"1min":149.02699999999999,"5min":149.02699999999999,"15min":149.02699999999999},"last":3.6930000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.9219999999999999,"15min":1.796},"min":{"1min":0.23799999999999999,"5min":0.193,"15min":0.193},"max":{"1min":7.3399999999999999,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.9350000000000001,"15min":2.5619999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":4.9690000000000003,"5min":49.783000000000001,"15min":49.783000000000001},"last":3.7450000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:19:54 2023","interfaces":[{"interface":"back","average":{"1min":10.465,"5min":2.6920000000000002,"15min":1.1519999999999999},"min":{"1min":0.30399999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":153.18199999999999,"5min":153.18199999999999,"15min":153.18199999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":10.079000000000001,"5min":2.5569999999999999,"15min":1.1240000000000001},"min":{"1min":0.26400000000000001,"5min":0.25,"15min":0.25},"max":{"1min":152.52000000000001,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":10.198,"5min":3.0299999999999998,"15min":3.0430000000000001},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":199.43799999999999,"5min":199.43799999999999,"15min":199.43799999999999},"last":12.609999999999999},{"interface":"front","average":{"1min":8.7400000000000002,"5min":2.956,"15min":2.9940000000000002},"min":{"1min":0.39000000000000001,"5min":0.315,"15min":0.311},"max":{"1min":199.32900000000001,"5min":199.32900000000001,"15min":199.32900000000001},"last":9.4689999999999994}]},{"osd":5,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.6619999999999999,"15min":2.9060000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":61.930999999999997,"5min":61.930999999999997,"15min":61.930999999999997},"last":0.80600000000000005},{"interface":"front","average":{"1min":3.2069999999999999,"5min":1.958,"15min":2.879},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":61.947000000000003,"5min":61.947000000000003,"15min":61.947000000000003},"last":3.806}]},{"osd":6,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.8380000000000001,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":4.274,"5min":12.91,"15min":12.91},"last":3.778},{"interface":"front","average":{"1min":2.5150000000000001,"5min":1.7629999999999999,"15min":2.3769999999999998},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:20:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.772,"15min":2.5139999999999998},"min":{"1min":0.378,"5min":0.376,"15min":0.376},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.3819999999999999,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":6.7270000000000003,"5min":9.2370000000000001,"15min":9.2370000000000001},"last":0.47899999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024091,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1102368,"kb_used_data":657112,"kb_used_omap":5,"kb_used_meta":445242,"kb_avail":92642784,"statfs":{"total":95995035648,"available":94866210816,"internally_reserved":0,"allocated":672882688,"data_stored":671819091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":455928442},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:20:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5099999999999998,"5min":2.133,"15min":2.0139999999999998},"min":{"1min":0.26600000000000001,"5min":0.19,"15min":0.19},"max":{"1min":29.021999999999998,"5min":30.733000000000001,"15min":30.733000000000001},"last":0.442},{"interface":"front","average":{"1min":3.3849999999999998,"5min":2.6389999999999998,"15min":2.7040000000000002},"min":{"1min":0.33700000000000002,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":29.077999999999999,"5min":30.279,"15min":30.279},"last":0.58999999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:20:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":1.762,"15min":2.3730000000000002},"min":{"1min":0.248,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":9.9909999999999997,"5min":17.588999999999999,"15min":17.588999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.3889999999999998},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":11.162000000000001,"5min":21.198,"15min":21.198},"last":8.5030000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:20:52 2023","interfaces":[{"interface":"back","average":{"1min":17.802,"5min":4.9589999999999996,"15min":1.909},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":185.89400000000001,"5min":185.89400000000001,"15min":185.89400000000001},"last":8.6449999999999996},{"interface":"front","average":{"1min":19.030000000000001,"5min":5.2480000000000002,"15min":2.016},"min":{"1min":0.38700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":186.72900000000001,"5min":186.72900000000001,"15min":186.72900000000001},"last":5.0490000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.1819999999999999,"15min":1.647},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":4.3739999999999997,"5min":11.162000000000001,"15min":15.835000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.137,"5min":1.8129999999999999,"15min":2.133},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":7.6210000000000004,"5min":12.752000000000001,"15min":15.023},"last":0.63}]},{"osd":5,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.6060000000000001,"15min":1.895},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.567,"5min":11.222,"15min":15.858000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.589,"5min":2.0350000000000001,"15min":2.226},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":9.6400000000000006,"5min":21.167000000000002,"15min":31.984999999999999},"last":8.4550000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.657,"5min":2.1509999999999998,"15min":2.2069999999999999},"min":{"1min":0.45700000000000002,"5min":0.34499999999999997,"15min":0.29299999999999998},"max":{"1min":9.6880000000000006,"5min":21.088999999999999,"15min":32.079000000000001},"last":8.5370000000000008},{"interface":"front","average":{"1min":2.3210000000000002,"5min":1.542,"15min":1.978},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":10.606,"5min":11.236000000000001,"15min":15.893000000000001},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:20:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.796,"15min":2.2080000000000002},"min":{"1min":0.38500000000000001,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":9.9269999999999996,"5min":12.798999999999999,"15min":15.050000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.974,"15min":2.2549999999999999},"min":{"1min":0.433,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":8.2219999999999995,"5min":21.135999999999999,"15min":32.055999999999997},"last":8.5660000000000007}]}]},{"osd":2,"up_from":128,"seq":549755813960,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":720744,"kb_used_data":477216,"kb_used_omap":7,"kb_used_meta":243512,"kb_avail":93024408,"statfs":{"total":95995035648,"available":95256993792,"internally_reserved":0,"allocated":488669184,"data_stored":487776487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":249356369},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":5.3449999999999998,"15min":3.0710000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":88.225999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":0.157},{"interface":"front","average":{"1min":5.8730000000000002,"5min":5.4880000000000004,"15min":3.6520000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":85.781000000000006,"5min":163.363,"15min":163.363},"last":0.214}]},{"osd":1,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":4.29,"15min":2.6669999999999998},"min":{"1min":0.24299999999999999,"5min":0.188,"15min":0.188},"max":{"1min":4.5679999999999996,"5min":222.50299999999999,"15min":222.50299999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":4.3620000000000001,"15min":3.2959999999999998},"min":{"1min":0.23000000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.8069999999999999,"5min":222.19399999999999,"15min":222.19399999999999},"last":0.23999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:20:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":2.2130000000000001,"15min":0.97199999999999998},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":5.694,"5min":81.183999999999997,"15min":81.183999999999997},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.411,"5min":2.25,"15min":1.0309999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.042999999999999,"5min":81.061999999999998,"15min":81.061999999999998},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":1.532,"15min":1.97},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.106999999999999,"5min":12.141999999999999,"15min":12.141999999999999},"last":3.5430000000000001},{"interface":"front","average":{"1min":2.7599999999999998,"5min":1.6870000000000001,"15min":2.2309999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.994999999999999,"5min":15.183,"15min":15.183},"last":3.573}]},{"osd":5,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":1.3089999999999999,"15min":1.6499999999999999},"min":{"1min":0.36399999999999999,"5min":0.309,"15min":0.309},"max":{"1min":4.9989999999999997,"5min":5.7460000000000004,"15min":5.7460000000000004},"last":1.609},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9079999999999999,"15min":2.3919999999999999},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.6840000000000002,"5min":10.215999999999999,"15min":10.215999999999999},"last":1.804}]},{"osd":6,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.5109999999999999,"15min":2.1259999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.4939999999999998,"5min":6.9779999999999998,"15min":6.9779999999999998},"last":1.093},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.4119999999999999,"15min":1.6140000000000001},"min":{"1min":0.39300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.883,"5min":7.327,"15min":7.327},"last":2.0710000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:20:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.583,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.6660000000000004,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":0.93200000000000005},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.7310000000000001,"15min":2.7549999999999999},"min":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.7779999999999996,"5min":9.3249999999999993,"15min":9.3249999999999993},"last":5.2030000000000003}]}]},{"osd":3,"up_from":149,"seq":639950127157,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541108,"kb_used_data":1310764,"kb_used_omap":2,"kb_used_meta":230077,"kb_avail":92204044,"statfs":{"total":95995035648,"available":94416941056,"internally_reserved":0,"allocated":1342222336,"data_stored":1341428465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":235598901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":1.3,"15min":0.84599999999999997},"min":{"1min":0.23300000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":8.9160000000000004,"5min":9.2729999999999997,"15min":9.2729999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.5680000000000001,"5min":1.7390000000000001,"15min":1.2110000000000001},"min":{"1min":0.223,"5min":0.191,"15min":0.191},"max":{"1min":11.946,"5min":11.946,"15min":11.946},"last":4.9930000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.8580000000000001,"15min":1.2509999999999999},"min":{"1min":0.26400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":16.260000000000002,"5min":16.260000000000002,"15min":16.260000000000002},"last":5.0810000000000004},{"interface":"front","average":{"1min":2.1179999999999999,"5min":1.5329999999999999,"15min":0.91700000000000004},"min":{"1min":0.25700000000000001,"5min":0.254,"15min":0.254},"max":{"1min":8.6140000000000008,"5min":12.757999999999999,"15min":12.757999999999999},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8610000000000002,"5min":1.768,"15min":1.218},"min":{"1min":0.28599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.871,"5min":11.871,"15min":11.871},"last":5.0510000000000002},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.1080000000000001,"15min":0.65000000000000002},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.6430000000000007,"5min":9.2230000000000008,"15min":9.2230000000000008},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.139,"15min":0.79100000000000004},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.64000000000000001},{"interface":"front","average":{"1min":2.4740000000000002,"5min":1.462,"15min":1.282},"min":{"1min":0.39900000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.6899999999999995,"5min":11.742000000000001,"15min":11.742000000000001},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4740000000000002,"5min":1.847,"15min":1.431},"min":{"1min":0.39000000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":15.041,"5min":15.041,"15min":15.041},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.6930000000000001,"5min":1.8169999999999999,"15min":1.4239999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":15.109999999999999,"5min":15.109999999999999,"15min":15.109999999999999},"last":5.4320000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.665,"15min":1.351},"min":{"1min":0.38900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.7710000000000008,"5min":11.798,"15min":11.798},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.875,"5min":1.587,"15min":1.327},"min":{"1min":0.36299999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.3559999999999999,"5min":11.552,"15min":11.552},"last":5.5410000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:20:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.3400000000000001,"15min":0.874},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.843,"5min":9.843,"15min":9.843},"last":0.751},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.6060000000000001,"15min":1.3460000000000001},"min":{"1min":0.44600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.7590000000000003,"5min":11.823,"15min":11.823},"last":0.58399999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019193,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1099752,"kb_used_data":509920,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":92645400,"statfs":{"total":95995035648,"available":94868889600,"internally_reserved":0,"allocated":522158080,"data_stored":521494244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.4370000000000001,"15min":1.4670000000000001},"min":{"1min":0.26000000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":19.577000000000002,"5min":19.577000000000002,"15min":19.577000000000002},"last":1.663},{"interface":"front","average":{"1min":3.0830000000000002,"5min":1.7430000000000001,"15min":1.837},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":19.594999999999999,"5min":19.594999999999999,"15min":19.594999999999999},"last":2.0990000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.5369999999999999,"15min":1.9470000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":34.412999999999997,"5min":91.849999999999994,"15min":91.849999999999994},"last":1.6930000000000001},{"interface":"front","average":{"1min":3.71,"5min":2.7210000000000001,"15min":2},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":34.604999999999997,"5min":91.885000000000005,"15min":91.885000000000005},"last":1.677}]},{"osd":2,"last update":"Mon Oct 9 06:20:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0960000000000001,"5min":1.524,"15min":1.778},"min":{"1min":0.316,"5min":0.255,"15min":0.255},"max":{"1min":6.8650000000000002,"5min":9.6739999999999995,"15min":9.6739999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.831,"5min":1.8029999999999999,"15min":1.9450000000000001},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.898,"5min":15.894,"15min":15.894},"last":2.1619999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:20:51 2023","interfaces":[{"interface":"back","average":{"1min":7.4139999999999997,"5min":3.597,"15min":1.5740000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":57.826999999999998,"5min":91.697000000000003,"15min":91.697000000000003},"last":2.1880000000000002},{"interface":"front","average":{"1min":8.1120000000000001,"5min":3.4510000000000001,"15min":1.5649999999999999},"min":{"1min":0.35699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":57.853000000000002,"5min":94.356999999999999,"15min":94.356999999999999},"last":1.71}]},{"osd":5,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.1639999999999999,"15min":1.0629999999999999},"min":{"1min":0.26300000000000001,"5min":0.254,"15min":0.24299999999999999},"max":{"1min":6.9000000000000004,"5min":6.9000000000000004,"15min":11.130000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.149,"5min":1.2929999999999999,"15min":1.353},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":12.552,"5min":12.552,"15min":12.552},"last":2.141}]},{"osd":6,"last update":"Mon Oct 9 06:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.70399999999999996,"15min":0.89200000000000002},"min":{"1min":0.28299999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.58999999999999997,"5min":4.907,"15min":9.7680000000000007},"last":1.647},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.4099999999999999,"15min":1.212},"min":{"1min":0.33500000000000002,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":7.5380000000000003,"5min":14.202999999999999,"15min":14.202999999999999},"last":3.3530000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:20:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.6419999999999999,"15min":1.2430000000000001},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":7.4610000000000003,"5min":49.648000000000003,"15min":49.648000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.6859999999999999,"15min":1.272},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":7.5960000000000001,"5min":49.750999999999998,"15min":49.750999999999998},"last":1.615}]}]},{"osd":5,"up_from":38,"seq":163208757558,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2434588,"kb_used_data":1536980,"kb_used_omap":0,"kb_used_meta":897600,"kb_avail":91310564,"statfs":{"total":95995035648,"available":93502017536,"internally_reserved":0,"allocated":1573867520,"data_stored":1573211682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919142400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.6599999999999999,"15min":1.675},"min":{"1min":0.216,"5min":0.216,"15min":0.215},"max":{"1min":6.8159999999999998,"5min":8.5139999999999993,"15min":18.637},"last":3.0649999999999999},{"interface":"front","average":{"1min":1.909,"5min":1.4690000000000001,"15min":1.6799999999999999},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":6.1790000000000003,"5min":8.4019999999999992,"15min":9.9939999999999998},"last":2.9279999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.423,"15min":2.125},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.8520000000000003,"5min":8.8170000000000002,"15min":11.292},"last":3.198},{"interface":"front","average":{"1min":3.1200000000000001,"5min":1.645,"15min":2.0859999999999999},"min":{"1min":0.437,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.467000000000001,"5min":10.467000000000001,"15min":10.467000000000001},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6339999999999999,"15min":2.2029999999999998},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.307},"max":{"1min":10.42,"5min":10.42,"15min":10.42},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.946,"5min":1.4259999999999999,"15min":2.044},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.5990000000000002,"5min":7.7510000000000003,"15min":7.7510000000000003},"last":1.7350000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:20:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4350000000000001,"5min":5.0060000000000002,"15min":2.085},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.9560000000000004,"5min":190.411,"15min":190.411},"last":3.4159999999999999},{"interface":"front","average":{"1min":3.069,"5min":4.9039999999999999,"15min":2.0539999999999998},"min":{"1min":0.38,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.33,"5min":190.70699999999999,"15min":190.70699999999999},"last":3.552}]},{"osd":4,"last update":"Mon Oct 9 06:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.4490000000000001,"15min":1.2210000000000001},"min":{"1min":0.28299999999999997,"5min":0.26200000000000001,"15min":0.20999999999999999},"max":{"1min":8.6389999999999993,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":0.999},{"interface":"front","average":{"1min":1.704,"5min":1.117,"15min":1.0660000000000001},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.25900000000000001},"max":{"1min":5.0570000000000004,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":1.661}]},{"osd":6,"last update":"Mon Oct 9 06:20:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5590000000000002,"5min":1.593,"15min":1.304},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.28899999999999998},"max":{"1min":13.567,"5min":13.567,"15min":13.567},"last":1.6930000000000001},{"interface":"front","average":{"1min":3.2589999999999999,"5min":1.5109999999999999,"15min":1.2190000000000001},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.23899999999999999},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.82099999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:20:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2570000000000001,"5min":1.534,"15min":1.2330000000000001},"min":{"1min":0.38900000000000001,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":9.5800000000000001,"5min":9.5800000000000001,"15min":9.5800000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.7749999999999999,"5min":1.2689999999999999,"15min":1.159},"min":{"1min":0.40000000000000002,"5min":0.371,"15min":0.29299999999999998},"max":{"1min":4.3310000000000004,"5min":7.0049999999999999,"15min":9.4580000000000002},"last":1.7130000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":663224320,"data_stored":663224320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1559040000,"data_stored":1559035928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":483917824,"data_stored":483917824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1332019200,"data_stored":1332019200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":513146880,"data_stored":513146880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1567363072,"data_stored":1567359000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":506134528,"data_stored":506134528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1523322880,"data_stored":1523318808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:21:03.888 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:21:03.888 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-10-09T06:21:03.889 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:03 smithi107 ceph-mon[212750]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:21:03.889 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:03 smithi107 ceph-mon[220921]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:21:03.889 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:03.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:03 smithi150 ceph-mon[194178]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:21:04.006 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:04.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:04.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:04.185 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:04.277 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:04.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:04.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:04.449 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:04.528 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:04.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:04.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:04.718 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:04.857 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:04.866 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:04.881 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:04 smithi107 ceph-mon[212750]: pgmap v663: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:21:04.881 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:04 smithi107 ceph-mon[220921]: pgmap v663: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:21:04.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:04.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:04.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:04 smithi150 ceph-mon[194178]: pgmap v663: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-09T06:21:04.987 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.34644.0:193194 8.d e134/129 8:b85abec2:::benchmark_data_smithi107_251897_object76280:head v 134'11804, mlcod=124'10248)", 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 328.86227814599999, 2023-10-09T06:21:05.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.037746214, 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133907+0000", 2023-10-09T06:21:05.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133906+0000", 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133913+0000", 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7150000000000001e-06 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133923+0000", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5629999999999998e-06 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.133928+0000", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0980000000000001e-06 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.146011+0000", 2023-10-09T06:21:05.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012083534999999999 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.146038+0000", 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7044000000000001e-05 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.171622+0000", 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025583854999999999 2023-10-09T06:21:05.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:15:36.171653+0000", 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1019999999999998e-05 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231195 8.f 8:f3e206e0:::benchmark_data_smithi107_251897_object114281:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:21:05.007 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.394444174, 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.136219622, 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231195 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:21:05.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601741+0000", 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601739+0000", 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601747+0000", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4429999999999997e-06 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601758+0000", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1917e-05 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.601768+0000", 2023-10-09T06:21:05.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2909999999999996e-06 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.607700+0000", 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005931955 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.607710+0000", 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0655e-05 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.737872+0000", 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1301620589999999 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.737961+0000", 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8196999999999996e-05 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.012 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231214 8.f 8:fb3281cb:::benchmark_data_smithi107_251897_object114300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.34778487, 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1060980840000001, 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231214 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648400+0000", 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648398+0000", 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648413+0000", 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5296999999999999e-05 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648453+0000", 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0074000000000003e-05 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648462+0000", 2023-10-09T06:21:05.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3799999999999994e-06 2023-10-09T06:21:05.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648506+0000", 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.456e-05 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:29.648514+0000", 2023-10-09T06:21:05.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3270000000000001e-06 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738194+0000", 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.089679364 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738327+0000", 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013300700000000001 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.738457+0000", 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013044099999999999 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.751429+0000", 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012972099000000001 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:05.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754396+0000", 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002966602 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754430+0000", 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4335000000000002e-05 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754498+0000", 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8104999999999999e-05 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.020 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231225 8.f 8:ff40690e:::benchmark_data_smithi107_251897_object114311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 274.31709965499999, 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.075537689000000005, 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:21:05.021 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231225 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679085+0000", 2023-10-09T06:21:05.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679084+0000", 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679091+0000", 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6279999999999999e-06 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679104+0000", 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2893e-05 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679111+0000", 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4660000000000002e-06 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679156+0000", 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5113e-05 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.679164+0000", 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8690000000000004e-06 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741160+0000", 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.061996277000000002 2023-10-09T06:21:05.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741294+0000", 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000133638 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.741424+0000", 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000130307 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.751436+0000", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010011483999999999 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754532+0000", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030958510000000002 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754564+0000", 2023-10-09T06:21:05.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1983999999999997e-05 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754623+0000", 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9682000000000002e-05 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:231232 8.f 8:f343d541:::benchmark_data_smithi107_251897_object114318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:21:05.028 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 274.29992589900002, 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.058893938999999999, 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 231232 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696259+0000", 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696258+0000", 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:05.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696267+0000", 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9530000000000005e-06 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696280+0000", 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3593e-05 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.696289+0000", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9630000000000004e-06 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744056+0000", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.047766955 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744155+0000", 2023-10-09T06:21:05.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8794000000000003e-05 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.744260+0000", 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000104666 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.754645+0000", 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010385079 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755033+0000", 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000388296 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755075+0000", 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2126000000000003e-05 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:30.755153+0000", 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8034000000000005e-05 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.34644.0:233225 8.f 8:f009cc95:::benchmark_data_smithi107_251897_object116311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 272.03519888300002, 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24422334800000001, 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.035 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.34644", 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 233225 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960986+0000", 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960985+0000", 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.960991+0000", 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0589999999999999e-06 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961002+0000", 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0822000000000001e-05 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961008+0000", 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7350000000000001e-06 2023-10-09T06:21:05.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961052+0000", 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4536999999999997e-05 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961155+0000", 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000102366 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.961237+0000", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2424999999999998e-05 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:32.963539+0000", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023016949999999999 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205068+0000", 2023-10-09T06:21:05.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24152984399999999 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205136+0000", 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7566999999999999e-05 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:16:33.205210+0000", 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3555999999999998e-05 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.041 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11016 9.6 e156/152 9:6568d624:::benchmark_data_smithi107_300016_object11015:head v 156'667, mlcod=156'665)", 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 175.63719532100001, 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.155686719, 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358990+0000", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.358989+0000", 2023-10-09T06:21:05.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359030+0000", 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1652e-05 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359047+0000", 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6206e-05 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359056+0000", 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8460000000000003e-06 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359118+0000", 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1297000000000006e-05 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359139+0000", 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1719e-05 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359945+0000", 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00080596499999999996 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514677+0000", 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.154731128 2023-10-09T06:21:05.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11011 9.0 e156/152 9:0e4dd42b:::benchmark_data_smithi107_300016_object11010:head v 156'682, mlcod=156'680)", 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 175.637096485, 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15559872299999999, 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359089+0000", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359088+0000", 2023-10-09T06:21:05.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359109+0000", 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0608999999999999e-05 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359117+0000", 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.858e-06 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359123+0000", 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9139999999999996e-06 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359193+0000", 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0149999999999998e-05 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.359216+0000", 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3261e-05 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514658+0000", 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.155441945 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514687+0000", 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9456999999999999e-05 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.051 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11024 9.6 e156/152 9:66e82b7d:::benchmark_data_smithi107_300016_object11023:head v 156'668, mlcod=156'665)", 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 175.635168674, 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15538681900000001, 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361016+0000", 2023-10-09T06:21:05.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.361085+0000", 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9665999999999993e-05 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514654+0000", 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15356840499999999 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514665+0000", 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0968000000000001e-05 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514713+0000", 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8148e-05 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514735+0000", 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2272e-05 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516372+0000", 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016370790000000001 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516403+0000", 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1127000000000002e-05 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.056 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:11064 9.2 e156/152 9:46ba2048:::benchmark_data_smithi107_300016_object11063:head v 156'714, mlcod=156'712)", 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 175.62220970600001, 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.142028131, 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373975+0000", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.373974+0000", 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.374000+0000", 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5729999999999999e-05 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514671+0000", 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.140670884 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514679+0000", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9270000000000005e-06 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514715+0000", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6514000000000003e-05 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.514732+0000", 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6410999999999999e-05 2023-10-09T06:21:05.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.515964+0000", 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012326259999999999 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:09.516004+0000", 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9150000000000003e-05 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:14197 9.2 e156/152 9:409344a9:::benchmark_data_smithi107_300016_object14196:head v 156'904, mlcod=156'902)", 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 174.52628228699999, 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060887881999999997, 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:21:05.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469903+0000", 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469902+0000", 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469926+0000", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4340000000000001e-05 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.469940+0000", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4282e-05 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529007+0000", 2023-10-09T06:21:05.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.059066891000000003 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529076+0000", 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8422000000000003e-05 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529092+0000", 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6433e-05 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.530758+0000", 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016657379999999999 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.530791+0000", 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2855999999999998e-05 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:14199 9.6 e156/152 9:6698baa0:::benchmark_data_smithi107_300016_object14198:head v 156'865, mlcod=156'862)", 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 174.52578740000001, 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060791981000000002, 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:21:05.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470398+0000", 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470397+0000", 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.470455+0000", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7723999999999998e-05 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529020+0000", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.058564931000000001 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529049+0000", 2023-10-09T06:21:05.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9266999999999999e-05 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529102+0000", 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3115000000000002e-05 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.529126+0000", 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3938e-05 2023-10-09T06:21:05.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.531150+0000", 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020239699999999999 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:10.531190+0000", 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9928000000000001e-05 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:22877 9.0 e156/152 9:0619543b:::benchmark_data_smithi107_300016_object22876:head v 156'1402, mlcod=156'1400)", 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 170.761184039, 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.054981192999999998, 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:21:05.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235001+0000", 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235000+0000", 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235028+0000", 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7268999999999999e-05 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235040+0000", 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2818e-05 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235049+0000", 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6899999999999998e-06 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235098+0000", 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9070999999999998e-05 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.235116+0000", 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8114e-05 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.236050+0000", 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00093338299999999998 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289982+0000", 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.053932718999999997 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:22889 9.6 e156/152 9:62aac67c:::benchmark_data_smithi107_300016_object22888:head v 156'1423, mlcod=156'1421)", 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 170.75577911600001, 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.052153469000000001, 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240406+0000", 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240405+0000", 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.240461+0000", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5998000000000002e-05 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289945+0000", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.049483965999999997 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.289958+0000", 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3225e-05 2023-10-09T06:21:05.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.290008+0000", 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9505000000000001e-05 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.290031+0000", 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3249000000000001e-05 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.292526+0000", 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024948890000000001 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:14.292560+0000", 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3581000000000003e-05 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.081 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:24238 9.2 e156/152 9:4195be16:::benchmark_data_smithi107_300016_object24237:head v 156'1526, mlcod=156'1524)", 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 169.83322405000001, 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.040898361000000001, 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162961+0000", 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162960+0000", 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162988+0000", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8102999999999998e-05 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163003+0000", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4438e-05 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163010+0000", 2023-10-09T06:21:05.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3239999999999997e-06 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163074+0000", 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3561999999999999e-05 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163093+0000", 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9729999999999999e-05 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.203828+0000", 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.040734609999999997 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.203859+0000", 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1599e-05 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:24237 9.6 e156/152 9:69328eaf:::benchmark_data_smithi107_300016_object24236:head v 156'1516, mlcod=156'1513)", 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 169.83322172000001, 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.041692811000000003, 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162963+0000", 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.162958+0000", 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999948 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163188+0000", 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000229597 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163199+0000", 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0966e-05 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.163206+0000", 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8000000000000001e-06 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.201974+0000", 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.038768449000000003 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.201997+0000", 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2906999999999999e-05 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.204634+0000", 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026364259999999999 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:15.204656+0000", 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2719000000000001e-05 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:82766 9.2 e169/152 9:405caae3:::benchmark_data_smithi107_300016_object82765:head v 169'5029, mlcod=169'5027)", 2023-10-09T06:21:05.091 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:14.018454+0000", 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 110.977731596, 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060746935000000002, 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018454+0000", 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018454+0000", 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018453+0000", 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018475+0000", 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2198e-05 2023-10-09T06:21:05.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018486+0000", 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0701e-05 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018492+0000", 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6889999999999998e-06 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018549+0000", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6848999999999997e-05 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.018569+0000", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9371000000000001e-05 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079147+0000", 2023-10-09T06:21:05.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.060578317 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079200+0000", 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3545999999999999e-05 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:82772 9.0 e169/152 9:002832b8:::benchmark_data_smithi107_300016_object82771:head v 169'5057, mlcod=169'5055)", 2023-10-09T06:21:05.096 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:14.022331+0000", 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 110.97385417700001, 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.062822093999999995, 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.022331+0000", 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.022331+0000", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.022330+0000", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.022354+0000", 2023-10-09T06:21:05.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3895000000000001e-05 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079540+0000", 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.057185937999999999 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079549+0000", 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0250000000000008e-06 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079685+0000", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000136262 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.079709+0000", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3747000000000002e-05 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.085108+0000", 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005399427 2023-10-09T06:21:05.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:14.085153+0000", 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4796999999999998e-05 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:124517 9.6 e194/152 9:60dfeabd:::benchmark_data_smithi107_300016_object26945:head v 194'7865, mlcod=194'7863)", 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:15.186591+0000", 2023-10-09T06:21:05.101 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 49.809594543999999, 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.046580112999999999, 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186591+0000", 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186591+0000", 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186590+0000", 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186601+0000", 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1561e-05 2023-10-09T06:21:05.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186623+0000", 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1849e-05 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.186632+0000", 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9779999999999994e-06 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.187082+0000", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00045004600000000001 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.187112+0000", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9955e-05 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.233128+0000", 2023-10-09T06:21:05.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.046015780999999999 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.233171+0000", 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.303e-05 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:124523 9.6 e194/152 9:6460a203:::benchmark_data_smithi107_300016_object26951:head v 194'7866, mlcod=194'7863)", 2023-10-09T06:21:05.106 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:15.193207+0000", 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 49.802977918000003, 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.039990738999999997, 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193207+0000", 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:05.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193207+0000", 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193206+0000", 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193216+0000", 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0176e-05 2023-10-09T06:21:05.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193256+0000", 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9755999999999997e-05 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.193266+0000", 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0186999999999999e-05 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.214542+0000", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021275813000000001 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.214568+0000", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6108000000000001e-05 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.233176+0000", 2023-10-09T06:21:05.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.018607882999999999 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:15.233198+0000", 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2053999999999999e-05 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:05.111 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:21:05.112 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.465 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:05.509 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:05.534 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.550 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:05.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:05.591 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:05.629 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:05.646 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:05.651 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:05.667 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:05.678 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.687 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:05.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.689 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.689 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.689 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.689 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:05.717 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3390867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:05.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3390867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:05.724 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:05.829 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:05.880 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3390867243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:05.882 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:05.984 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:05.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:06.096 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:06.168 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:06.630 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:06 smithi107 ceph-mon[212750]: pgmap v664: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:21:06.632 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:06.674 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:21:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:06 smithi107 ceph-mon[220921]: pgmap v664: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:21:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:06 smithi150 ceph-mon[194178]: pgmap v664: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T06:21:07.087 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:07.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:07.191 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:07.241 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:07.683 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:07.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:07.800 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:07.865 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:07.946 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:07.947 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:07.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:08.049 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:08.117 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:08.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:08.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:08.313 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:08.373 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:08.425 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:08.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:08.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:08.529 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:08.554 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:08.652 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:08.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:08.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:08.679 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:21:08.759 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:08.812 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:08 smithi107 ceph-mon[212750]: pgmap v665: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:21:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:08 smithi107 ceph-mon[220921]: pgmap v665: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:21:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:08.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:08 smithi150 ceph-mon[194178]: pgmap v665: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-09T06:21:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:09.592 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:09.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:09.697 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:09.747 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:09.811 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:09.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[220921]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:21:09.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:09 smithi107 ceph-mon[212750]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.929 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:09.972 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/596217795' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: osdmap e205: 8 total, 8 up, 8 in 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:09 smithi150 ceph-mon[194178]: failed to encode map e205 with expected crc 2023-10-09T06:21:09.978 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:10.067 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:10.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:10.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:10.234 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:10.327 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:10.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:10.436 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:10.483 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:10.585 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:10.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:10.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:10.708 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 1 pg_num to 11 2023-10-09T06:21:10.798 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:10 smithi150 ceph-mon[194178]: pgmap v667: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:21:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:21:11.054 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:11.055 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:10 smithi107 ceph-mon[220921]: pgmap v667: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:21:11.056 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:21:11.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:11.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:11.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:10 smithi107 ceph-mon[212750]: pgmap v667: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2023-10-09T06:21:11.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T06:21:11.158 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:11.209 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:11.439 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:11.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:11.544 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:11.548 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:11.636 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:11.800 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:21:11.896 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:11.897 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:11.930 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:11.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:12.037 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1818257763' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: osdmap e206: 8 total, 8 up, 8 in 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: failed to encode map e206 with expected crc 2023-10-09T06:21:12.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:12.088 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:12.188 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:12.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:12.300 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:12.326 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:12.367 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:12.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:12.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:12] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T06:21:12.473 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:12.554 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:12.834 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: pgmap v669: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[220921]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: pgmap v669: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:21:12.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T06:21:12.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:21:12.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:12 smithi107 ceph-mon[212750]: failed to encode map e207 with expected crc 2023-10-09T06:21:12.855 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:21:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:13.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:13.123 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:13.149 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: pgmap v669: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: failed to encode map e207 with expected crc 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: osdmap e207: 8 total, 8 up, 8 in 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: failed to encode map e207 with expected crc 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: failed to encode map e207 with expected crc 2023-10-09T06:21:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:12 smithi150 ceph-mon[194178]: failed to encode map e207 with expected crc 2023-10-09T06:21:13.472 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:13.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:13.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:13.657 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:13.753 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:21:13.753 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: failed to encode map e208 with expected crc 2023-10-09T06:21:13.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:13.772 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:13.787 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:13.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:13.834 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:13.907 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:14.006 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:21:14.006 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.007 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:14.015 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: osdmap e208: 8 total, 8 up, 8 in 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: failed to encode map e208 with expected crc 2023-10-09T06:21:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T06:21:14.480 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:14.492 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:14.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:14.531 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:14.577 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:14.597 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:14.609 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:14.639 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:14.684 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:14.720 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:14.724 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:14.769 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:14.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:15.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: pgmap v672: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2523383816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: pgmap v672: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:21:15.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2523383816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: pgmap v672: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: osdmap e209: 8 total, 8 up, 8 in 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: failed to encode map e209 with expected crc 2023-10-09T06:21:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2523383816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[212750]: pgmap v675: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: failed to encode map e210 with expected crc 2023-10-09T06:21:15.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:15 smithi107 ceph-mon[220921]: pgmap v675: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T06:21:15.909 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:15.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:16.019 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:16.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: osdmap e210: 8 total, 8 up, 8 in 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: failed to encode map e210 with expected crc 2023-10-09T06:21:16.070 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:15 smithi150 ceph-mon[194178]: pgmap v675: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T06:21:16.072 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:16.144 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:16.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:16.253 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:16.308 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:16.396 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:16.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:16.491 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:16.491 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:21:16.491 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:21:16.492 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.170142 2023-10-09T06:21:16.492 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.17014181643830906 2023-10-09T06:21:16.506 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:16.623 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:16.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:16.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: failed to encode map e211 with expected crc 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:16.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:16.943 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:16.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:17.060 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: osdmap e211: 8 total, 8 up, 8 in 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: failed to encode map e211 with expected crc 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:17.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:17.114 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:17.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:17.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:17.243 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:21:17.243 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:21:17.243 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:21:17.243 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:21:17.243 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:21:17.316 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:17.365 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:17.441 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:17.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:17.548 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:17.626 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:17.738 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:17.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:17.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:17.897 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:17.995 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:18.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:18.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:18.110 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:18.146 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[212750]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:21:18.146 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[212750]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.146 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[212750]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.146 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[212750]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.146 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[212750]: pgmap v678: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:21:18.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[220921]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:21:18.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[220921]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[220921]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[220921]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:17 smithi107 ceph-mon[220921]: pgmap v678: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:21:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:17 smithi150 ceph-mon[194178]: osdmap e212: 8 total, 8 up, 8 in 2023-10-09T06:21:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:17 smithi150 ceph-mon[194178]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:17 smithi150 ceph-mon[194178]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:17 smithi150 ceph-mon[194178]: failed to encode map e212 with expected crc 2023-10-09T06:21:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:17 smithi150 ceph-mon[194178]: pgmap v678: 99 pgs: 6 remapped+peering, 93 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-09T06:21:18.158 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:18.182 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:21:18.194 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:18.231 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:18.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:18.338 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:18.403 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:18.527 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:18.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:18.635 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:18.700 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:18.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T06:21:18.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T06:21:18.839 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:21:18.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T06:21:18.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: failed to encode map e213 with expected crc 2023-10-09T06:21:18.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2801130428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:18.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:18.874 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:21:18.875 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:18.917 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:18.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:19.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:19.069 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:19.143 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:19.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:19.149 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2801130428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:19.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:18 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:19.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: osdmap e213: 8 total, 8 up, 8 in 2023-10-09T06:21:19.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: failed to encode map e213 with expected crc 2023-10-09T06:21:19.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2801130428' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:19.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:18 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:19.248 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:19.306 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:19.777 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:19.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:19.885 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:19.947 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[212750]: pgmap v681: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 92 KiB/s, 0 objects/s recovering 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:19 smithi107 ceph-mon[220921]: pgmap v681: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 92 KiB/s, 0 objects/s recovering 2023-10-09T06:21:20.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: osdmap e214: 8 total, 8 up, 8 in 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: failed to encode map e214 with expected crc 2023-10-09T06:21:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:19 smithi150 ceph-mon[194178]: pgmap v681: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 92 KiB/s, 0 objects/s recovering 2023-10-09T06:21:20.254 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:20.258 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:20.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:20.379 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:20.413 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:20.451 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:20.485 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:20.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:20.591 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:20.633 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:20.699 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:20.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:20.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:20.809 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:20.855 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:20.934 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:20.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:20.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:21.045 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:21.089 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:21.170 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:21.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:21.278 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:21.335 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:21.421 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:21.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:21.544 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:21.606 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:21.672 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:21.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:21.784 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:21.830 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:21.938 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:21.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:22.049 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:22.118 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:22.119 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.2 primary-affinity to 0.170142 (8111502) 2023-10-09T06:21:22.299 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:21:22.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[212750]: pgmap v682: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 70 KiB/s, 0 objects/s recovering 2023-10-09T06:21:22.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2807471226' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[212750]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:22.363 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:22] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T06:21:22.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[220921]: pgmap v682: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 70 KiB/s, 0 objects/s recovering 2023-10-09T06:21:22.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2807471226' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.369 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:22 smithi107 ceph-mon[220921]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:22 smithi150 ceph-mon[194178]: pgmap v682: 99 pgs: 2 peering, 1 remapped+peering, 96 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 70 KiB/s, 0 objects/s recovering 2023-10-09T06:21:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2807471226' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]: dispatch 2023-10-09T06:21:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:22 smithi150 ceph-mon[194178]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:21:22.981 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:22.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:23.092 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:23.140 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:23.223 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:23.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:23.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:23.328 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:23.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]': finished 2023-10-09T06:21:23.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[212750]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]': finished 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:21:23.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:23 smithi107 ceph-mon[220921]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.383 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.17014181643830906}]': finished 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: osdmap e215: 8 total, 8 up, 8 in 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.390 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:23 smithi150 ceph-mon[194178]: failed to encode map e215 with expected crc 2023-10-09T06:21:23.474 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:23.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:23.585 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.606 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:23.650 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:23.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:23.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:23.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64723 9.2 9:4bf0c3c2:::benchmark_data_smithi107_300016_object64722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:21:23.750 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.668121144, 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.362261308, 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64723 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078541+0000", 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078567+0000", 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6197e-05 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078577+0000", 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078583+0000", 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4130000000000003e-06 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079048+0000", 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046495299999999999 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079196+0000", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014766599999999999 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079363+0000", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016698200000000001 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.084453+0000", 2023-10-09T06:21:23.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050901360000000003 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440670+0000", 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.356217269 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440709+0000", 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9119999999999998e-05 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440762+0000", 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2203000000000001e-05 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440804+0000", 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1916999999999999e-05 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64724 9.7 e163/160 9:e68efaaf:::benchmark_data_smithi107_300016_object64723:head v 163'4118, mlcod=0'0)", 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.66802236000001, 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35938271500000002, 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:21:23.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078682+0000", 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0822999999999999e-05 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078689+0000", 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078694+0000", 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0590000000000002e-06 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081003+0000", 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023091069999999999 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081031+0000", 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8022e-05 2023-10-09T06:21:23.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437591+0000", 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35656025299999999 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438024+0000", 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043272699999999999 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.762 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64725 9.f e163/152 9:f1989478:::benchmark_data_smithi107_300016_object64724:head v 163'3999, mlcod=163'3997)", 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.667956683, 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35934786400000002, 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967296 2023-10-09T06:21:23.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078736+0000", 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9133e-05 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078741+0000", 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5060000000000003e-06 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078745+0000", 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1790000000000001e-06 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083200+0000", 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044543509999999996 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083218+0000", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8397999999999999e-05 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438028+0000", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35480992500000003 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438055+0000", 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6554e-05 2023-10-09T06:21:23.767 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64727 9.8 9:1ddec4f7:::benchmark_data_smithi107_300016_object64726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.665073896, 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35651537999999999, 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.768 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64727 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.769 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081588+0000", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081613+0000", 2023-10-09T06:21:23.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5406e-05 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081626+0000", 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2520999999999999e-05 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081634+0000", 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:21:23.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081691+0000", 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6190999999999998e-05 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081807+0000", 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011626299999999999 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081946+0000", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013932899999999999 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087000+0000", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050537489999999997 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437968+0000", 2023-10-09T06:21:23.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35096775800000002 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438021+0000", 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3038999999999998e-05 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438065+0000", 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4045999999999998e-05 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438105+0000", 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0014e-05 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64729 9.3 9:cba02ed8:::benchmark_data_smithi107_300016_object64728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.66501170500001, 2023-10-09T06:21:23.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37691757999999997, 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64729 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:21:23.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081651+0000", 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081673+0000", 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2178999999999999e-05 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081681+0000", 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9219999999999997e-06 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081688+0000", 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2469999999999997e-06 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085025+0000", 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033377429999999998 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085147+0000", 2023-10-09T06:21:23.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000122089 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085280+0000", 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013249099999999999 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444158+0000", 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35887852100000001 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444204+0000", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5355999999999997e-05 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456528+0000", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01232391 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456591+0000", 2023-10-09T06:21:23.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.458569+0000", 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019784849999999999 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64730 9.f e163/152 9:f3222a22:::benchmark_data_smithi107_300016_object64729:head v 163'4000, mlcod=163'3997)", 2023-10-09T06:21:23.782 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.66493534099999, 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35634943299999999, 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081727+0000", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081887+0000", 2023-10-09T06:21:23.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016013400000000001 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081898+0000", 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0875e-05 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081916+0000", 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7195000000000001e-05 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085279+0000", 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033639469999999999 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085297+0000", 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7669e-05 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438058+0000", 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35276105099999999 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438077+0000", 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9317000000000002e-05 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.787 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64733 9.2 9:4941674d:::benchmark_data_smithi107_300016_object64732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.65992736999999, 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.360635498, 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64733 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.788 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086735+0000", 2023-10-09T06:21:23.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086763+0000", 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7846000000000001e-05 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086772+0000", 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9039999999999998e-06 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086778+0000", 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2759999999999997e-06 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086950+0000", 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172048 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087082+0000", 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013214800000000001 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087233+0000", 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000151213 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437967+0000", 2023-10-09T06:21:23.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35073413799999997 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440845+0000", 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028772379999999998 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447284+0000", 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064393209999999996 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447333+0000", 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9299999999999999e-05 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447372+0000", 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8277e-05 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.794 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64729 9.3 e163/160 ondisk, result = 0)", 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.65783961899999, 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.355337442, 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.795 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088823+0000", 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.796 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088829+0000", 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5189999999999998e-06 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088840+0000", 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088848+0000", 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.391e-06 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444142+0000", 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35529475900000002 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444151+0000", 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444161+0000", 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0239e-05 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64727 9.8 e163/152 ondisk, result = 0)", 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.65671827599999, 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.348172755, 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.800 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089944+0000", 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089951+0000", 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3820000000000001e-06 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437572+0000", 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34762106599999998 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.802 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437921+0000", 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034914000000000001 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438007+0000", 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6438000000000001e-05 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438015+0000", 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4359999999999996e-06 2023-10-09T06:21:23.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438118+0000", 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103149 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64737 9.2 9:42e1c82a:::benchmark_data_smithi107_300016_object64736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:21:23.804 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.65630813000001, 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.359336867, 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64737 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.805 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090354+0000", 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090375+0000", 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1294e-05 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437567+0000", 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34719175099999999 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.807 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437894+0000", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032692099999999998 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437999+0000", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010511299999999999 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438153+0000", 2023-10-09T06:21:23.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015426700000000001 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438341+0000", 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018829099999999999 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447252+0000", 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0089106849999999998 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447436+0000", 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018348799999999999 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449593+0000", 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002157413 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449650+0000", 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7450000000000001e-05 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449692+0000", 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1686e-05 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.811 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64736 9.a e163/160 9:553011b2:::benchmark_data_smithi107_300016_object64735:head v 163'4090, mlcod=0'0)", 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 155.655943084, 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35900552499999999, 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.812 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090719+0000", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090765+0000", 2023-10-09T06:21:23.813 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5793e-05 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437575+0000", 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34680993900000001 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437924+0000", 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034926999999999999 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.814 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437970+0000", 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5179999999999998e-05 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438005+0000", 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5361000000000002e-05 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.815 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449697+0000", 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011691906 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449726+0000", 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8932999999999999e-05 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.816 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:95062 9.1 9:8b4de063:::benchmark_data_smithi107_300016_object95061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:31.531595+0000", 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 112.215068899, 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27977949200000002, 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 95062 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.817 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.531595+0000", 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.531595+0000", 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.818 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.531594+0000", 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.531616+0000", 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2617999999999999e-05 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.778760+0000", 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.247143893 2023-10-09T06:21:23.819 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.778775+0000", 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4789e-05 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.783521+0000", 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047455309999999999 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.820 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.783664+0000", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000143101 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.785676+0000", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020123290000000002 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.799652+0000", 2023-10-09T06:21:23.821 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013976354 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.803081+0000", 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034283460000000001 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.810658+0000", 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075776799999999998 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.822 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.810710+0000", 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1544e-05 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:31.811374+0000", 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00066411400000000005 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.823 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:112415 9.3 9:c7576938:::benchmark_data_smithi107_300016_object14843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 86.150231274000006, 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29174138700000002, 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.824 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 112415 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.825 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596434+0000", 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7590000000000002e-06 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.826 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596440+0000", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3129999999999998e-06 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596445+0000", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0579999999999997e-06 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601164+0000", 2023-10-09T06:21:23.827 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047186659999999998 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601299+0000", 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013497699999999999 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601421+0000", 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012268199999999999 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.828 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.607663+0000", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0062417699999999998 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.608492+0000", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082930000000000005 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887363+0000", 2023-10-09T06:21:23.829 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27887095200000001 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887422+0000", 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.889e-05 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.830 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.888174+0000", 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075141200000000002 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:127242 9.7 e195/188 9:e1ce63c8:::benchmark_data_smithi107_300016_object29670:head v 195'8070, mlcod=195'8068)", 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:18.768959+0000", 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 64.977704246000002, 2023-10-09T06:21:23.831 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28140934299999998, 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768959+0000", 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768959+0000", 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.832 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768958+0000", 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768966+0000", 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9799999999999998e-06 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.833 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768976+0000", 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6800000000000005e-06 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:18.768983+0000", 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0690000000000004e-06 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.834 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:19.027003+0000", 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25802003499999998 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:19.027031+0000", 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7606999999999998e-05 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:19.050337+0000", 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.023306535 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.835 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:19.050368+0000", 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1185e-05 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141939 9.7 e201/188 9:ee97ad70:::benchmark_data_smithi107_300016_object44367:head v 201'9015, mlcod=201'9011)", 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.8157724, 2023-10-09T06:21:23.836 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29274838399999997, 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:21:23.837 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930890+0000", 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930897+0000", 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.838 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930907+0000", 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0074e-05 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930913+0000", 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3160000000000002e-06 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.839 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935291+0000", 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043775979999999999 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935312+0000", 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1523e-05 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223606+0000", 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28829408499999998 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.840 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223639+0000", 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3176999999999998e-05 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141946 9.1 9:85283973:::benchmark_data_smithi107_300016_object44374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.939207+0000", 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.807456617, 2023-10-09T06:21:23.841 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28014460000000002, 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141946 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.842 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939207+0000", 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939207+0000", 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939205+0000", 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.843 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939212+0000", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7159999999999997e-06 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939223+0000", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0574000000000001e-05 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939229+0000", 2023-10-09T06:21:23.844 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9669999999999998e-06 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939272+0000", 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3377999999999997e-05 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939369+0000", 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7391000000000003e-05 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939477+0000", 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000107976 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219019+0000", 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.279541121 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219080+0000", 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1631000000000007e-05 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219269+0000", 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000189006 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219312+0000", 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2620000000000002e-05 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219351+0000", 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9555999999999999e-05 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.848 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141944 9.f e201/152 9:f293b650:::benchmark_data_smithi107_300016_object44372:head v 201'8823, mlcod=201'8821)", 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.807279297000001, 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30728261400000001, 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939383+0000", 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939389+0000", 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939397+0000", 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2600000000000005e-06 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939404+0000", 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8419999999999999e-06 2023-10-09T06:21:23.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939455+0000", 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1220000000000001e-05 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939471+0000", 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5988e-05 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.852 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246587+0000", 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.307116161 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246667+0000", 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9617999999999996e-05 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141947 9.2 9:429ff582:::benchmark_data_smithi107_300016_object44375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.806257604999999, 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29032069100000002, 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141947 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.854 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940405+0000", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940410+0000", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8690000000000001e-06 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940422+0000", 2023-10-09T06:21:23.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1296e-05 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940429+0000", 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1130000000000003e-06 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940473+0000", 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4391999999999998e-05 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219069+0000", 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27859536699999998 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219175+0000", 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106272 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.226985+0000", 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0078100460000000002 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230576+0000", 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035914990000000002 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230642+0000", 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5840000000000007e-05 2023-10-09T06:21:23.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230689+0000", 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6653999999999998e-05 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230726+0000", 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7589000000000002e-05 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.860 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141951 9.1 9:831efef4:::benchmark_data_smithi107_300016_object44379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.802472657999999, 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.286240618, 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:23.861 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141951 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.862 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944189+0000", 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944195+0000", 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8109999999999999e-06 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219289+0000", 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.275094218 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219299+0000", 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219377+0000", 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7982e-05 2023-10-09T06:21:23.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219481+0000", 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103836 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219580+0000", 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9399000000000002e-05 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227278+0000", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0076973400000000004 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227425+0000", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014719900000000001 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230360+0000", 2023-10-09T06:21:23.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029345669999999999 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230398+0000", 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8917e-05 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230431+0000", 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2916000000000001e-05 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.867 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141948 9.f e201/152 9:f4f21468:::benchmark_data_smithi107_300016_object44376:head v 201'8824, mlcod=201'8821)", 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 42.802169392000003, 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30220260900000001, 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944493+0000", 2023-10-09T06:21:23.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944500+0000", 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218928+0000", 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27442793999999998 2023-10-09T06:21:23.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218942+0000", 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.412e-05 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218983+0000", 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0599999999999998e-05 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219005+0000", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2347000000000001e-05 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246673+0000", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027668321999999999 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246697+0000", 2023-10-09T06:21:23.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3408999999999999e-05 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:21:23.873 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:21:23.873 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:23.878 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:23.918 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:23.936 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:23.957 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:23.997 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:24.039 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:24.042 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:24.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:24.073 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:24.109 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:24.155 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.163 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:24.174 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.174 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:24.222 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:24.320 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: pgmap v684: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.321 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:21:24.322 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/119807140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:24.322 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:24.322 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: pgmap v684: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T06:21:24.322 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:21:24.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/119807140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:24.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: pgmap v684: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 61 KiB/s, 0 objects/s recovering 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: failed to encode map e216 with expected crc 2023-10-09T06:21:24.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: osdmap e216: 8 total, 8 up, 8 in 2023-10-09T06:21:24.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/119807140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:24.425 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:24.483 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:25.962 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:21:26.086 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:26.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:26.192 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:26.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:26 smithi107 ceph-mon[212750]: pgmap v686: 99 pgs: 99 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-09T06:21:26.263 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:26 smithi107 ceph-mon[220921]: pgmap v686: 99 pgs: 99 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-09T06:21:26.274 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:26.370 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:26.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:26.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:26.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:26 smithi150 ceph-mon[194178]: pgmap v686: 99 pgs: 99 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-09T06:21:26.478 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:26.527 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:26.656 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:26.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:26.778 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:26.834 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:26.945 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:26.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:27.068 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:27.126 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:27.176 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:27.177 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:27.177 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:21:27.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:27.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:27.281 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:27.326 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:27.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1736870459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:27.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:27.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1736870459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:27.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1736870459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:21:28.083 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:28.091 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:28.092 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:21:28.092 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:21:28.092 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-10-09T06:21:28.092 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T06:21:28.107 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:28.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:28.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:28.211 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:28.257 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:28.349 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:28.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:28.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: pgmap v687: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:21:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[220921]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: pgmap v687: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:21:28.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:28 smithi107 ceph-mon[212750]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: pgmap v687: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: osdmap e217: 8 total, 8 up, 8 in 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:28 smithi150 ceph-mon[194178]: failed to encode map e217 with expected crc 2023-10-09T06:21:28.456 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:28.537 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:28.769 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:28.774 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:28.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:28.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:28.817 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:28.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:28.917 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:28.958 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:28.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:29.062 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:29.090 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:29.328 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[212750]: 1.0 scrub ok 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[212750]: 1.4 scrub starts 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[212750]: 1.4 scrub ok 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[220921]: 1.0 scrub starts 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[220921]: 1.0 scrub ok 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[220921]: 1.4 scrub starts 2023-10-09T06:21:29.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:29 smithi107 ceph-mon[220921]: 1.4 scrub ok 2023-10-09T06:21:29.356 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:29.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:29.460 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:29.471 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:29 smithi150 ceph-mon[194178]: 1.0 scrub starts 2023-10-09T06:21:29.472 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:29 smithi150 ceph-mon[194178]: 1.0 scrub ok 2023-10-09T06:21:29.472 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:29 smithi150 ceph-mon[194178]: 1.4 scrub starts 2023-10-09T06:21:29.472 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:29 smithi150 ceph-mon[194178]: 1.4 scrub ok 2023-10-09T06:21:29.553 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:29.638 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:29.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:29.749 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:29.815 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:29.888 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:29.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:29.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:30.056 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:30.144 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:30.144 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:30.181 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:21:30.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:30 smithi107 ceph-mon[220921]: pgmap v689: 99 pgs: 99 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:21:30.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:30 smithi107 ceph-mon[212750]: pgmap v689: 99 pgs: 99 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:21:30.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:30 smithi150 ceph-mon[194178]: pgmap v689: 99 pgs: 99 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T06:21:30.688 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:30.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:30.804 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:30.850 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.2 scrub starts 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.2 scrub ok 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.3 scrub starts 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.3 scrub ok 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.1 scrub starts 2023-10-09T06:21:31.277 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[220921]: 1.1 scrub ok 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.2 scrub ok 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.3 scrub starts 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.3 scrub ok 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T06:21:31.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:31 smithi107 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T06:21:31.291 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:31.292 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:21:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:31.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:31.393 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:31.402 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.2 scrub starts 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.2 scrub ok 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.3 scrub starts 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.3 scrub ok 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.1 scrub starts 2023-10-09T06:21:31.473 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:31 smithi150 ceph-mon[194178]: 1.1 scrub ok 2023-10-09T06:21:31.477 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:31.557 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:31.577 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:31.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:31.603 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:31.633 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:31.675 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:31.690 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:31.707 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:31.741 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:31.752 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:31.794 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:31.828 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:31.837 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:31.841 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.841 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.841 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.842 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:31.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:31.945 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:31.950 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:32.014 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:32.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:32] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-10-09T06:21:32.263 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: 1.6 scrub starts 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: 1.6 scrub ok 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: 1.8 scrub starts 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: 1.5 scrub starts 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: pgmap v690: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: 1.8 scrub ok 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3525280524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:32.265 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:32.266 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:32.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:32.367 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:32.434 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:32.517 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T06:21:32.517 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T06:21:32.517 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T06:21:32.517 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: pgmap v690: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3525280524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: 1.6 scrub starts 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: 1.6 scrub ok 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: 1.8 scrub starts 2023-10-09T06:21:32.518 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: 1.5 scrub starts 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: pgmap v690: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: 1.8 scrub ok 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3525280524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:32.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:32.578 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:32.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:32.687 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:32.740 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:32.834 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:32.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:32.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:32.939 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:32.970 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:33.248 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T06:21:33.559 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4293784920' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[220921]: 1.7 scrub starts 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[220921]: 1.7 scrub ok 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4293784920' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.560 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:33 smithi150 ceph-mon[194178]: 1.7 scrub starts 2023-10-09T06:21:33.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:33 smithi150 ceph-mon[194178]: 1.7 scrub ok 2023-10-09T06:21:33.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4293784920' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:21:33.953 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:33.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:33.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:34.057 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:34.112 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:21:34.112 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:21:34.112 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:21:34.112 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:21:34.112 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:21:34.141 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:34.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:34 smithi150 ceph-mon[194178]: pgmap v691: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:21:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:34 smithi150 ceph-mon[194178]: failed to encode map e218 with expected crc 2023-10-09T06:21:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:34 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:34.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:34 smithi150 ceph-mon[194178]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:21:34.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[212750]: pgmap v691: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:21:34.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[212750]: failed to encode map e218 with expected crc 2023-10-09T06:21:34.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:34.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[212750]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:21:34.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[220921]: pgmap v691: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-09T06:21:34.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[220921]: failed to encode map e218 with expected crc 2023-10-09T06:21:34.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:21:34.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:34 smithi107 ceph-mon[220921]: osdmap e218: 8 total, 8 up, 8 in 2023-10-09T06:21:34.750 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:34.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:34.855 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:34.928 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:35.285 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:21:35.302 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:21:35.309 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:35.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:35.419 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:35.438 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:35.497 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:35.524 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: 1.9 scrub starts 2023-10-09T06:21:35.524 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: 1.9 scrub ok 2023-10-09T06:21:35.524 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:21:35.524 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1207608894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: 1.9 scrub starts 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: 1.9 scrub ok 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:21:35.525 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1207608894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.526 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.614 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:35.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: 1.9 scrub starts 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: 1.9 scrub ok 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: osdmap e219: 8 total, 8 up, 8 in 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: failed to encode map e219 with expected crc 2023-10-09T06:21:35.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1207608894' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:35.725 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:35.772 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:35.816 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:35.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:35.843 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:35.920 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:35.946 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:36.216 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:36.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:36.321 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:36.382 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: pgmap v694: 99 pgs: 99 active+clean; 959 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:36 smithi150 ceph-mon[194178]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.457 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: pgmap v694: 99 pgs: 99 active+clean; 959 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[212750]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: pgmap v694: 99 pgs: 99 active+clean; 959 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: osdmap e220: 8 total, 8 up, 8 in 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:36 smithi107 ceph-mon[220921]: failed to encode map e220 with expected crc 2023-10-09T06:21:36.957 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:36.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:37.065 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:37.152 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:37.444 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:37.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:37.567 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:37.636 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:37.785 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:37.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:37.901 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:37.967 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:48686 8.2 8:4cde1e59:::benchmark_data_smithi107_251897_object48685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2023-10-09T06:21:38.090 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 537.04836679499999, 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20579032899999999, 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 48686 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.091 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036654+0000", 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036653+0000", 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036694+0000", 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1156999999999999e-05 2023-10-09T06:21:38.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036709+0000", 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5041e-05 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036715+0000", 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7830000000000004e-06 2023-10-09T06:21:38.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036752+0000", 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7527999999999997e-05 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036856+0000", 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103558 2023-10-09T06:21:38.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.036956+0000", 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100714 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.038727+0000", 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017710110000000001 2023-10-09T06:21:38.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.038924+0000", 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000196911 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242375+0000", 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20345057799999999 2023-10-09T06:21:38.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242410+0000", 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4947999999999998e-05 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:41.242444+0000", 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4567999999999998e-05 2023-10-09T06:21:38.098 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:62892 8.2 8:4af4e149:::benchmark_data_smithi107_251897_object62891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 526.61334872999998, 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.309153503, 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.099 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 62892 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:21:38.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471672+0000", 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471671+0000", 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471735+0000", 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4172999999999997e-05 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471752+0000", 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7258e-05 2023-10-09T06:21:38.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471758+0000", 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4949999999999996e-06 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471816+0000", 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7766000000000002e-05 2023-10-09T06:21:38.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.471935+0000", 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000118294 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.472060+0000", 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012510700000000001 2023-10-09T06:21:38.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.476873+0000", 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048130719999999998 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.476879+0000", 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5810000000000002e-06 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780714+0000", 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30383470600000001 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780762+0000", 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7963999999999997e-05 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.780826+0000", 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3746999999999998e-05 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:62902 8.2 8:4da830c8:::benchmark_data_smithi107_251897_object62901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e108)", 2023-10-09T06:21:38.107 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 526.60601335399997, 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30722065199999998, 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 62902 2023-10-09T06:21:38.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479008+0000", 2023-10-09T06:21:38.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479005+0000", 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479071+0000", 2023-10-09T06:21:38.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5686999999999996e-05 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479084+0000", 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3267e-05 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479092+0000", 2023-10-09T06:21:38.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3429999999999998e-06 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479138+0000", 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5998999999999998e-05 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479259+0000", 2023-10-09T06:21:38.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012126299999999999 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.479386+0000", 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127069 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.481372+0000", 2023-10-09T06:21:38.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019865870000000002 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.481910+0000", 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000537631 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786138+0000", 2023-10-09T06:21:38.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30422770599999999 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786189+0000", 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1708e-05 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:12:51.786228+0000", 2023-10-09T06:21:38.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8720000000000002e-05 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:75955 8.2 8:4848e1b1:::benchmark_data_smithi107_251897_object75954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 516.38874868100004, 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28918334099999998, 2023-10-09T06:21:38.116 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 75955 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.117 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696272+0000", 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696270+0000", 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696336+0000", 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5167999999999997e-05 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696350+0000", 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4881e-05 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696357+0000", 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3289999999999999e-06 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696404+0000", 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.702e-05 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696519+0000", 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0001149 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.696647+0000", 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012799099999999999 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.699310+0000", 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026631950000000001 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.701937+0000", 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026272370000000001 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985263+0000", 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28332602200000001 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985393+0000", 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012951400000000001 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.985456+0000", 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2872000000000004e-05 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.124 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:75967 8.2 8:411c9d04:::benchmark_data_smithi107_251897_object75966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e109)", 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 516.37987261900003, 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28091582799999998, 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.125 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 75967 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705148+0000", 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705147+0000", 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705184+0000", 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7125000000000001e-05 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705200+0000", 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5871000000000001e-05 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705207+0000", 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0090000000000001e-06 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705258+0000", 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1437999999999998e-05 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705400+0000", 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014183500000000001 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.705531+0000", 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013155600000000001 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.708330+0000", 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027981019999999998 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.708685+0000", 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00035532100000000002 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986008+0000", 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27732359699999998 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986034+0000", 2023-10-09T06:21:38.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5587000000000001e-05 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:01.986064+0000", 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0083e-05 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:167069 8.2 8:4fc8af52:::benchmark_data_smithi107_251897_object50155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.98393260900002, 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25242683300000002, 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.135 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 167069 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101088+0000", 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101087+0000", 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101093+0000", 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3070000000000002e-06 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101104+0000", 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1467999999999999e-05 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101110+0000", 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3899999999999998e-06 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101158+0000", 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8038999999999999e-05 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101306+0000", 2023-10-09T06:21:38.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014781899999999999 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.101453+0000", 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014681399999999999 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.105161+0000", 2023-10-09T06:21:38.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037081760000000001 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.105887+0000", 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072546799999999995 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353412+0000", 2023-10-09T06:21:38.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247525683 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353471+0000", 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8606000000000001e-05 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353515+0000", 2023-10-09T06:21:38.142 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4168999999999997e-05 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:167076 8.2 8:404a6a5b:::benchmark_data_smithi107_251897_object50162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.96842235499997, 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23699437900000001, 2023-10-09T06:21:38.143 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 167076 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116599+0000", 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.145 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116597+0000", 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116606+0000", 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6690000000000001e-06 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116632+0000", 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6047e-05 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116645+0000", 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2262999999999999e-05 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116733+0000", 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8582e-05 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.116979+0000", 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024556900000000001 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:21:38.148 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.117103+0000", 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012399700000000001 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.121152+0000", 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004049708 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.149 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.121436+0000", 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00028356799999999999 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353539+0000", 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23210345499999999 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.150 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353569+0000", 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9068e-05 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.353593+0000", 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4399000000000001e-05 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231202 8.b 8:da53e5b5:::benchmark_data_smithi107_251897_object114288:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 308.48812354299997, 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.094812632, 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.152 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231202 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.153 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596897+0000", 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596896+0000", 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596901+0000", 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7650000000000001e-06 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596912+0000", 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0589e-05 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596918+0000", 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9939999999999997e-06 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596959+0000", 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1072000000000002e-05 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:38.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596966+0000", 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8079999999999999e-06 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.676763+0000", 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0797972950000001 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.680993+0000", 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042302499999999996 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.681374+0000", 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00038049299999999998 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.158 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.690404+0000", 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090298470000000006 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691560+0000", 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001156921 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691620+0000", 2023-10-09T06:21:38.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9765000000000001e-05 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.691710+0000", 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9808000000000005e-05 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233197 8.b 8:d053390f:::benchmark_data_smithi107_251897_object116283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 305.15291557900002, 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26465855500000002, 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233197 2023-10-09T06:21:38.161 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932105+0000", 2023-10-09T06:21:38.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932104+0000", 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932112+0000", 2023-10-09T06:21:38.163 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8869999999999992e-06 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932128+0000", 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5917999999999999e-05 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932134+0000", 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5560000000000004e-06 2023-10-09T06:21:38.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.936939+0000", 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048044639999999996 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.937036+0000", 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7293000000000005e-05 2023-10-09T06:21:38.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.937103+0000", 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6816000000000005e-05 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941206+0000", 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041027770000000002 2023-10-09T06:21:38.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196612+0000", 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.255406313 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196671+0000", 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8581e-05 2023-10-09T06:21:38.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.196764+0000", 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3227999999999996e-05 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.168 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233199 8.b 8:d15cf55a:::benchmark_data_smithi107_251897_object116285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 305.15286996100002, 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27300426900000002, 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233199 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.169 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932151+0000", 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932153+0000", 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7879999999999998e-06 2023-10-09T06:21:38.171 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932160+0000", 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932164+0000", 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2820000000000001e-06 2023-10-09T06:21:38.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941229+0000", 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090653000000000001 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941345+0000", 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116309 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.941461+0000", 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011604600000000001 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945642+0000", 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041809969999999997 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.174 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.204998+0000", 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25935536799999998 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.205050+0000", 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1820000000000002e-05 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.205155+0000", 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105695 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.176 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233200 8.b 8:df5e5b62:::benchmark_data_smithi107_251897_object116286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 305.15285363100003, 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.284936368, 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233200 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.177 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.178 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932167+0000", 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932169+0000", 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8959999999999999e-06 2023-10-09T06:21:38.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932174+0000", 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0200000000000002e-06 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.932176+0000", 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2840000000000001e-06 2023-10-09T06:21:38.180 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945722+0000", 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013545756000000001 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.945858+0000", 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013620700000000001 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.181 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.946003+0000", 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014461399999999999 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.955361+0000", 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0093577720000000003 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.182 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.216959+0000", 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159835399999998 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217014+0000", 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4593999999999999e-05 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.183 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217104+0000", 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0131999999999997e-05 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233227 8.b 8:dfa0532a:::benchmark_data_smithi107_251897_object116313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:38.184 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 305.12149492399999, 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25373610600000002, 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233227 2023-10-09T06:21:38.185 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963526+0000", 2023-10-09T06:21:38.186 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963525+0000", 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963534+0000", 2023-10-09T06:21:38.187 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.918e-06 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963548+0000", 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4175999999999999e-05 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963553+0000", 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2399999999999998e-06 2023-10-09T06:21:38.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963604+0000", 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.075e-05 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963732+0000", 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012822300000000001 2023-10-09T06:21:38.189 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.963849+0000", 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116897 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.967834+0000", 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0039854490000000003 2023-10-09T06:21:38.190 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217144+0000", 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24931002699999999 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217186+0000", 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1783999999999998e-05 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.191 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217262+0000", 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5878999999999996e-05 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:233231 8.b 8:d0139bd7:::benchmark_data_smithi107_251897_object116317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-10-09T06:21:38.192 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 305.116653036, 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24903313799999999, 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 233231 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.193 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968368+0000", 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.194 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968367+0000", 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968371+0000", 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9659999999999998e-06 2023-10-09T06:21:38.195 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968380+0000", 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4239999999999999e-06 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968384+0000", 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0790000000000002e-06 2023-10-09T06:21:38.196 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968420+0000", 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5225999999999998e-05 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968521+0000", 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010161400000000001 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.197 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.968599+0000", 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7506e-05 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:32.971331+0000", 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002732319 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.198 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217292+0000", 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245960447 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217326+0000", 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4758999999999997e-05 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.199 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:33.217401+0000", 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4668999999999999e-05 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:68752 9.e 9:74c48cb7:::benchmark_data_smithi107_300016_object68751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-10-09T06:21:38.200 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 163.77810697699999, 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16018084899999999, 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 68752 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.201 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306914+0000", 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.202 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306912+0000", 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306959+0000", 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6937999999999997e-05 2023-10-09T06:21:38.203 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306976+0000", 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6347000000000001e-05 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.306982+0000", 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2419999999999997e-06 2023-10-09T06:21:38.204 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307041+0000", 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9388999999999997e-05 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307224+0000", 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018262700000000001 2023-10-09T06:21:38.205 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.307418+0000", 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019383900000000001 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.309930+0000", 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002512065 2023-10-09T06:21:38.206 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.312639+0000", 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00270893 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.464532+0000", 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15189308000000001 2023-10-09T06:21:38.207 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.464601+0000", 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9190000000000007e-05 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:18:54.467095+0000", 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024937549999999998 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.208 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:82724 9.e 9:71b15de5:::benchmark_data_smithi107_300016_object82723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 144.22999284900001, 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14464223500000001, 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.209 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82724 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.210 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855028+0000", 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855027+0000", 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:38.211 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855057+0000", 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9502e-05 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855064+0000", 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7800000000000001e-06 2023-10-09T06:21:38.212 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855069+0000", 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1989999999999999e-06 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855121+0000", 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2414999999999999e-05 2023-10-09T06:21:38.213 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855304+0000", 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018354600000000001 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.855547+0000", 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024204000000000001 2023-10-09T06:21:38.214 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.859695+0000", 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041487620000000003 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.869371+0000", 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0096756419999999999 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.215 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.998875+0000", 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.12950377699999999 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.998940+0000", 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5136999999999997e-05 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.216 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:13.999670+0000", 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00073044400000000002 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:83753 9.e 9:7eb22873:::benchmark_data_smithi107_300016_object83752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e171)", 2023-10-09T06:21:38.217 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 142.13661478700001, 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.276253264, 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 83753 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.218 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948406+0000", 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.219 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948404+0000", 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948466+0000", 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1511000000000001e-05 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.220 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948484+0000", 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8709999999999999e-05 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.948491+0000", 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0999999999999998e-06 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.221 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949392+0000", 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00090033300000000001 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949525+0000", 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013276299999999999 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.222 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.949682+0000", 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015716800000000001 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.955464+0000", 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057822680000000001 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.223 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:15.956844+0000", 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013803540000000001 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.223817+0000", 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26697311299999998 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.224 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.223888+0000", 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0650999999999998e-05 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:16.224659+0000", 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077126199999999999 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.225 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:104644 9.e 9:7de52a14:::benchmark_data_smithi107_300016_object7072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:46.419368+0000", 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 111.66565328900001, 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20690633899999999, 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.226 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 104644 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419368+0000", 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.227 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419368+0000", 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419366+0000", 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.228 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419374+0000", 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5159999999999997e-06 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419387+0000", 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3807e-05 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.229 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419393+0000", 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.49e-06 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419442+0000", 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9379000000000003e-05 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.230 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419587+0000", 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000144909 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.419723+0000", 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135895 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.423278+0000", 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00355543 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.425983+0000", 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00270484 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.625911+0000", 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.199927308 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.625956+0000", 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4922999999999997e-05 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.233 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.626274+0000", 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00031843099999999999 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDPGPushReply(9.e 182/166 [PushReplyOp(9:79eed43c:::benchmark_data_smithi107_300016_object69291:head)])", 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:46.464857+0000", 2023-10-09T06:21:38.234 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 111.620164414, 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.157637949, 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464857+0000", 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464857+0000", 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464856+0000", 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464861+0000", 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.062e-06 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464870+0000", 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2650000000000002e-06 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.237 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.464875+0000", 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8500000000000002e-06 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621854+0000", 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15697899300000001 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.238 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.622494+0000", 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00064039100000000003 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45275.0:104645 9.4 e182/152 9:205b0543:::benchmark_data_smithi107_300016_object7073:head v 182'6521, mlcod=182'6518)", 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:46.477124+0000", 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 111.607896846, 2023-10-09T06:21:38.239 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15332479800000001, 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.477124+0000", 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.240 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.477124+0000", 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.477123+0000", 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.477130+0000", 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6749999999999996e-06 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621831+0000", 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14470107099999999 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621841+0000", 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7180000000000008e-06 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621882+0000", 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0936000000000002e-05 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621898+0000", 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6467e-05 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.630412+0000", 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0085139780000000002 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.630449+0000", 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6619000000000003e-05 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:104657 9.e 9:75165e4a:::benchmark_data_smithi107_300016_object7085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:46.481385+0000", 2023-10-09T06:21:38.245 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 111.60363548399999, 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.196114018, 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 104657 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.481385+0000", 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.481385+0000", 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.481384+0000", 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.481392+0000", 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6670000000000002e-06 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621813+0000", 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.140420774 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.621831+0000", 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8284e-05 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.622532+0000", 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00070035799999999999 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.622683+0000", 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015113600000000001 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.622832+0000", 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014965000000000001 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.628917+0000", 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0060849069999999996 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:38.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.628932+0000", 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4918e-05 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.677205+0000", 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.048272324999999998 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:38.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.677247+0000", 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2765999999999997e-05 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:46.677499+0000", 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00025208700000000001 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:38.253 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:38.254 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:21:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:38.255 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:38.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:38.306 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:21:38.356 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:38.378 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[220921]: pgmap v696: 99 pgs: 99 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:21:38.378 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:21:38.378 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:21:38.379 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[212750]: pgmap v696: 99 pgs: 99 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:21:38.379 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:21:38.379 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:38 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:21:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:38 smithi150 ceph-mon[194178]: pgmap v696: 99 pgs: 99 active+clean; 933 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T06:21:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:38 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:21:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:38 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:21:38.458 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:38.479 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:38.523 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:38.567 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:38.568 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:38.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:38.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:38.629 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:38.667 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:38.675 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:38.704 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:38.735 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:38.737 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:38.777 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.785 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:38.786 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:39.100 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:39.101 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:21:39.101 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:39.101 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-09T06:21:39.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:21:39.171 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:39.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:39.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:39.281 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:39.353 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/373727805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:39.639 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:39.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/373727805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:39.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:39.646 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/373727805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:39.704 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:39.747 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:39.779 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:39.818 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:40.052 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:40.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:40.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:40.232 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:40.295 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:40.342 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[212750]: pgmap v697: 99 pgs: 99 active+clean; 783 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-10-09T06:21:40.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:21:40.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T06:21:40.343 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[220921]: pgmap v697: 99 pgs: 99 active+clean; 783 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-10-09T06:21:40.343 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[220921]: 1.5 scrub starts 2023-10-09T06:21:40.343 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:40 smithi107 ceph-mon[220921]: 1.5 scrub ok 2023-10-09T06:21:40.485 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:40.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:40.593 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:40.619 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:40 smithi150 ceph-mon[194178]: pgmap v697: 99 pgs: 99 active+clean; 783 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-10-09T06:21:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:40 smithi150 ceph-mon[194178]: 1.5 scrub starts 2023-10-09T06:21:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:40 smithi150 ceph-mon[194178]: 1.5 scrub ok 2023-10-09T06:21:41.070 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:41.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:41.180 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:41.271 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:41.389 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:41.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:41.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:41.555 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:41.756 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:41.776 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:41.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:41.860 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:41.860 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:41.887 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:41.954 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:42.105 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:42.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:42.214 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:42.249 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:42] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-10-09T06:21:42.264 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:42.397 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:42.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:42.499 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:42.556 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:42 smithi150 ceph-mon[194178]: pgmap v698: 99 pgs: 99 active+clean; 743 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T06:21:42.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:42 smithi107 ceph-mon[212750]: pgmap v698: 99 pgs: 99 active+clean; 743 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T06:21:42.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:42 smithi107 ceph-mon[220921]: pgmap v698: 99 pgs: 99 active+clean; 743 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T06:21:42.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:21:42.757 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:42.850 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:42.850 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:21:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:42.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:42.960 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:43.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:43.260 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:43.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:43.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:43.379 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:43.399 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:21:43.433 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/134801470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.434 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:43 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/134801470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.434 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:43 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.453 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:43.552 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:43.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:43.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:43.625 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:43.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/134801470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:43 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:21:43.669 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:43.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:43 smithi107 conmon[212706]: 2023-10-09T06:21:43.613+0000 7f7c794f9700 -1 mon.a@0(leader).osd e221 definitely_dead 0 2023-10-09T06:21:43.726 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: pgmap v699: 99 pgs: 99 active+clean; 713 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:44.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:44 smithi150 ceph-mon[194178]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:44.401 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.5. 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: pgmap v699: 99 pgs: 99 active+clean; 713 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:21:44.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[212750]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: pgmap v699: 99 pgs: 99 active+clean; 713 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: failed to encode map e221 with expected crc 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-09T06:21:44.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:44 smithi107 ceph-mon[220921]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-09T06:21:45.223 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:45.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:45.327 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:45.356 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:45.535 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.537 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[212750]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.538 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:45 smithi107 ceph-mon[220921]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:45.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:45.640 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:45.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1328798735' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-09T06:21:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: osdmap e222: 8 total, 8 up, 7 in 2023-10-09T06:21:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:45 smithi150 ceph-mon[194178]: failed to encode map e222 with expected crc 2023-10-09T06:21:45.697 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:45.742 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:45.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:45.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:45.846 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:45.877 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:45.886 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:45.908 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:45.930 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:45.951 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:45.972 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:45.991 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:46.010 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:46.029 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:46.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:46.033 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.034 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.035 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.035 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:21:46.064 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:21:46.064 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:21:46.064 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:21:46.065 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:21:46.065 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:21:46.134 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:46.161 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:46.307 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:46.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:46.411 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:46.495 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: pgmap v702: 99 pgs: 7 peering, 92 active+clean; 528 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3044838316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:46.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: pgmap v702: 99 pgs: 7 peering, 92 active+clean; 528 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:21:46.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.784 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.784 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.784 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.784 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3044838316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: pgmap v702: 99 pgs: 7 peering, 92 active+clean; 528 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: osdmap e223: 8 total, 8 up, 7 in 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: failed to encode map e223 with expected crc 2023-10-09T06:21:46.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3044838316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:46.789 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:46.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:46.902 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:46.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:21:46.987 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:47.162 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:47.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:47.276 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:47.343 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:47.506 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:47.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:47.563 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 221'11773 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 221'11612 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T06:21:47.599 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 221'11763 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 221'11716 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 221'11870 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 221'11882 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 221'11957 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 221'11803 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 221'11590 2023-10-09T06:21:47.600 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.601 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 221'11773 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 221'11612 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 221'11763 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 221'11716 2023-10-09T06:21:47.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 221'11870 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 221'11882 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 221'11957 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 221'11803 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 221'11590 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T06:21:47.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:47 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.616 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:47.653 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:21:47 smithi150 conmon[174357]: logger=cleanup t=2023-10-09T06:21:47.276524404Z level=info msg="Completed cleanup jobs" duration=128.718936ms 2023-10-09T06:21:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 221'11773 2023-10-09T06:21:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 221'11612 2023-10-09T06:21:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: osdmap e224: 8 total, 8 up, 7 in 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 221'11763 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 221'11716 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.7 starting backfill to osd.2 from (0'0,0'0] MAX to 221'11870 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 221'11882 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e224 with expected crc 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 221'11957 2023-10-09T06:21:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 221'11803 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 221'11590 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: osdmap e225: 8 total, 8 up, 7 in 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:47 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:47.698 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:47.806 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:47.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:47.830 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:47.911 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:47.981 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:48.171 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:48.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:48.278 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:48.322 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:48.374 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:48.541 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:48.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:48.655 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: pgmap v705: 99 pgs: 2 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 87 active+clean; 502 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 876 op/s; 948/24153 objects misplaced (3.925%); 131 KiB/s, 0 objects/s recovering 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: pgmap v705: 99 pgs: 2 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 87 active+clean; 502 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 876 op/s; 948/24153 objects misplaced (3.925%); 131 KiB/s, 0 objects/s recovering 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.658 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:48 smithi107 ceph-mon[220921]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: pgmap v705: 99 pgs: 2 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 87 active+clean; 502 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 876 op/s; 948/24153 objects misplaced (3.925%); 131 KiB/s, 0 objects/s recovering 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e225 with expected crc 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:21:48.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: osdmap e226: 8 total, 8 up, 7 in 2023-10-09T06:21:48.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:48 smithi150 ceph-mon[194178]: failed to encode map e226 with expected crc 2023-10-09T06:21:48.733 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:48.801 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:21:49.437 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:49.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:49.543 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:49.575 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:49.617 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:49.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:49 smithi107 ceph-mon[220921]: failed to encode map e226 with expected crc 2023-10-09T06:21:49.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:49 smithi107 ceph-mon[212750]: failed to encode map e226 with expected crc 2023-10-09T06:21:49.721 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:49.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:49 smithi150 ceph-mon[194178]: failed to encode map e226 with expected crc 2023-10-09T06:21:49.776 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:50.181 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:50.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:50.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:50.320 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:50.453 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: pgmap v708: 99 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 84 active+clean; 443 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 371 op/s; 1794/21330 objects misplaced (8.411%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.454 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:50 smithi150 ceph-mon[194178]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.563 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:50.563 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:21:50.564 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T06:21:50.564 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:21:50.564 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:21:50.564 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T06:21:50.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: pgmap v708: 99 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 84 active+clean; 443 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 371 op/s; 1794/21330 objects misplaced (8.411%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[212750]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: pgmap v708: 99 pgs: 2 active+remapped+backfilling, 3 active+remapped+backfill_wait, 3 remapped+peering, 7 peering, 84 active+clean; 443 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 41 KiB/s rd, 109 KiB/s wr, 371 op/s; 1794/21330 objects misplaced (8.411%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: osdmap e227: 8 total, 8 up, 7 in 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:50 smithi107 ceph-mon[220921]: failed to encode map e227 with expected crc 2023-10-09T06:21:50.778 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:21:50.778 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:21:50.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:75994 8.3 8:cd98f0c9:::benchmark_data_smithi107_251897_object75993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e111)", 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:13:02.830213+0000", 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 527.94505246999995, 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0794299599999999, 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.779 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 75994 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.830213+0000", 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.830213+0000", 2023-10-09T06:21:50.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.830211+0000", 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.833971+0000", 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037597189999999999 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.833985+0000", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3987e-05 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.833994+0000", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1600000000000004e-06 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.834616+0000", 2023-10-09T06:21:50.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000622279 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:02.834629+0000", 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2938999999999999e-05 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.904463+0000", 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.069834108 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.904629+0000", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016562799999999999 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.904789+0000", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016024799999999999 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.908655+0000", 2023-10-09T06:21:50.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0038660700000000001 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.908718+0000", 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3360000000000003e-05 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.908850+0000", 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000131765 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.908915+0000", 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4622000000000006e-05 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:13:03.909643+0000", 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072828499999999996 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:162322 8.d 8:b0b2eb65:::benchmark_data_smithi107_251897_object45408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 419.586830152, 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.051913984, 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 162322 2023-10-09T06:21:50.787 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188434+0000", 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188444+0000", 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0066e-05 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188459+0000", 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5274e-05 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188466+0000", 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.131e-06 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.191241+0000", 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002774136 2023-10-09T06:21:50.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.191248+0000", 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8590000000000005e-06 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223345+0000", 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.032096401 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223500+0000", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015488200000000001 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223624+0000", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000124547 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.229333+0000", 2023-10-09T06:21:50.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057091090000000004 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240173+0000", 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010839867 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240236+0000", 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2704000000000001e-05 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240349+0000", 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113404 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176903 8.4 8:27754661:::benchmark_data_smithi107_251897_object59989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.07245196000002, 2023-10-09T06:21:50.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1790666839999999, 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176903 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.795 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702812+0000", 2023-10-09T06:21:50.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702818+0000", 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0780000000000001e-06 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702831+0000", 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605e-05 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702837+0000", 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7039999999999999e-06 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702889+0000", 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2024000000000002e-05 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702901+0000", 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2029e-05 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872747+0000", 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16984611299999999 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872752+0000", 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:21:50.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859323+0000", 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9865709140000001 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859469+0000", 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000145601 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859570+0000", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010074800000000001 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872610+0000", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013039996 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.876987+0000", 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043772840000000004 2023-10-09T06:21:50.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881653+0000", 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046657829999999997 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881708+0000", 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5614e-05 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881880+0000", 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017200200000000001 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176909 8.4 8:22feb7e8:::benchmark_data_smithi107_251897_object59995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.06824138500002, 2023-10-09T06:21:50.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1750447849999999, 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176909 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707022+0000", 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707030+0000", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8760000000000005e-06 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707048+0000", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7941999999999999e-05 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707054+0000", 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9989999999999997e-06 2023-10-09T06:21:50.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707113+0000", 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8666999999999999e-05 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707120+0000", 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3939999999999998e-06 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872779+0000", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.165658478 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872784+0000", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9880000000000004e-06 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864131+0000", 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9913474199999999 2023-10-09T06:21:50.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864242+0000", 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110747 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864344+0000", 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010181 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.876999+0000", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012655680000000001 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877174+0000", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017434199999999999 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881928+0000", 2023-10-09T06:21:50.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047547379999999997 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881972+0000", 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3727000000000003e-05 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882069+0000", 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6674000000000003e-05 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.811 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176918 8.d 8:b53b7e9b:::benchmark_data_smithi107_251897_object60004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.05230719399998, 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1380759039999999, 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.812 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176918 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722957+0000", 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722963+0000", 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1469999999999998e-06 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722976+0000", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2649999999999999e-05 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722982+0000", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1870000000000002e-06 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.723026+0000", 2023-10-09T06:21:50.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3983000000000002e-05 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.723034+0000", 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0139999999999998e-06 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.871899+0000", 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14886518500000001 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.871912+0000", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3059e-05 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850243+0000", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.978331117 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850348+0000", 2023-10-09T06:21:50.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104239 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850442+0000", 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4621000000000001e-05 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856219+0000", 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057764180000000002 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856284+0000", 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4817e-05 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.860850+0000", 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045668949999999996 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.860890+0000", 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9864999999999999e-05 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861034+0000", 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014390900000000001 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.820 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176926 8.4 8:2d6c801d:::benchmark_data_smithi107_251897_object60012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.04173670900002, 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1546555550000002, 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176926 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733527+0000", 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733534+0000", 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5610000000000004e-06 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733550+0000", 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6461999999999999e-05 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733557+0000", 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.951e-06 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733609+0000", 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2036999999999997e-05 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733618+0000", 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9420000000000001e-06 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872805+0000", 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13918652400000001 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872809+0000", 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0659999999999997e-06 2023-10-09T06:21:50.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868505+0000", 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.995695926 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868615+0000", 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110449 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868720+0000", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105222 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877006+0000", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082859040000000002 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877206+0000", 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019972699999999999 2023-10-09T06:21:50.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888036+0000", 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010830358 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888083+0000", 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6508999999999998e-05 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888184+0000", 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010166300000000001 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176930 8.d 8:b7d0ab16:::benchmark_data_smithi107_251897_object60016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.03542674099998, 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.126278691, 2023-10-09T06:21:50.829 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176930 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739838+0000", 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739843+0000", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1479999999999997e-06 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739858+0000", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4751e-05 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739863+0000", 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1619999999999999e-06 2023-10-09T06:21:50.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739908+0000", 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5120000000000002e-05 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739916+0000", 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2509999999999994e-06 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872532+0000", 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13261576799999999 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872537+0000", 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1209999999999998e-06 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856314+0000", 2023-10-09T06:21:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9837771659999999 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856430+0000", 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115678 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856532+0000", 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000102012 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861142+0000", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046098279999999998 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861431+0000", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00028886999999999999 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.865951+0000", 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045204829999999996 2023-10-09T06:21:50.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.865993+0000", 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1465999999999999e-05 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.866117+0000", 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012464099999999999 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.837 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176936 8.4 8:22e0b0cb:::benchmark_data_smithi107_251897_object60022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.00612137000002, 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1192077970000001, 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.838 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176936 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769142+0000", 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769150+0000", 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6930000000000007e-06 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769167+0000", 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7257000000000001e-05 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769187+0000", 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9686000000000001e-05 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769239+0000", 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2289e-05 2023-10-09T06:21:50.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769249+0000", 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7450000000000007e-06 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872828+0000", 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.103578663 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872835+0000", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8569999999999997e-06 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872651+0000", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.999816456 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872800+0000", 2023-10-09T06:21:50.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014936699999999999 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872917+0000", 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011622500000000001 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882080+0000", 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0091632380000000006 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882556+0000", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047562599999999999 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888224+0000", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0056688099999999998 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888260+0000", 2023-10-09T06:21:50.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5235999999999999e-05 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888352+0000", 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2367999999999994e-05 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:199436 8.d 8:ba48c3db:::benchmark_data_smithi107_251897_object82522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T06:21:50.846 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 366.60083345499999, 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0633727580000001, 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 199436 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174431+0000", 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174437+0000", 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174453+0000", 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6113e-05 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174459+0000", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9200000000000001e-06 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174522+0000", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2923999999999996e-05 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174530+0000", 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:21:50.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.181995+0000", 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0074647359999999996 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.182005+0000", 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8849999999999994e-06 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225165+0000", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0431607389999999 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225304+0000", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000138639 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225396+0000", 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1552999999999996e-05 2023-10-09T06:21:50.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230017+0000", 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046212960000000004 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.234320+0000", 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043027839999999996 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.853 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237733+0000", 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034128320000000002 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237769+0000", 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237805+0000", 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5726000000000003e-05 2023-10-09T06:21:50.854 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:199440 8.d 8:b1aca3b0:::benchmark_data_smithi107_251897_object82526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 366.60079417499998, 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0634993260000001, 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.855 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 199440 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:21:50.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174474+0000", 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.027e-06 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174482+0000", 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2260000000000005e-06 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174486+0000", 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6219999999999998e-06 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174598+0000", 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000112007 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174606+0000", 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6989999999999992e-06 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.184897+0000", 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010290875 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.184909+0000", 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1392999999999999e-05 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230049+0000", 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.045139968 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230145+0000", 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6695999999999997e-05 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230229+0000", 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3279e-05 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237876+0000", 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0076473210000000003 2023-10-09T06:21:50.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237909+0000", 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3229000000000003e-05 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237916+0000", 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0840000000000003e-06 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237948+0000", 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1696000000000003e-05 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237971+0000", 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2626e-05 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231182 8.8 8:1499a486:::benchmark_data_smithi107_251897_object114268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.20918355600003, 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.088702015, 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231182 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.864 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566081+0000", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566086+0000", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8790000000000002e-06 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566097+0000", 2023-10-09T06:21:50.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1422e-05 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566104+0000", 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2280000000000003e-06 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581612+0000", 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015507917 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581747+0000", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135372 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581874+0000", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127367 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.603525+0000", 2023-10-09T06:21:50.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.021651113999999999 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608048+0000", 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045228769999999998 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608056+0000", 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9440000000000005e-06 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.654661+0000", 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0466043810000001 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.654784+0000", 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012337900000000001 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231183 8.8 8:18f72ceb:::benchmark_data_smithi107_251897_object114269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.20761333799999, 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.087511761, 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231183 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.871 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567651+0000", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567656+0000", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5390000000000001e-06 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567665+0000", 2023-10-09T06:21:50.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9449999999999999e-06 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567670+0000", 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6010000000000002e-06 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587108+0000", 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019437578 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587234+0000", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012631599999999999 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587314+0000", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9302999999999996e-05 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608084+0000", 2023-10-09T06:21:50.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020770617000000002 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608090+0000", 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4040000000000003e-06 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655105+0000", 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047015048 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655164+0000", 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9188999999999999e-05 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231188 8.8 8:1a06de55:::benchmark_data_smithi107_251897_object114274:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.20038396000001, 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0803605979999999, 2023-10-09T06:21:50.877 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231188 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574880+0000", 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574885+0000", 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8049999999999997e-06 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574893+0000", 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9859999999999994e-06 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574898+0000", 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8559999999999998e-06 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592228+0000", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017330007000000001 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592360+0000", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013205199999999999 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592479+0000", 2023-10-09T06:21:50.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119114 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608169+0000", 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.01568955 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608175+0000", 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0469999999999999e-06 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655187+0000", 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047012791 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655242+0000", 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4656e-05 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231189 8.8 8:17d1e237:::benchmark_data_smithi107_251897_object114275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.20036288799997, 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.080469938, 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231189 2023-10-09T06:21:50.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574902+0000", 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574904+0000", 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1169999999999998e-06 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574909+0000", 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7740000000000004e-06 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574912+0000", 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2559999999999998e-06 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597542+0000", 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022629911999999999 2023-10-09T06:21:50.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597662+0000", 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120011 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597760+0000", 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7397000000000003e-05 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608199+0000", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010439321999999999 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608204+0000", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3709999999999999e-06 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655287+0000", 2023-10-09T06:21:50.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047082477 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655373+0000", 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5600999999999996e-05 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231200 8.8 8:19497162:::benchmark_data_smithi107_251897_object114286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.890 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.18252527499999, 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1339951880000001, 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231200 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.891 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592739+0000", 2023-10-09T06:21:50.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592746+0000", 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2910000000000002e-06 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592762+0000", 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5903999999999999e-05 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592769+0000", 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2899999999999997e-06 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608231+0000", 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015461805 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608236+0000", 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0869999999999997e-06 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707544+0000", 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0993079569999999 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707644+0000", 2023-10-09T06:21:50.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010024800000000001 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707766+0000", 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012206600000000001 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.721160+0000", 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013393693999999999 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.896 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726598+0000", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0054379540000000001 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726652+0000", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4061000000000003e-05 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726735+0000", 2023-10-09T06:21:50.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3263999999999994e-05 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231204 8.8 8:148b492c:::benchmark_data_smithi107_251897_object114290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.17933970899998, 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1386598640000001, 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231204 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595925+0000", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595932+0000", 2023-10-09T06:21:50.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4150000000000002e-06 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595947+0000", 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5262000000000001e-05 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595954+0000", 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6080000000000001e-06 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608567+0000", 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012613447 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608574+0000", 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7889999999999997e-06 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.712886+0000", 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.104311611 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.713018+0000", 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013194699999999999 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.713140+0000", 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012193600000000001 2023-10-09T06:21:50.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726754+0000", 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013614131999999999 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734474+0000", 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.007720144 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734509+0000", 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5472999999999997e-05 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734586+0000", 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6318999999999999e-05 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.905 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231191 8.0 8:0eb72051:::benchmark_data_smithi107_251897_object114277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.17843353299997, 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.106525803, 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231191 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596831+0000", 2023-10-09T06:21:50.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596836+0000", 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1610000000000002e-06 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596847+0000", 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1122e-05 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596853+0000", 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9220000000000002e-06 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608255+0000", 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011402123 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608260+0000", 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.994e-06 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703247+0000", 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0949865670000001 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703358+0000", 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110809 2023-10-09T06:21:50.910 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231194 8.0 8:0cbc2529:::benchmark_data_smithi107_251897_object114280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.17840688799998, 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.106776822, 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.911 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231194 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596858+0000", 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596861+0000", 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3479999999999998e-06 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596866+0000", 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.49e-06 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596870+0000", 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608278+0000", 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011408702999999999 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608283+0000", 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9649999999999999e-06 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703571+0000", 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0952875129999999 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703635+0000", 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4727000000000006e-05 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231198 8.0 8:080624c4:::benchmark_data_smithi107_251897_object114284:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:21:50.916 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.17839244499999, 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.12997952, 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231198 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:21:50.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:21:50.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596875+0000", 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0490000000000002e-06 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596880+0000", 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8910000000000003e-06 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596883+0000", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.253e-06 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608302+0000", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011419123 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608307+0000", 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1109999999999999e-06 2023-10-09T06:21:50.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703683+0000", 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0953758200000001 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for missing object", 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703802+0000", 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011947600000000001 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726773+0000", 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022970059000000001 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726853+0000", 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0047000000000001e-05 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.922 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231206 8.0 8:0fa8cc49:::benchmark_data_smithi107_251897_object114292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 321.16291665699998, 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1043428710000001, 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231206 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.923 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:21:50.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612347+0000", 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612353+0000", 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4909999999999996e-06 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612367+0000", 2023-10-09T06:21:50.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.419e-05 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612373+0000", 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8980000000000001e-06 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612410+0000", 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6918999999999997e-05 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612415+0000", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7620000000000003e-06 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703839+0000", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0914244500000001 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703923+0000", 2023-10-09T06:21:50.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3504999999999994e-05 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.704006+0000", 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3324000000000004e-05 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.712425+0000", 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0084194160000000007 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716586+0000", 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041602469999999997 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716623+0000", 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7771999999999997e-05 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:21:50.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716692+0000", 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8279000000000001e-05 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:21:50.930 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:21:51.593 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:51.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:51.607 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:21:51.665 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:51.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:51.737 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.737 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T06:21:51.737 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.737 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.738 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.739 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.739 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.739 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.747 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:51.814 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:51.823 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:51.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:21:51.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: osdmap e228: 8 total, 8 up, 7 in 2023-10-09T06:21:51.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.876 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.877 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: failed to encode map e228 with expected crc 2023-10-09T06:21:51.877 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.877 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:51.929 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:51.957 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:52.154 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:52.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:52.253 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:21:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:21:52] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T06:21:52.260 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:52.292 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:52.337 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:52.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:52.440 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:52.504 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: pgmap v711: 99 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 412 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 375 op/s; 2874/19851 objects misplaced (14.478%); 13 MiB/s, 206 objects/s recovering 2023-10-09T06:21:52.504 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.504 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: pgmap v711: 99 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 412 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 375 op/s; 2874/19851 objects misplaced (14.478%); 13 MiB/s, 206 objects/s recovering 2023-10-09T06:21:52.506 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.508 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:52.660 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:52.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:52.749 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: pgmap v711: 99 pgs: 2 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 412 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 375 op/s; 2874/19851 objects misplaced (14.478%); 13 MiB/s, 206 objects/s recovering 2023-10-09T06:21:52.749 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.749 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: osdmap e229: 8 total, 8 up, 7 in 2023-10-09T06:21:52.749 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e229 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: osdmap e230: 8 total, 8 up, 7 in 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: failed to encode map e230 with expected crc 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:21:52.751 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:52.751 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:52.763 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:52.791 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:52.833 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:52.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:52.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:52.936 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:52.965 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:53.077 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:21:53.085 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:53.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:53.188 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:53.248 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:53.310 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:53.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:53.413 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:53.445 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:53.449 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:21:53.450 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "" 2023-10-09T06:21:53.450 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:21:53.494 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:53.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:53.597 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:53.624 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:53.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:53.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:53.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[212750]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T06:21:53.811 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.812 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.812 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.812 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:53 smithi107 ceph-mon[220921]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.847 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:53.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1914703835' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: osdmap e231: 8 total, 8 up, 7 in 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:53 smithi150 ceph-mon[194178]: failed to encode map e231 with expected crc 2023-10-09T06:21:53.951 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:53.977 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:54.313 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:54.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:54.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:54.427 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:54.464 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:54.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[212750]: pgmap v714: 99 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 92 active+clean; 382 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 260 op/s; 1926/18405 objects misplaced (10.465%); 18 MiB/s, 293 objects/s recovering 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[212750]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[212750]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[220921]: pgmap v714: 99 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 92 active+clean; 382 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 260 op/s; 1926/18405 objects misplaced (10.465%); 18 MiB/s, 293 objects/s recovering 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[220921]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[220921]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:21:54.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:54 smithi107 ceph-mon[220921]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.638 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:54.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:54.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:54.746 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:54.785 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:54 smithi150 ceph-mon[194178]: pgmap v714: 99 pgs: 1 peering, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 92 active+clean; 382 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 260 op/s; 1926/18405 objects misplaced (10.465%); 18 MiB/s, 293 objects/s recovering 2023-10-09T06:21:54.785 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:54 smithi150 ceph-mon[194178]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:54 smithi150 ceph-mon[194178]: osdmap e232: 8 total, 8 up, 7 in 2023-10-09T06:21:54.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:54 smithi150 ceph-mon[194178]: failed to encode map e232 with expected crc 2023-10-09T06:21:54.792 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:54.849 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:54.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:54.859 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:21:54.909 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:21:54.939 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:21:54.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:54.975 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:21:55.008 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:55.027 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:21:55.071 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:21:55.104 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:21:55.132 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:55.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:55.145 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:21:55.168 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.170 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:21:55.205 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:55.205 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:55.239 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:55.329 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:55.435 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:55.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:55.545 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:21:55.586 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:55.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:21:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:55.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:21:55.744 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:55.818 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2295461143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:55.819 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2295461143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:55.825 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2295461143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:21:56.040 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:21:56.078 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:56.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:56.183 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:56.217 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:56.284 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:56.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:56.327 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:21:56.393 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:56.495 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:56.513 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:56.619 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:56.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:56 smithi107 ceph-mon[212750]: pgmap v717: 99 pgs: 1 peering, 98 active+clean; 326 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 40 MiB/s, 640 objects/s recovering 2023-10-09T06:21:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:56 smithi107 ceph-mon[220921]: pgmap v717: 99 pgs: 1 peering, 98 active+clean; 326 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 40 MiB/s, 640 objects/s recovering 2023-10-09T06:21:56.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:56 smithi150 ceph-mon[194178]: pgmap v717: 99 pgs: 1 peering, 98 active+clean; 326 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 344 op/s; 40 MiB/s, 640 objects/s recovering 2023-10-09T06:21:56.734 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:56.819 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:56.963 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:56.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:57.062 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:57.080 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:57.123 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:57.181 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:57.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:57.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:57.286 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:21:57.311 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:57.353 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:21:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:57.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:21:57.457 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:57.557 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:57.807 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:57.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:57.918 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:57.973 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:58.112 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:58.125 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:58.125 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:21:57 smithi150 ceph-mon[194178]: pgmap v718: 99 pgs: 99 active+clean; 276 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 29 MiB/s, 459 objects/s recovering 2023-10-09T06:21:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:58.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:58.127 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:58.127 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:21:57 smithi107 ceph-mon[220921]: pgmap v718: 99 pgs: 99 active+clean; 276 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 29 MiB/s, 459 objects/s recovering 2023-10-09T06:21:58.129 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:21:58.129 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:21:57 smithi107 ceph-mon[212750]: pgmap v718: 99 pgs: 99 active+clean; 276 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 392 op/s; 29 MiB/s, 459 objects/s recovering 2023-10-09T06:21:58.201 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:21:58.227 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:21:58.338 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:58.371 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:21:58.371 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:21:58.643 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:21:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:58.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:21:58.750 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:21:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:58.837 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:21:58.883 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:21:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:58.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:21:58.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:21:59.089 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:59.204 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:21:59.360 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:21:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:59.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:21:59.475 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:21:59.512 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:59.580 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:21:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:59.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:21:59.664 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:21:59.691 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:21:59.711 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:21:59.711 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:21:59.729 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:59.816 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:21:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:59.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:21:59.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:21:59.946 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:21:59.989 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:21:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:21:59.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:00.092 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:00.095 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:22:00.120 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[220921]: pgmap v719: 99 pgs: 99 active+clean; 168 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 16 MiB/s, 261 objects/s recovering 2023-10-09T06:22:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1272981039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[212750]: pgmap v719: 99 pgs: 99 active+clean; 168 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 16 MiB/s, 261 objects/s recovering 2023-10-09T06:22:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1272981039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:00 smithi150 ceph-mon[194178]: pgmap v719: 99 pgs: 99 active+clean; 168 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 492 op/s; 16 MiB/s, 261 objects/s recovering 2023-10-09T06:22:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1272981039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:00.432 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:00.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:00.539 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:00.577 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:00.634 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:00.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:00.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:00.742 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:00.820 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:00.991 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 97569 objects 2023-10-09T06:22:00.991 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :144.09 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[212750]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:01 smithi150 ceph-mon[194178]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.185 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:01.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:01.244 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:01.289 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:01.318 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: osdmap e233: 8 total, 8 up, 7 in 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.352 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.353 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:01 smithi107 ceph-mon[220921]: failed to encode map e233 with expected crc 2023-10-09T06:22:01.362 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:01.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:01.412 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:01.413 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:01.466 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:01.496 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:01.554 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:01.574 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:01.595 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:01.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:01.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:01.615 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:01.633 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:01.652 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:01.670 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:01.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:01.689 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:22:01.700 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:01.728 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:01.775 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:01.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:01.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:01.908 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:02.098 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:02] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-10-09T06:22:02.142 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:02.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:02.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:02 smithi150 ceph-mon[194178]: pgmap v721: 99 pgs: 99 active+clean; 108 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 14 MiB/s, 228 objects/s recovering 2023-10-09T06:22:02.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4197419089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:02.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:02.188 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:22:02.246 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:02.275 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[212750]: pgmap v721: 99 pgs: 99 active+clean; 108 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 14 MiB/s, 228 objects/s recovering 2023-10-09T06:22:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4197419089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[220921]: pgmap v721: 99 pgs: 99 active+clean; 108 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 14 MiB/s, 228 objects/s recovering 2023-10-09T06:22:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4197419089' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:02.576 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:02.777 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:02.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:02.880 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:02.907 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:02.951 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:02.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:03.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:03.081 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:03.141 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:22:03.176 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:03.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:03.280 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:03.305 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:03.348 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:03.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:03.401 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T06:22:03.401 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T06:22:03.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:03.481 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:03.510 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:03.529 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:03.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:03.633 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:03.661 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:03.808 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:03.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:03.911 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:03.939 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:04.110 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:22:04.136 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:04.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:04.177 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:04.186 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:22:04.186 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:22:04.186 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:22:04.187 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:22:04.211 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:04 smithi150 ceph-mon[194178]: 1.1 scrub starts 2023-10-09T06:22:04.211 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:04 smithi150 ceph-mon[194178]: 1.1 scrub ok 2023-10-09T06:22:04.211 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:04 smithi150 ceph-mon[194178]: pgmap v722: 99 pgs: 99 active+clean; 61 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:22:04.211 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/307968802' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.211 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.239 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:04.265 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[212750]: pgmap v722: 99 pgs: 99 active+clean; 61 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/307968802' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[220921]: 1.1 scrub starts 2023-10-09T06:22:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[220921]: 1.1 scrub ok 2023-10-09T06:22:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[220921]: pgmap v722: 99 pgs: 99 active+clean; 61 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 501 op/s; 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:22:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/307968802' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:22:04.576 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:04.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:04.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:04.641 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:22:04.681 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:04.781 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:04.843 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:04.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:04.896 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:22:04.947 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:04.974 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:05.023 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:05.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:05.102 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:22:05.128 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:05.156 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:05.203 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:05.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:05.265 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:05.306 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:05.332 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:05.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:05.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[212750]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:22:05.343 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.344 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[220921]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.344 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[220921]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.344 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:05.344 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[220921]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:22:05.344 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:05 smithi107 ceph-mon[220921]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:05.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:05 smithi150 ceph-mon[194178]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:05 smithi150 ceph-mon[194178]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:05 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:22:05.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:05 smithi150 ceph-mon[194178]: osdmap e234: 8 total, 8 up, 7 in 2023-10-09T06:22:05.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:05 smithi150 ceph-mon[194178]: failed to encode map e234 with expected crc 2023-10-09T06:22:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:05.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:05.480 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:05.511 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:05.758 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:05.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:05.861 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:05.888 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:05.928 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:05.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:06.032 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:06.058 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:06.131 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:22:06.197 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:06.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:06.302 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:06.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[212750]: failed to encode map e234 with expected crc 2023-10-09T06:22:06.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[212750]: pgmap v724: 99 pgs: 99 active+clean; 750 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T06:22:06.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:06.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[220921]: failed to encode map e234 with expected crc 2023-10-09T06:22:06.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[220921]: pgmap v724: 99 pgs: 99 active+clean; 750 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T06:22:06.326 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:06.328 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:06.336 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:06.336 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:06.373 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:06 smithi150 ceph-mon[194178]: failed to encode map e234 with expected crc 2023-10-09T06:22:06.373 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:06 smithi150 ceph-mon[194178]: pgmap v724: 99 pgs: 99 active+clean; 750 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T06:22:06.373 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:06.429 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:06.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:06.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:06.532 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:06.562 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:06.707 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:06.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:06.811 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:06.839 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:06.885 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:22:06.886 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:22:06.886 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:22:06.886 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:22:06.886 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:22:06.920 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:06.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:07.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:07.065 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:07.109 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:07.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:07.212 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:07.237 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:07.237 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:22:07.237 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:07.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:22:07.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:07.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:07.240 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:07.284 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:07.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:07.387 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:07.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:07.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/735174812' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: osdmap e235: 8 total, 8 up, 7 in 2023-10-09T06:22:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:07.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:07.417 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:07.564 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:07.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:07.667 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:07.694 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:07.714 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:22:07.939 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:07.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:08.043 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:08.061 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:22:08.069 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:08.113 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:08.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:08.181 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_4' removed 2023-10-09T06:22:08.217 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:08.252 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:08.340 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: pgmap v726: 99 pgs: 99 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 278 op/s 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1485476854' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: pgmap v726: 99 pgs: 99 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 278 op/s 2023-10-09T06:22:08.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1485476854' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: pgmap v726: 99 pgs: 99 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 278 op/s 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: failed to encode map e235 with expected crc 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1485476854' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:22:08.906 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:08.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:08.935 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:22:09.010 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:09.041 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:09.054 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:22:09.054 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:22:09.055 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T06:22:09.055 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T06:22:09.165 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:09.165 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.165 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[212750]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.166 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:09 smithi107 ceph-mon[220921]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:09.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:09.269 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:09.296 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: osdmap e236: 8 total, 8 up, 7 in 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:09 smithi150 ceph-mon[194178]: failed to encode map e236 with expected crc 2023-10-09T06:22:09.359 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:09.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:09.385 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:09.417 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:09.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:09.458 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:09.465 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:09.478 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:09.495 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:09.497 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:09.522 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:09.539 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:09.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:09.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.547 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.548 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:22:09.574 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:09.574 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:09.598 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:09.642 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:09.673 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:09.816 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:09.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:09.920 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:09.950 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:10.156 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:22:10.178 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:10.192 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:10.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:10.199 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[220921]: pgmap v728: 83 pgs: 83 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:10.199 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1271276238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:10.199 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:22:10.200 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[212750]: pgmap v728: 83 pgs: 83 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:10.201 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1271276238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:10.201 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:22:10.246 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:22:10.297 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:10.323 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:10.423 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:10.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:10.527 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:10.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:10 smithi150 ceph-mon[194178]: pgmap v728: 83 pgs: 83 active+clean; 430 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:10.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1271276238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:10.552 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:22:10.554 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:10.597 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:10.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:10.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:10.730 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:10.925 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:10.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:10.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:10.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:22:11.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:11.058 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:11.101 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:11.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:11.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:11.207 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:11.246 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T06:22:11.246 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:22:11.247 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[212750]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:22:11.247 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[220921]: failed to encode map e237 with expected crc 2023-10-09T06:22:11.247 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:22:11.247 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:11 smithi107 ceph-mon[220921]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:22:11.250 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:11.343 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:11.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:11 smithi150 ceph-mon[194178]: failed to encode map e237 with expected crc 2023-10-09T06:22:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/930034132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:22:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:11 smithi150 ceph-mon[194178]: osdmap e237: 8 total, 8 up, 7 in 2023-10-09T06:22:11.446 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:11.474 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:11.516 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:11.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:11.623 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:11.650 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:12.201 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:22:12.282 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:12.290 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:12] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-10-09T06:22:12.290 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[212750]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.291 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[212750]: pgmap v730: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:12.291 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[220921]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.292 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[220921]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.292 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[220921]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.292 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[220921]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.292 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:12 smithi107 ceph-mon[220921]: pgmap v730: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:12.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:12.311 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:12.394 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:12.420 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:12.514 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:12.652 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:12 smithi150 ceph-mon[194178]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:12 smithi150 ceph-mon[194178]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:12 smithi150 ceph-mon[194178]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:12 smithi150 ceph-mon[194178]: failed to encode map e237 with expected crc 2023-10-09T06:22:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:12 smithi150 ceph-mon[194178]: pgmap v730: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:12.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:12.756 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:12.785 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:13.042 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:13.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:13.146 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:13.178 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:13.226 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:13.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:13.334 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:13.688 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:13.736 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:13.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:13.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:13.875 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:14.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:14.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:14.129 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:14.160 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:14.184 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:22:14.186 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_5' created 2023-10-09T06:22:14.212 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:14.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:14.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:14.317 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:14.344 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:14.387 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:14.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:14.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[212750]: pgmap v731: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3104880679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[220921]: pgmap v731: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3104880679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:14.495 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:14.518 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:14 smithi150 ceph-mon[194178]: pgmap v731: 83 pgs: 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:14.518 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3104880679' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.518 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T06:22:14.519 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:14.520 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:14.561 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:14.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:14.667 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:14.694 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:14.735 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:14.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:14.840 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:14.869 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:15.152 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:15.196 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:22:15.277 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:15.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:15.381 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:15.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:22:15.411 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:15.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:15.463 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:15.463 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2099887709' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: osdmap e238: 8 total, 8 up, 7 in 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: failed to encode map e238 with expected crc 2023-10-09T06:22:15.464 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:22:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:15.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:15.566 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:15.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:15.637 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:15.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:15.744 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:15.772 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:16.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T06:22:16.461 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: pgmap v733: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: pgmap v733: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:16 smithi107 ceph-mon[220921]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.471 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:16.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:16.485 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:22:16.576 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: pgmap v733: 99 pgs: 3 creating+peering, 13 unknown, 83 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2074695292' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: osdmap e239: 8 total, 8 up, 7 in 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:16 smithi150 ceph-mon[194178]: failed to encode map e239 with expected crc 2023-10-09T06:22:16.607 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:16.654 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:16.654 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:16.656 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:16.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:16.763 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:16.789 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:17.180 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:22:17.198 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:17.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:17.301 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:17.330 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:17.375 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:17.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:17.478 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:17.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:17 smithi107 ceph-mon[212750]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:22:17.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:17.503 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:17 smithi107 ceph-mon[220921]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:22:17.503 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:17.505 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:17.615 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:17.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:17.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:17 smithi150 ceph-mon[194178]: osdmap e240: 8 total, 8 up, 7 in 2023-10-09T06:22:17.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:17.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:22:17.719 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:17.729 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:17.769 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:17.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:17.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:17.873 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:17.963 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:18.003 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:18.052 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:18.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:18.112 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:18.163 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.180 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:18.200 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:18.201 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:18.223 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:18.241 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:18.244 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:18.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:18.267 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:18.286 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:18.309 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:18.329 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:18.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.330 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.331 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:22:18.349 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:18.379 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.379 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.379 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.380 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: failed to encode map e239 with expected crc 2023-10-09T06:22:18.380 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: pgmap v736: 99 pgs: 3 creating+peering, 9 unknown, 87 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:18.380 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3039524421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:18.381 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:18.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:18.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:18.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:18.473 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:18.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: failed to encode map e239 with expected crc 2023-10-09T06:22:18.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: pgmap v736: 99 pgs: 3 creating+peering, 9 unknown, 87 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3039524421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: failed to encode map e240 with expected crc 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: failed to encode map e239 with expected crc 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: pgmap v736: 99 pgs: 3 creating+peering, 9 unknown, 87 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:18.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3039524421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:18.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:18.567 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:18.610 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:18.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:18.715 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:18.748 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:18.796 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:18.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:18.897 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:18.928 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:19.576 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:19.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:19.680 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:19.716 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:19.765 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:19.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:19.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:19.871 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:19.902 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:19.946 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:19.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:20.052 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.080 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:20.122 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:20.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:20.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:20.228 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:20.260 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:20.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:20 smithi150 ceph-mon[194178]: pgmap v737: 99 pgs: 3 creating+peering, 96 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:20.404 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:20.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:20.508 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:20.535 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:20.578 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:20.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:20.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:20 smithi107 ceph-mon[212750]: pgmap v737: 99 pgs: 3 creating+peering, 96 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:20.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:20 smithi107 ceph-mon[220921]: pgmap v737: 99 pgs: 3 creating+peering, 96 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:20.684 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.711 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:20.752 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:20.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:20.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:20.859 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:20.886 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:20.928 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:20.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:21.033 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:21.065 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:21.518 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:21.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:21.539 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:22:21.623 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:21.652 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:21.840 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:21.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:21.943 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:21.976 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:22.025 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:22.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:22.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:22.132 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:22.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:22] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-10-09T06:22:22.160 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:22.203 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:22.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:22.213 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T06:22:22.218 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:22:22.255 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:22.311 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:22.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:22 smithi150 ceph-mon[194178]: pgmap v738: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:22.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:22.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/952777978' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.338 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:22.379 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:22.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:22.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:22.430 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:22:22.430 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:22:22.431 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:22:22.431 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:22:22.431 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:22:22.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[212750]: pgmap v738: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:22.454 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:22.454 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/952777978' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.454 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[220921]: pgmap v738: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:22.454 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:22.455 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/952777978' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.455 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:22:22.484 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:22.515 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:23.213 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:22:23.235 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2007086151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2007086151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.237 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:23 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.616 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:23.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/241665569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:22:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: osdmap e241: 8 total, 8 up, 7 in 2023-10-09T06:22:23.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:23.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2007086151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:23 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:22:23.720 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:23.752 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:23.801 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:23.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:23.904 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:22:23.909 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:23.936 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:23.980 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:23.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:24.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:24.115 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:24.441 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:22:24.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: pgmap v740: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:24.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:22:24.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:22:24.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.506 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.506 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: pgmap v740: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:24.507 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:22:24.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:22:24.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:24 smithi107 ceph-mon[220921]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.618 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:24.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e241 with expected crc 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: pgmap v740: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:22:24.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: osdmap e242: 8 total, 8 up, 7 in 2023-10-09T06:22:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:24 smithi150 ceph-mon[194178]: failed to encode map e242 with expected crc 2023-10-09T06:22:24.723 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.755 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:24.803 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:24.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:24.906 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:24.932 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:24.977 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:24.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:24.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:25.080 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.112 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:25.160 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:25.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:25.266 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:25.294 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:25.332 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:25.337 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:25.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:25.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:22:25.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:25 smithi150 ceph-mon[194178]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:25 smithi150 ceph-mon[194178]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:25 smithi150 ceph-mon[194178]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.445 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:25.470 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:25.512 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:25.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[212750]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[220921]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[220921]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.546 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:25 smithi107 ceph-mon[220921]: failed to encode map e242 with expected crc 2023-10-09T06:22:25.620 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:25.632 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T06:22:25.648 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:25.650 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:22:25.650 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:22:25.702 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:22:25.702 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:22:25.702 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_363200 2023-10-09T06:22:25.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:22:25.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:22:25.793 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:26.059 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:26.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:26.162 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:26.166 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:26.210 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:26.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:26 smithi107 ceph-mon[212750]: pgmap v742: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:26.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1947853906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:26.224 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:26.478 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:26.516 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:26.555 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:26.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:26.633 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:26.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:26 smithi107 ceph-mon[220921]: pgmap v742: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:26.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1947853906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:26.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:26 smithi150 ceph-mon[194178]: pgmap v742: 99 pgs: 99 active+clean; 378 KiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-10-09T06:22:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1947853906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:26.675 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:26.678 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.678 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.678 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.679 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.679 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.679 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.679 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.679 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:22:26.702 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2348 2332 145.741 145.75 0.0033664 0.00683932 2023-10-09T06:22:27.372 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:27.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:27.479 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:27.520 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:27.604 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:27.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:27.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 6026 6010 187.798 229.875 0.00279234 0.00531642 2023-10-09T06:22:27.712 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:27.743 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:27.784 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:27.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:27.891 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:27.946 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:27 smithi150 ceph-mon[194178]: pgmap v743: 99 pgs: 99 active+clean; 17 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-10-09T06:22:28.357 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:28.357 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:27 smithi107 ceph-mon[220921]: pgmap v743: 99 pgs: 99 active+clean; 17 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-10-09T06:22:28.358 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:28.358 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:27 smithi107 ceph-mon[212750]: pgmap v743: 99 pgs: 99 active+clean; 17 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-10-09T06:22:28.552 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:28.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:28.660 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:28.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 8934 8918 185.775 181.75 0.00342144 0.0053757 2023-10-09T06:22:28.747 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:28.838 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:28.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:28.948 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:28.997 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:29.072 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:29.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:29.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:29.178 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:29.204 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:29.609 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:29.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:29.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 11164 11148 174.171 139.375 0.00316932 0.00573647 2023-10-09T06:22:29.714 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:29.773 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:29.991 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:29.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:30.036 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:30.097 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:30 smithi107 ceph-mon[212750]: pgmap v744: 99 pgs: 99 active+clean; 231 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 461 op/s 2023-10-09T06:22:30.100 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:30.133 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:30.197 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:30.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:30.209 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:30.209 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:30.302 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:30.340 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:30.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:30 smithi107 ceph-mon[220921]: pgmap v744: 99 pgs: 99 active+clean; 231 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 461 op/s 2023-10-09T06:22:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:30 smithi150 ceph-mon[194178]: pgmap v744: 99 pgs: 99 active+clean; 231 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 29 MiB/s wr, 461 op/s 2023-10-09T06:22:30.415 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:30.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:30.523 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:30.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:30.703 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 13552 13536 169.18 149.25 0.00533503 0.005906 2023-10-09T06:22:31.110 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3405095245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.110 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.133 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:22:31.218 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:31.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:31.324 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:31.371 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3405095245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.371 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.377 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3405095245' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:22:31.529 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:22:31.706 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 15696 15680 163.238 134 0.00682641 0.00611892 2023-10-09T06:22:32.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:32] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-10-09T06:22:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: pgmap v745: 99 pgs: 99 active+clean; 536 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: pgmap v745: 99 pgs: 99 active+clean; 536 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:32.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: pgmap v745: 99 pgs: 99 active+clean; 536 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: osdmap e243: 8 total, 8 up, 7 in 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: failed to encode map e243 with expected crc 2023-10-09T06:22:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:32.706 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 17418 17402 155.294 107.625 0.00357806 0.00643556 2023-10-09T06:22:32.735 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:32.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:32.841 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:32.869 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:33.498 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:22:33.591 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:33.597 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:22:33.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:33.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:33.705 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.706 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 19285 19269 150.466 116.688 0.0100816 0.00663876 2023-10-09T06:22:33.741 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:33.805 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:33.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:33.910 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:33.943 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:34.013 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:34.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:34.074 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:34.120 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:34.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:34 smithi107 ceph-mon[220921]: pgmap v747: 99 pgs: 99 active+clean; 949 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:22:34.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:34 smithi107 ceph-mon[212750]: pgmap v747: 99 pgs: 99 active+clean; 949 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:22:34.184 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:34 smithi150 ceph-mon[194178]: pgmap v747: 99 pgs: 99 active+clean; 949 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-09T06:22:34.405 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:34.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:34.516 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:34.588 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:34.706 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 21141 21125 146.637 116 0.00412712 0.00681478 2023-10-09T06:22:34.744 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:34.844 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:34.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:34.950 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:35.003 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:35.088 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:35.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:35.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:35.156 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:22:35.194 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:35.254 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:35.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:22:35.707 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 23513 23497 146.785 148.25 0.00599932 0.00680766 2023-10-09T06:22:36.092 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:36.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:36.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:36.261 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:36.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: pgmap v748: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[212750]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: pgmap v748: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:22:36.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:36 smithi107 ceph-mon[220921]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: pgmap v748: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2780519739' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: osdmap e244: 8 total, 8 up, 7 in 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:36 smithi150 ceph-mon[194178]: failed to encode map e244 with expected crc 2023-10-09T06:22:36.707 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 25186 25170 142.947 104.562 0.00595337 0.00695125 2023-10-09T06:22:36.970 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:37.074 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:37.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:37.181 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:37.253 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:37.599 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:37.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:37.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:37.707 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:37.708 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 27034 27018 140.659 115.5 0.00691135 0.00710307 2023-10-09T06:22:37.781 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:37.830 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:37.875 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:37.912 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:37.950 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:37.985 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:38.019 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:38.054 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:38.092 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.097 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:22:38.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:38 smithi107 ceph-mon[212750]: pgmap v750: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:22:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2856365176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:38 smithi107 ceph-mon[220921]: pgmap v750: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:22:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2856365176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:38.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:38 smithi150 ceph-mon[194178]: pgmap v750: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T06:22:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2856365176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:38.708 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 29102 29086 139.781 129.25 0.00903701 0.00714851 2023-10-09T06:22:38.947 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:38.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:38.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:39.040 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:22:39.054 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:39.146 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:39.237 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:39.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:39.344 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:39.386 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:39.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:39.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:39.566 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:39.602 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:39.683 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:39.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:39.709 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 30551 30535 136.266 90.5625 0.00726012 0.00733281 2023-10-09T06:22:39.792 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:39.836 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:39.922 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:39.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:40.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:40.065 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:40.140 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:40.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:40.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:40.246 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:40.272 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:40.316 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:40.319 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:40 smithi150 ceph-mon[194178]: pgmap v751: 99 pgs: 99 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:22:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:40.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:40 smithi107 ceph-mon[212750]: pgmap v751: 99 pgs: 99 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:22:40.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:40 smithi107 ceph-mon[220921]: pgmap v751: 99 pgs: 99 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T06:22:40.422 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:40.458 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:40.538 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:40.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:40.645 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:40.711 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 32151 32135 133.838 100 0.00922931 0.00746409 2023-10-09T06:22:40.719 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:40.895 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:40.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:41.001 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:41.063 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:41.166 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:41.174 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:41.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:41.278 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:33147 10.2 10:40730c22:::benchmark_data_smithi107_363200_object33146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:22:41.327559+0000", 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0040680109999999998, 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0041204529999999996, 2023-10-09T06:22:41.333 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 33147 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327559+0000", 2023-10-09T06:22:41.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327559+0000", 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327557+0000", 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327582+0000", 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4403e-05 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327598+0000", 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6628000000000001e-05 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:22:41.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327608+0000", 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8199999999999992e-06 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327657+0000", 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8980000000000002e-05 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327811+0000", 2023-10-09T06:22:41.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000153967 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.327962+0000", 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015096700000000001 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:22:41.330146+0000", 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021840290000000001 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:22:41.338 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:22:41.339 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:22:41.339 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:22:41.339 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:22:41.339 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 1 2023-10-09T06:22:41.339 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:22:41.357 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:41.524 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:41.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:41.634 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:41.674 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:41.710 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 33765 33749 131.769 100.875 0.0089005 0.00758312 2023-10-09T06:22:41.745 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:41.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:41.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:41.853 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:41.913 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:42.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:42] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-10-09T06:22:42.158 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:22:42.531 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:42.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:42.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:42.541 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:42 smithi107 ceph-mon[212750]: pgmap v752: 99 pgs: 99 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:22:42.542 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:42.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:42 smithi107 ceph-mon[220921]: pgmap v752: 99 pgs: 99 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:22:42.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:42.640 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:42 smithi150 ceph-mon[194178]: pgmap v752: 99 pgs: 99 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:22:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:22:42.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:42.711 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 35308 35292 129.69 96.4375 0.00564931 0.00770569 2023-10-09T06:22:42.758 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:42.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:42.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:42.780 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:22:42.865 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:42.901 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:42.966 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:42.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:42.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:43.074 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:43.102 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:43.185 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:43.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:43.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:43.292 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:22:43.293 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:43.354 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: 1.8 scrub starts 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: 1.8 scrub ok 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:43 smithi150 ceph-mon[194178]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.528 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T06:22:43.551 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[212750]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: 1.8 scrub starts 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: 1.8 scrub ok 2023-10-09T06:22:43.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2057799817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:22:43.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: osdmap e245: 8 total, 8 up, 7 in 2023-10-09T06:22:43.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:43 smithi107 ceph-mon[220921]: failed to encode map e245 with expected crc 2023-10-09T06:22:43.557 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:22:43.686 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:43.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:43.711 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 36706 36690 127.34 87.375 0.0193611 0.00784899 2023-10-09T06:22:43.793 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:43.848 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:44.291 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:44.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:44.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:44.431 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:44 smithi107 ceph-mon[212750]: pgmap v754: 99 pgs: 99 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T06:22:44.431 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:44 smithi107 ceph-mon[220921]: pgmap v754: 99 pgs: 99 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T06:22:44.442 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:44 smithi150 ceph-mon[194178]: pgmap v754: 99 pgs: 99 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-09T06:22:44.447 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:44.508 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:44.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:44.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:44.573 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:44.617 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:44.654 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:44.664 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:44.710 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 37950 37934 124.73 77.75 0.0140308 0.00801032 2023-10-09T06:22:44.712 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:44.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:44.751 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:44.819 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:44.858 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:44.920 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:44.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:45.027 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:45.075 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:45.147 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:45.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:45.255 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:45.323 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:45.516 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:45.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:45.622 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:22:45.665 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:45.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:22:45.715647+0000 min lat: 0.00112444 max lat: 0.323912 avg lat: 0.00809703 2023-10-09T06:22:45.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:22:45.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 39533 39517 123.415 98.9375 0.00532114 0.00809703 2023-10-09T06:22:45.744 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:22:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:45.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:22:45.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:45.876 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:46.190 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:22:46.386 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:46.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:46.489 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:46.513 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:46 smithi150 ceph-mon[194178]: pgmap v755: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:22:46.513 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1102634702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.513 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.516 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:46.590 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:46.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:46.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[212750]: pgmap v755: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:22:46.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1102634702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[220921]: pgmap v755: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:22:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1102634702' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:22:46.696 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:46.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 40612 40596 120.751 67.4375 0.0146258 0.00827686 2023-10-09T06:22:46.778 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:47.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:47.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: osdmap e246: 8 total, 8 up, 7 in 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: failed to encode map e246 with expected crc 2023-10-09T06:22:47.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:47.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:22:47.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 15 41888 41873 118.89 79.8125 0.00884109 0.00840624 2023-10-09T06:22:48.130 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:48.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:48.244 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:48.270 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:48.319 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:48.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:48.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[212750]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[212750]: pgmap v757: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:22:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[220921]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[220921]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:48 smithi107 ceph-mon[220921]: pgmap v757: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:22:48.427 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:48.497 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:48 smithi150 ceph-mon[194178]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:48 smithi150 ceph-mon[194178]: failed to encode map e246 with expected crc 2023-10-09T06:22:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:48 smithi150 ceph-mon[194178]: pgmap v757: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T06:22:48.715 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 43026 43010 116.811 71.0625 0.0036726 0.00855635 2023-10-09T06:22:48.894 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:48.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:49.007 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:49.064 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:49.133 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:49.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:49.237 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:49.274 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:49.344 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:49.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:49.450 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:49.455 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:22:49.456 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:22:49.456 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:22:49.456 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:22:49.456 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:22:49.488 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:49.553 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:49.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:49.659 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:49.723 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 44476 44460 115.681 90.625 0.0104938 0.00863772 2023-10-09T06:22:49.731 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:49.848 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:49.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:49.915 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:22:49.951 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:22:49.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:49.987 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:22:50.014 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:50.032 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:22:50.053 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:22:50.078 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:50.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:50.083 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:22:50.120 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:22:50.159 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:22:50.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.169 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.170 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.170 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.170 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.170 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:22:50.183 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:50.228 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:22:50.250 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:50.614 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:50.618 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[220921]: pgmap v758: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:22:50.618 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/906025693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:50.618 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:50.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:50.619 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[212750]: pgmap v758: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:22:50.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/906025693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:50.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:50.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:50 smithi150 ceph-mon[194178]: pgmap v758: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T06:22:50.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/906025693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:22:50.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:22:50.720 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:50.723 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 45902 45886 114.619 89.125 0.0020969 0.00871532 2023-10-09T06:22:50.749 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:50.895 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:22:51.253 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[212750]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:22:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:51 smithi107 ceph-mon[220921]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4072534704' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: osdmap e247: 8 total, 8 up, 7 in 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:51 smithi150 ceph-mon[194178]: failed to encode map e247 with expected crc 2023-10-09T06:22:51.723 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 46929 46913 112.68 64.1875 0.00461754 0.00887023 2023-10-09T06:22:51.748 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:51.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:51.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:51.900 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:22:51.900 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:22:51.946 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:52.069 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:52.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:52.140 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:52.182 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:52.183 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:52.259 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:52.390 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:52.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:52 smithi107 ceph-mon[212750]: pgmap v760: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:22:52.406 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:22:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:22:52] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-10-09T06:22:52.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:52 smithi107 ceph-mon[220921]: pgmap v760: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:22:52.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:52.508 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:52.577 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:52 smithi150 ceph-mon[194178]: pgmap v760: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:22:52.724 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 48114 48098 111.251 74.0625 0.0118667 0.00898312 2023-10-09T06:22:52.810 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:52.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:52.916 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:52.983 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:53.065 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:53.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:53.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:53.227 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:53.251 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.379 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:53.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:22:53.484 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:53.512 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:53.725 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 49361 49345 110.056 77.9375 0.00931669 0.00908118 2023-10-09T06:22:53.962 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:22:53.967 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:53.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:54.073 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:54.101 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:54.171 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:54.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:54.254 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:54 smithi107 ceph-mon[212750]: pgmap v761: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T06:22:54.284 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:54.339 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:54 smithi150 ceph-mon[194178]: pgmap v761: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T06:22:54.499 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:54.515 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:54.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:54 smithi107 ceph-mon[220921]: pgmap v761: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T06:22:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:54.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:54.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:54.674 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:54.725 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 50543 50527 108.809 73.875 0.0172175 0.00918517 2023-10-09T06:22:54.745 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:54.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:54.856 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:54.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:54.976 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:54.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:55.081 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:22:55.152 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:55.155 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:55.725 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 51862 51846 107.93 82.4375 0.010797 0.00925077 2023-10-09T06:22:55.958 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:22:55.958 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:22:56.474 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:22:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:56.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:22:56.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:22:56.609 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:56 smithi107 ceph-mon[212750]: pgmap v762: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:22:56.609 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:56 smithi107 ceph-mon[220921]: pgmap v762: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:22:56.612 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:56.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:56 smithi150 ceph-mon[194178]: pgmap v762: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:22:56.725 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 52668 52652 106.075 50.375 0.00794071 0.00940328 2023-10-09T06:22:57.195 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:22:57.197 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:22:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:57.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:22:57.298 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:22:57.307 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:22:57.385 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:57.522 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:22:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:57.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:22:57.538 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/608610739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/608610739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.539 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.635 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:22:57.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/608610739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:22:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:22:57.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:57.725 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 53849 53833 105.067 73.8125 0.00868548 0.00951308 2023-10-09T06:22:57.752 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:22:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:57.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:22:57.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:22:57.936 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: pgmap v763: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: pgmap v763: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:58 smithi107 ceph-mon[220921]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: pgmap v763: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T06:22:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:22:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:22:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: osdmap e248: 8 total, 8 up, 7 in 2023-10-09T06:22:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:58 smithi150 ceph-mon[194178]: failed to encode map e248 with expected crc 2023-10-09T06:22:58.727 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 54839 54823 103.753 61.875 0.0739524 0.00962533 2023-10-09T06:22:59.129 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:22:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:59.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:22:59.234 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.275 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:59.342 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:59.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:22:59 smithi107 ceph-mon[212750]: failed to encode map e248 with expected crc 2023-10-09T06:22:59.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:22:59 smithi107 ceph-mon[220921]: failed to encode map e248 with expected crc 2023-10-09T06:22:59.448 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:22:59.484 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:22:59.495 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:59.567 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:22:59.571 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:22:59 smithi150 ceph-mon[194178]: failed to encode map e248 with expected crc 2023-10-09T06:22:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:59.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:22:59.673 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:59.708 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:59.730 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 56052 56036 102.924 75.8125 0.00904738 0.0097105 2023-10-09T06:22:59.778 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:59.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:59.820 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:22:59.829 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:22:59.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:22:59.924 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:22:59.980 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:22:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:22:59.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:22:59.998 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:00.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:00.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:00.098 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:00.127 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:00.154 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:00.169 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:00.210 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:00.253 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:00.285 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:00.291 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.292 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:23:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:00 smithi107 ceph-mon[212750]: pgmap v765: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:23:00.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/660907030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:00 smithi107 ceph-mon[220921]: pgmap v765: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:23:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/660907030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:00 smithi150 ceph-mon[194178]: pgmap v765: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:23:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/660907030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:00.731 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 57294 57278 102.199 77.625 0.00645272 0.00978062 2023-10-09T06:23:01.122 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:01.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:01.164 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:23:01.228 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:01.255 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:01.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:01.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:01 smithi107 ceph-mon[212750]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:01 smithi107 ceph-mon[220921]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:01.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:01.482 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:01 smithi150 ceph-mon[194178]: from='client.35808 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:01.731 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 58435 58419 101.341 71.3125 0.0109227 0.00986232 2023-10-09T06:23:01.807 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":765,"stamp":"2023-10-09T06:22:59.048751+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3470006176,"num_objects":52963,"num_object_clones":0,"num_object_copies":158889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53000,"num_write_kb":3388125,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":337988,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54765,"ondisk_log_size":54765,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14014196,"kb_used_data":10120444,"kb_used_omap":20,"kb_used_meta":3893675,"kb_avail":642201868,"statfs":{"total":671965249536,"available":657614712832,"internally_reserved":0,"allocated":10363334656,"data_stored":10356841218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":3987123854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":707264512,"num_objects":10792,"num_object_clones":0,"num_object_copies":32376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10792,"num_write_kb":690688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.832330"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":227,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225796+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:22:50.225796+0000","last_peered":"2023-10-09T06:22:50.225796+0000","last_clean":"2023-10-09T06:22:50.225796+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:22:50.225796+0000","last_undegraded":"2023-10-09T06:22:50.225796+0000","last_fullsized":"2023-10-09T06:22:50.225796+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222790+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:22:50.222790+0000","last_peered":"2023-10-09T06:22:50.222790+0000","last_clean":"2023-10-09T06:22:50.222790+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:22:50.222790+0000","last_undegraded":"2023-10-09T06:22:50.222790+0000","last_fullsized":"2023-10-09T06:22:50.222790+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":218,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.513910+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:22:46.513910+0000","last_peered":"2023-10-09T06:22:46.513910+0000","last_clean":"2023-10-09T06:22:46.513910+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:22:46.513910+0000","last_undegraded":"2023-10-09T06:22:46.513910+0000","last_fullsized":"2023-10-09T06:22:46.513910+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423917+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:22:46.423917+0000","last_peered":"2023-10-09T06:22:46.423917+0000","last_clean":"2023-10-09T06:22:46.423917+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:22:46.423917+0000","last_undegraded":"2023-10-09T06:22:46.423917+0000","last_fullsized":"2023-10-09T06:22:46.423917+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222543+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:22:50.222543+0000","last_peered":"2023-10-09T06:22:50.222543+0000","last_clean":"2023-10-09T06:22:50.222543+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:22:50.222543+0000","last_undegraded":"2023-10-09T06:22:50.222543+0000","last_fullsized":"2023-10-09T06:22:50.222543+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227999+0000","last_change":"2023-10-09T06:21:45.411474+0000","last_active":"2023-10-09T06:22:50.227999+0000","last_peered":"2023-10-09T06:22:50.227999+0000","last_clean":"2023-10-09T06:22:50.227999+0000","last_became_active":"2023-10-09T06:21:45.410968+0000","last_became_peered":"2023-10-09T06:21:45.410968+0000","last_unstale":"2023-10-09T06:22:50.227999+0000","last_undegraded":"2023-10-09T06:22:50.227999+0000","last_fullsized":"2023-10-09T06:22:50.227999+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:53.196763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":207,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423663+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:22:46.423663+0000","last_peered":"2023-10-09T06:22:46.423663+0000","last_clean":"2023-10-09T06:22:46.423663+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:22:46.423663+0000","last_undegraded":"2023-10-09T06:22:46.423663+0000","last_fullsized":"2023-10-09T06:22:46.423663+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222143+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:22:50.222143+0000","last_peered":"2023-10-09T06:22:50.222143+0000","last_clean":"2023-10-09T06:22:50.222143+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:22:50.222143+0000","last_undegraded":"2023-10-09T06:22:50.222143+0000","last_fullsized":"2023-10-09T06:22:50.222143+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228800+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:22:50.228800+0000","last_peered":"2023-10-09T06:22:50.228800+0000","last_clean":"2023-10-09T06:22:50.228800+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:22:50.228800+0000","last_undegraded":"2023-10-09T06:22:50.228800+0000","last_fullsized":"2023-10-09T06:22:50.228800+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":182,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222350+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:22:50.222350+0000","last_peered":"2023-10-09T06:22:50.222350+0000","last_clean":"2023-10-09T06:22:50.222350+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:22:50.222350+0000","last_undegraded":"2023-10-09T06:22:50.222350+0000","last_fullsized":"2023-10-09T06:22:50.222350+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":97,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229093+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:22:50.229093+0000","last_peered":"2023-10-09T06:22:50.229093+0000","last_clean":"2023-10-09T06:22:50.229093+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:22:50.229093+0000","last_undegraded":"2023-10-09T06:22:50.229093+0000","last_fullsized":"2023-10-09T06:22:50.229093+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":177,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514294+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:22:46.514294+0000","last_peered":"2023-10-09T06:22:46.514294+0000","last_clean":"2023-10-09T06:22:46.514294+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:22:46.514294+0000","last_undegraded":"2023-10-09T06:22:46.514294+0000","last_fullsized":"2023-10-09T06:22:46.514294+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":254,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226156+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:22:50.226156+0000","last_peered":"2023-10-09T06:22:50.226156+0000","last_clean":"2023-10-09T06:22:50.226156+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:22:50.226156+0000","last_undegraded":"2023-10-09T06:22:50.226156+0000","last_fullsized":"2023-10-09T06:22:50.226156+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226923+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:22:50.226923+0000","last_peered":"2023-10-09T06:22:50.226923+0000","last_clean":"2023-10-09T06:22:50.226923+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:22:50.226923+0000","last_undegraded":"2023-10-09T06:22:50.226923+0000","last_fullsized":"2023-10-09T06:22:50.226923+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":97,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228251+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:22:50.228251+0000","last_peered":"2023-10-09T06:22:50.228251+0000","last_clean":"2023-10-09T06:22:50.228251+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:22:50.228251+0000","last_undegraded":"2023-10-09T06:22:50.228251+0000","last_fullsized":"2023-10-09T06:22:50.228251+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229688+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:22:50.229688+0000","last_peered":"2023-10-09T06:22:50.229688+0000","last_clean":"2023-10-09T06:22:50.229688+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:22:50.229688+0000","last_undegraded":"2023-10-09T06:22:50.229688+0000","last_fullsized":"2023-10-09T06:22:50.229688+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":216,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423662+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:22:46.423662+0000","last_peered":"2023-10-09T06:22:46.423662+0000","last_clean":"2023-10-09T06:22:46.423662+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:22:46.423662+0000","last_undegraded":"2023-10-09T06:22:46.423662+0000","last_fullsized":"2023-10-09T06:22:46.423662+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":175,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228960+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:22:50.228960+0000","last_peered":"2023-10-09T06:22:50.228960+0000","last_clean":"2023-10-09T06:22:50.228960+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:22:50.228960+0000","last_undegraded":"2023-10-09T06:22:50.228960+0000","last_fullsized":"2023-10-09T06:22:50.228960+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228248+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:22:50.228248+0000","last_peered":"2023-10-09T06:22:50.228248+0000","last_clean":"2023-10-09T06:22:50.228248+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:22:50.228248+0000","last_undegraded":"2023-10-09T06:22:50.228248+0000","last_fullsized":"2023-10-09T06:22:50.228248+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423952+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:22:46.423952+0000","last_peered":"2023-10-09T06:22:46.423952+0000","last_clean":"2023-10-09T06:22:46.423952+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:22:46.423952+0000","last_undegraded":"2023-10-09T06:22:46.423952+0000","last_fullsized":"2023-10-09T06:22:46.423952+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.224216+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:22:50.224216+0000","last_peered":"2023-10-09T06:22:50.224216+0000","last_clean":"2023-10-09T06:22:50.224216+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:22:50.224216+0000","last_undegraded":"2023-10-09T06:22:50.224216+0000","last_fullsized":"2023-10-09T06:22:50.224216+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222686+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:22:50.222686+0000","last_peered":"2023-10-09T06:22:50.222686+0000","last_clean":"2023-10-09T06:22:50.222686+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:22:50.222686+0000","last_undegraded":"2023-10-09T06:22:50.222686+0000","last_fullsized":"2023-10-09T06:22:50.222686+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"247'3137","reported_seq":3155,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:54.985918+0000","last_change":"2023-10-09T06:22:15.195712+0000","last_active":"2023-10-09T06:22:54.985918+0000","last_peered":"2023-10-09T06:22:54.985918+0000","last_clean":"2023-10-09T06:22:54.985918+0000","last_became_active":"2023-10-09T06:22:15.195422+0000","last_became_peered":"2023-10-09T06:22:15.195422+0000","last_unstale":"2023-10-09T06:22:54.985918+0000","last_undegraded":"2023-10-09T06:22:54.985918+0000","last_fullsized":"2023-10-09T06:22:54.985918+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3137,"log_dups_size":0,"ondisk_log_size":3137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:32.535427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205586432,"num_objects":3137,"num_object_clones":0,"num_object_copies":9411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3137,"num_write_kb":200768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":95,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228636+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:22:50.228636+0000","last_peered":"2023-10-09T06:22:50.228636+0000","last_clean":"2023-10-09T06:22:50.228636+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:22:50.228636+0000","last_undegraded":"2023-10-09T06:22:50.228636+0000","last_fullsized":"2023-10-09T06:22:50.228636+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227948+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:22:50.227948+0000","last_peered":"2023-10-09T06:22:50.227948+0000","last_clean":"2023-10-09T06:22:50.227948+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:22:50.227948+0000","last_undegraded":"2023-10-09T06:22:50.227948+0000","last_fullsized":"2023-10-09T06:22:50.227948+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"248'3314","reported_seq":3332,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.841866+0000","last_change":"2023-10-09T06:22:15.412855+0000","last_active":"2023-10-09T06:22:57.841866+0000","last_peered":"2023-10-09T06:22:57.841866+0000","last_clean":"2023-10-09T06:22:57.841866+0000","last_became_active":"2023-10-09T06:22:15.412520+0000","last_became_peered":"2023-10-09T06:22:15.412520+0000","last_unstale":"2023-10-09T06:22:57.841866+0000","last_undegraded":"2023-10-09T06:22:57.841866+0000","last_fullsized":"2023-10-09T06:22:57.841866+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3314,"log_dups_size":0,"ondisk_log_size":3314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:49.054096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3314,"num_write_kb":212096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":95,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228721+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:22:50.228721+0000","last_peered":"2023-10-09T06:22:50.228721+0000","last_clean":"2023-10-09T06:22:50.228721+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:22:50.228721+0000","last_undegraded":"2023-10-09T06:22:50.228721+0000","last_fullsized":"2023-10-09T06:22:50.228721+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225938+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:22:50.225938+0000","last_peered":"2023-10-09T06:22:50.225938+0000","last_clean":"2023-10-09T06:22:50.225938+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:22:50.225938+0000","last_undegraded":"2023-10-09T06:22:50.225938+0000","last_fullsized":"2023-10-09T06:22:50.225938+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"247'3243","reported_seq":3261,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:55.043571+0000","last_change":"2023-10-09T06:22:15.411974+0000","last_active":"2023-10-09T06:22:55.043571+0000","last_peered":"2023-10-09T06:22:55.043571+0000","last_clean":"2023-10-09T06:22:55.043571+0000","last_became_active":"2023-10-09T06:22:15.411649+0000","last_became_peered":"2023-10-09T06:22:15.411649+0000","last_unstale":"2023-10-09T06:22:55.043571+0000","last_undegraded":"2023-10-09T06:22:55.043571+0000","last_fullsized":"2023-10-09T06:22:55.043571+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3243,"log_dups_size":0,"ondisk_log_size":3243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:27.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212533248,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3243,"num_write_kb":207552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":109,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228891+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:22:50.228891+0000","last_peered":"2023-10-09T06:22:50.228891+0000","last_clean":"2023-10-09T06:22:50.228891+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:22:50.228891+0000","last_undegraded":"2023-10-09T06:22:50.228891+0000","last_fullsized":"2023-10-09T06:22:50.228891+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":227,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227045+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:22:50.227045+0000","last_peered":"2023-10-09T06:22:50.227045+0000","last_clean":"2023-10-09T06:22:50.227045+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:22:50.227045+0000","last_undegraded":"2023-10-09T06:22:50.227045+0000","last_fullsized":"2023-10-09T06:22:50.227045+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"247'3246","reported_seq":3264,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:55.970128+0000","last_change":"2023-10-09T06:22:15.411313+0000","last_active":"2023-10-09T06:22:55.970128+0000","last_peered":"2023-10-09T06:22:55.970128+0000","last_clean":"2023-10-09T06:22:55.970128+0000","last_became_active":"2023-10-09T06:22:15.410917+0000","last_became_peered":"2023-10-09T06:22:15.410917+0000","last_unstale":"2023-10-09T06:22:55.970128+0000","last_undegraded":"2023-10-09T06:22:55.970128+0000","last_fullsized":"2023-10-09T06:22:55.970128+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3246,"log_dups_size":0,"ondisk_log_size":3246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:24.915932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3246,"num_write_kb":207744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229575+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:22:50.229575+0000","last_peered":"2023-10-09T06:22:50.229575+0000","last_clean":"2023-10-09T06:22:50.229575+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:22:50.229575+0000","last_undegraded":"2023-10-09T06:22:50.229575+0000","last_fullsized":"2023-10-09T06:22:50.229575+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":198,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.422917+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:22:46.422917+0000","last_peered":"2023-10-09T06:22:46.422917+0000","last_clean":"2023-10-09T06:22:46.422917+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:22:46.422917+0000","last_undegraded":"2023-10-09T06:22:46.422917+0000","last_fullsized":"2023-10-09T06:22:46.422917+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"247'3239","reported_seq":3257,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:56.660476+0000","last_change":"2023-10-09T06:22:15.196669+0000","last_active":"2023-10-09T06:22:56.660476+0000","last_peered":"2023-10-09T06:22:56.660476+0000","last_clean":"2023-10-09T06:22:56.660476+0000","last_became_active":"2023-10-09T06:22:15.196399+0000","last_became_peered":"2023-10-09T06:22:15.196399+0000","last_unstale":"2023-10-09T06:22:56.660476+0000","last_undegraded":"2023-10-09T06:22:56.660476+0000","last_fullsized":"2023-10-09T06:22:56.660476+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3239,"log_dups_size":0,"ondisk_log_size":3239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.953962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212271104,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3239,"num_write_kb":207296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":540,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228110+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:22:50.228110+0000","last_peered":"2023-10-09T06:22:50.228110+0000","last_clean":"2023-10-09T06:22:50.228110+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:22:50.228110+0000","last_undegraded":"2023-10-09T06:22:50.228110+0000","last_fullsized":"2023-10-09T06:22:50.228110+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222891+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:22:50.222891+0000","last_peered":"2023-10-09T06:22:50.222891+0000","last_clean":"2023-10-09T06:22:50.222891+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:22:50.222891+0000","last_undegraded":"2023-10-09T06:22:50.222891+0000","last_fullsized":"2023-10-09T06:22:50.222891+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226256+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:22:50.226256+0000","last_peered":"2023-10-09T06:22:50.226256+0000","last_clean":"2023-10-09T06:22:50.226256+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:22:50.226256+0000","last_undegraded":"2023-10-09T06:22:50.226256+0000","last_fullsized":"2023-10-09T06:22:50.226256+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"247'3243","reported_seq":3260,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:55.967746+0000","last_change":"2023-10-09T06:22:15.411262+0000","last_active":"2023-10-09T06:22:55.967746+0000","last_peered":"2023-10-09T06:22:55.967746+0000","last_clean":"2023-10-09T06:22:55.967746+0000","last_became_active":"2023-10-09T06:22:15.410925+0000","last_became_peered":"2023-10-09T06:22:15.410925+0000","last_unstale":"2023-10-09T06:22:55.967746+0000","last_undegraded":"2023-10-09T06:22:55.967746+0000","last_fullsized":"2023-10-09T06:22:55.967746+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3243,"log_dups_size":0,"ondisk_log_size":3243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:51.768992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212533248,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3243,"num_write_kb":207552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":529,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.428714+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:22:46.428714+0000","last_peered":"2023-10-09T06:22:46.428714+0000","last_clean":"2023-10-09T06:22:46.428714+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:22:46.428714+0000","last_undegraded":"2023-10-09T06:22:46.428714+0000","last_fullsized":"2023-10-09T06:22:46.428714+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226802+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:22:50.226802+0000","last_peered":"2023-10-09T06:22:50.226802+0000","last_clean":"2023-10-09T06:22:50.226802+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:22:50.226802+0000","last_undegraded":"2023-10-09T06:22:50.226802+0000","last_fullsized":"2023-10-09T06:22:50.226802+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226424+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:22:50.226424+0000","last_peered":"2023-10-09T06:22:50.226424+0000","last_clean":"2023-10-09T06:22:50.226424+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:22:50.226424+0000","last_undegraded":"2023-10-09T06:22:50.226424+0000","last_fullsized":"2023-10-09T06:22:50.226424+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"248'3363","reported_seq":3381,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.845268+0000","last_change":"2023-10-09T06:22:15.196292+0000","last_active":"2023-10-09T06:22:57.845268+0000","last_peered":"2023-10-09T06:22:57.845268+0000","last_clean":"2023-10-09T06:22:57.845268+0000","last_became_active":"2023-10-09T06:22:15.195400+0000","last_became_peered":"2023-10-09T06:22:15.195400+0000","last_unstale":"2023-10-09T06:22:57.845268+0000","last_undegraded":"2023-10-09T06:22:57.845268+0000","last_fullsized":"2023-10-09T06:22:57.845268+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3363,"log_dups_size":0,"ondisk_log_size":3363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:20.452846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220397568,"num_objects":3363,"num_object_clones":0,"num_object_copies":10089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3363,"num_write_kb":215232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225752+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:22:50.225752+0000","last_peered":"2023-10-09T06:22:50.225752+0000","last_clean":"2023-10-09T06:22:50.225752+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:22:50.225752+0000","last_undegraded":"2023-10-09T06:22:50.225752+0000","last_fullsized":"2023-10-09T06:22:50.225752+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226249+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:22:50.226249+0000","last_peered":"2023-10-09T06:22:50.226249+0000","last_clean":"2023-10-09T06:22:50.226249+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:22:50.226249+0000","last_undegraded":"2023-10-09T06:22:50.226249+0000","last_fullsized":"2023-10-09T06:22:50.226249+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"248'3440","reported_seq":3458,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.864173+0000","last_change":"2023-10-09T06:22:15.195487+0000","last_active":"2023-10-09T06:22:57.864173+0000","last_peered":"2023-10-09T06:22:57.864173+0000","last_clean":"2023-10-09T06:22:57.864173+0000","last_became_active":"2023-10-09T06:22:15.195089+0000","last_became_peered":"2023-10-09T06:22:15.195089+0000","last_unstale":"2023-10-09T06:22:57.864173+0000","last_undegraded":"2023-10-09T06:22:57.864173+0000","last_fullsized":"2023-10-09T06:22:57.864173+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3440,"log_dups_size":0,"ondisk_log_size":3440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:34.432191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225443840,"num_objects":3440,"num_object_clones":0,"num_object_copies":10320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3440,"num_write_kb":220160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":525,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.431579+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:22:46.431579+0000","last_peered":"2023-10-09T06:22:46.431579+0000","last_clean":"2023-10-09T06:22:46.431579+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:22:46.431579+0000","last_undegraded":"2023-10-09T06:22:46.431579+0000","last_fullsized":"2023-10-09T06:22:46.431579+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222157+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:22:50.222157+0000","last_peered":"2023-10-09T06:22:50.222157+0000","last_clean":"2023-10-09T06:22:50.222157+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:22:50.222157+0000","last_undegraded":"2023-10-09T06:22:50.222157+0000","last_fullsized":"2023-10-09T06:22:50.222157+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":177,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514669+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:22:46.514669+0000","last_peered":"2023-10-09T06:22:46.514669+0000","last_clean":"2023-10-09T06:22:46.514669+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:22:46.514669+0000","last_undegraded":"2023-10-09T06:22:46.514669+0000","last_fullsized":"2023-10-09T06:22:46.514669+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"248'3280","reported_seq":3297,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:58.067729+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:22:58.067729+0000","last_peered":"2023-10-09T06:22:58.067729+0000","last_clean":"2023-10-09T06:22:58.067729+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:22:58.067729+0000","last_undegraded":"2023-10-09T06:22:58.067729+0000","last_fullsized":"2023-10-09T06:22:58.067729+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3280,"log_dups_size":0,"ondisk_log_size":3280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3280,"num_write_kb":209920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228084+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:22:50.228084+0000","last_peered":"2023-10-09T06:22:50.228084+0000","last_clean":"2023-10-09T06:22:50.228084+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:22:50.228084+0000","last_undegraded":"2023-10-09T06:22:50.228084+0000","last_fullsized":"2023-10-09T06:22:50.228084+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":228,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.518601+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:22:46.518601+0000","last_peered":"2023-10-09T06:22:46.518601+0000","last_clean":"2023-10-09T06:22:46.518601+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:22:46.518601+0000","last_undegraded":"2023-10-09T06:22:46.518601+0000","last_fullsized":"2023-10-09T06:22:46.518601+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222874+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:22:50.222874+0000","last_peered":"2023-10-09T06:22:50.222874+0000","last_clean":"2023-10-09T06:22:50.222874+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:22:50.222874+0000","last_undegraded":"2023-10-09T06:22:50.222874+0000","last_fullsized":"2023-10-09T06:22:50.222874+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":171,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514015+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:22:46.514015+0000","last_peered":"2023-10-09T06:22:46.514015+0000","last_clean":"2023-10-09T06:22:46.514015+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:22:46.514015+0000","last_undegraded":"2023-10-09T06:22:46.514015+0000","last_fullsized":"2023-10-09T06:22:46.514015+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"248'3394","reported_seq":3412,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.702931+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:22:57.702931+0000","last_peered":"2023-10-09T06:22:57.702931+0000","last_clean":"2023-10-09T06:22:57.702931+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:22:57.702931+0000","last_undegraded":"2023-10-09T06:22:57.702931+0000","last_fullsized":"2023-10-09T06:22:57.702931+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3394,"log_dups_size":0,"ondisk_log_size":3394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222429184,"num_objects":3394,"num_object_clones":0,"num_object_copies":10182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3394,"num_write_kb":217216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":530,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423009+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:22:46.423009+0000","last_peered":"2023-10-09T06:22:46.423009+0000","last_clean":"2023-10-09T06:22:46.423009+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:22:46.423009+0000","last_undegraded":"2023-10-09T06:22:46.423009+0000","last_fullsized":"2023-10-09T06:22:46.423009+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514126+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:22:46.514126+0000","last_peered":"2023-10-09T06:22:46.514126+0000","last_clean":"2023-10-09T06:22:46.514126+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:22:46.514126+0000","last_undegraded":"2023-10-09T06:22:46.514126+0000","last_fullsized":"2023-10-09T06:22:46.514126+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226806+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:22:50.226806+0000","last_peered":"2023-10-09T06:22:50.226806+0000","last_clean":"2023-10-09T06:22:50.226806+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:22:50.226806+0000","last_undegraded":"2023-10-09T06:22:50.226806+0000","last_fullsized":"2023-10-09T06:22:50.226806+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":192,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514556+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:22:46.514556+0000","last_peered":"2023-10-09T06:22:46.514556+0000","last_clean":"2023-10-09T06:22:46.514556+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:22:46.514556+0000","last_undegraded":"2023-10-09T06:22:46.514556+0000","last_fullsized":"2023-10-09T06:22:46.514556+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"247'3289","reported_seq":3307,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:55.980628+0000","last_change":"2023-10-09T06:22:15.195697+0000","last_active":"2023-10-09T06:22:55.980628+0000","last_peered":"2023-10-09T06:22:55.980628+0000","last_clean":"2023-10-09T06:22:55.980628+0000","last_became_active":"2023-10-09T06:22:15.195391+0000","last_became_peered":"2023-10-09T06:22:15.195391+0000","last_unstale":"2023-10-09T06:22:55.980628+0000","last_undegraded":"2023-10-09T06:22:55.980628+0000","last_fullsized":"2023-10-09T06:22:55.980628+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3289,"log_dups_size":0,"ondisk_log_size":3289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:00.178768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215547904,"num_objects":3289,"num_object_clones":0,"num_object_copies":9867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3289,"num_write_kb":210496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":540,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228134+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:22:50.228134+0000","last_peered":"2023-10-09T06:22:50.228134+0000","last_clean":"2023-10-09T06:22:50.228134+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:22:50.228134+0000","last_undegraded":"2023-10-09T06:22:50.228134+0000","last_fullsized":"2023-10-09T06:22:50.228134+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.429276+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:22:46.429276+0000","last_peered":"2023-10-09T06:22:46.429276+0000","last_clean":"2023-10-09T06:22:46.429276+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:22:46.429276+0000","last_undegraded":"2023-10-09T06:22:46.429276+0000","last_fullsized":"2023-10-09T06:22:46.429276+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":218,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514279+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:22:46.514279+0000","last_peered":"2023-10-09T06:22:46.514279+0000","last_clean":"2023-10-09T06:22:46.514279+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:22:46.514279+0000","last_undegraded":"2023-10-09T06:22:46.514279+0000","last_fullsized":"2023-10-09T06:22:46.514279+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":214,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.517344+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:22:46.517344+0000","last_peered":"2023-10-09T06:22:46.517344+0000","last_clean":"2023-10-09T06:22:46.517344+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:22:46.517344+0000","last_undegraded":"2023-10-09T06:22:46.517344+0000","last_fullsized":"2023-10-09T06:22:46.517344+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"248'3366","reported_seq":3384,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.735851+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:22:57.735851+0000","last_peered":"2023-10-09T06:22:57.735851+0000","last_clean":"2023-10-09T06:22:57.735851+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:22:57.735851+0000","last_undegraded":"2023-10-09T06:22:57.735851+0000","last_fullsized":"2023-10-09T06:22:57.735851+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3366,"log_dups_size":0,"ondisk_log_size":3366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3366,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":178,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.431629+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:22:46.431629+0000","last_peered":"2023-10-09T06:22:46.431629+0000","last_clean":"2023-10-09T06:22:46.431629+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:22:46.431629+0000","last_undegraded":"2023-10-09T06:22:46.431629+0000","last_fullsized":"2023-10-09T06:22:46.431629+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":521,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228007+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:22:50.228007+0000","last_peered":"2023-10-09T06:22:50.228007+0000","last_clean":"2023-10-09T06:22:50.228007+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:22:50.228007+0000","last_undegraded":"2023-10-09T06:22:50.228007+0000","last_fullsized":"2023-10-09T06:22:50.228007+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":174,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294971+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:22:57.294971+0000","last_peered":"2023-10-09T06:22:57.294971+0000","last_clean":"2023-10-09T06:22:57.294971+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:22:57.294971+0000","last_undegraded":"2023-10-09T06:22:57.294971+0000","last_fullsized":"2023-10-09T06:22:57.294971+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227559+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:22:50.227559+0000","last_peered":"2023-10-09T06:22:50.227559+0000","last_clean":"2023-10-09T06:22:50.227559+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:22:50.227559+0000","last_undegraded":"2023-10-09T06:22:50.227559+0000","last_fullsized":"2023-10-09T06:22:50.227559+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"247'3163","reported_seq":3180,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:55.053311+0000","last_change":"2023-10-09T06:22:15.192849+0000","last_active":"2023-10-09T06:22:55.053311+0000","last_peered":"2023-10-09T06:22:55.053311+0000","last_clean":"2023-10-09T06:22:55.053311+0000","last_became_active":"2023-10-09T06:22:15.192540+0000","last_became_peered":"2023-10-09T06:22:15.192540+0000","last_unstale":"2023-10-09T06:22:55.053311+0000","last_undegraded":"2023-10-09T06:22:55.053311+0000","last_fullsized":"2023-10-09T06:22:55.053311+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3163,"log_dups_size":0,"ondisk_log_size":3163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:21.157073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207290368,"num_objects":3163,"num_object_clones":0,"num_object_copies":9489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3163,"num_write_kb":202432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":528,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514451+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:22:46.514451+0000","last_peered":"2023-10-09T06:22:46.514451+0000","last_clean":"2023-10-09T06:22:46.514451+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:22:46.514451+0000","last_undegraded":"2023-10-09T06:22:46.514451+0000","last_fullsized":"2023-10-09T06:22:46.514451+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227894+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:22:50.227894+0000","last_peered":"2023-10-09T06:22:50.227894+0000","last_clean":"2023-10-09T06:22:50.227894+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:22:50.227894+0000","last_undegraded":"2023-10-09T06:22:50.227894+0000","last_fullsized":"2023-10-09T06:22:50.227894+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":199,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.062255+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:22:50.062255+0000","last_peered":"2023-10-09T06:22:50.062255+0000","last_clean":"2023-10-09T06:22:50.062255+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:22:50.062255+0000","last_undegraded":"2023-10-09T06:22:50.062255+0000","last_fullsized":"2023-10-09T06:22:50.062255+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":221,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423905+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:22:46.423905+0000","last_peered":"2023-10-09T06:22:46.423905+0000","last_clean":"2023-10-09T06:22:46.423905+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:22:46.423905+0000","last_undegraded":"2023-10-09T06:22:46.423905+0000","last_fullsized":"2023-10-09T06:22:46.423905+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"248'3448","reported_seq":3466,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.711121+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:22:57.711121+0000","last_peered":"2023-10-09T06:22:57.711121+0000","last_clean":"2023-10-09T06:22:57.711121+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:22:57.711121+0000","last_undegraded":"2023-10-09T06:22:57.711121+0000","last_fullsized":"2023-10-09T06:22:57.711121+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3448,"log_dups_size":0,"ondisk_log_size":3448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225968128,"num_objects":3448,"num_object_clones":0,"num_object_copies":10344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3448,"num_write_kb":220672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227884+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:22:50.227884+0000","last_peered":"2023-10-09T06:22:50.227884+0000","last_clean":"2023-10-09T06:22:50.227884+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:22:50.227884+0000","last_undegraded":"2023-10-09T06:22:50.227884+0000","last_fullsized":"2023-10-09T06:22:50.227884+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229022+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:22:50.229022+0000","last_peered":"2023-10-09T06:22:50.229022+0000","last_clean":"2023-10-09T06:22:50.229022+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:22:50.229022+0000","last_undegraded":"2023-10-09T06:22:50.229022+0000","last_fullsized":"2023-10-09T06:22:50.229022+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":227,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.514558+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:22:46.514558+0000","last_peered":"2023-10-09T06:22:46.514558+0000","last_clean":"2023-10-09T06:22:46.514558+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:22:46.514558+0000","last_undegraded":"2023-10-09T06:22:46.514558+0000","last_fullsized":"2023-10-09T06:22:46.514558+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.062309+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:22:50.062309+0000","last_peered":"2023-10-09T06:22:50.062309+0000","last_clean":"2023-10-09T06:22:50.062309+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:22:50.062309+0000","last_undegraded":"2023-10-09T06:22:50.062309+0000","last_fullsized":"2023-10-09T06:22:50.062309+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"248'3353","reported_seq":3371,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.841965+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:22:57.841965+0000","last_peered":"2023-10-09T06:22:57.841965+0000","last_clean":"2023-10-09T06:22:57.841965+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:22:57.841965+0000","last_undegraded":"2023-10-09T06:22:57.841965+0000","last_fullsized":"2023-10-09T06:22:57.841965+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3353,"log_dups_size":0,"ondisk_log_size":3353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3353,"num_write_kb":214592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":531,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226275+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:22:50.226275+0000","last_peered":"2023-10-09T06:22:50.226275+0000","last_clean":"2023-10-09T06:22:50.226275+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:22:50.226275+0000","last_undegraded":"2023-10-09T06:22:50.226275+0000","last_fullsized":"2023-10-09T06:22:50.226275+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222690+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:22:50.222690+0000","last_peered":"2023-10-09T06:22:50.222690+0000","last_clean":"2023-10-09T06:22:50.222690+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:22:50.222690+0000","last_undegraded":"2023-10-09T06:22:50.222690+0000","last_fullsized":"2023-10-09T06:22:50.222690+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":204,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.518516+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:22:46.518516+0000","last_peered":"2023-10-09T06:22:46.518516+0000","last_clean":"2023-10-09T06:22:46.518516+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:22:46.518516+0000","last_undegraded":"2023-10-09T06:22:46.518516+0000","last_fullsized":"2023-10-09T06:22:46.518516+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222018+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:22:50.222018+0000","last_peered":"2023-10-09T06:22:50.222018+0000","last_clean":"2023-10-09T06:22:50.222018+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:22:50.222018+0000","last_undegraded":"2023-10-09T06:22:50.222018+0000","last_fullsized":"2023-10-09T06:22:50.222018+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"248'3421","reported_seq":3439,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.841832+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:22:57.841832+0000","last_peered":"2023-10-09T06:22:57.841832+0000","last_clean":"2023-10-09T06:22:57.841832+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:22:57.841832+0000","last_undegraded":"2023-10-09T06:22:57.841832+0000","last_fullsized":"2023-10-09T06:22:57.841832+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3421,"log_dups_size":0,"ondisk_log_size":3421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198656,"num_objects":3421,"num_object_clones":0,"num_object_copies":10263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3421,"num_write_kb":218944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":550,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229808+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:22:50.229808+0000","last_peered":"2023-10-09T06:22:50.229808+0000","last_clean":"2023-10-09T06:22:50.229808+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:22:50.229808+0000","last_undegraded":"2023-10-09T06:22:50.229808+0000","last_fullsized":"2023-10-09T06:22:50.229808+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":232,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228395+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:22:50.228395+0000","last_peered":"2023-10-09T06:22:50.228395+0000","last_clean":"2023-10-09T06:22:50.228395+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:22:50.228395+0000","last_undegraded":"2023-10-09T06:22:50.228395+0000","last_fullsized":"2023-10-09T06:22:50.228395+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":202,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423857+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:22:46.423857+0000","last_peered":"2023-10-09T06:22:46.423857+0000","last_clean":"2023-10-09T06:22:46.423857+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:22:46.423857+0000","last_undegraded":"2023-10-09T06:22:46.423857+0000","last_fullsized":"2023-10-09T06:22:46.423857+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":201,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225857+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:22:50.225857+0000","last_peered":"2023-10-09T06:22:50.225857+0000","last_clean":"2023-10-09T06:22:50.225857+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:22:50.225857+0000","last_undegraded":"2023-10-09T06:22:50.225857+0000","last_fullsized":"2023-10-09T06:22:50.225857+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":204,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.423017+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:22:46.423017+0000","last_peered":"2023-10-09T06:22:46.423017+0000","last_clean":"2023-10-09T06:22:46.423017+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:22:46.423017+0000","last_undegraded":"2023-10-09T06:22:46.423017+0000","last_fullsized":"2023-10-09T06:22:46.423017+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:46.430979+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:22:46.430979+0000","last_peered":"2023-10-09T06:22:46.430979+0000","last_clean":"2023-10-09T06:22:46.430979+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:22:46.430979+0000","last_undegraded":"2023-10-09T06:22:46.430979+0000","last_fullsized":"2023-10-09T06:22:46.430979+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":53,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226055+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:22:50.226055+0000","last_peered":"2023-10-09T06:22:50.226055+0000","last_clean":"2023-10-09T06:22:50.226055+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:22:50.226055+0000","last_undegraded":"2023-10-09T06:22:50.226055+0000","last_fullsized":"2023-10-09T06:22:50.226055+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226721+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:22:50.226721+0000","last_peered":"2023-10-09T06:22:50.226721+0000","last_clean":"2023-10-09T06:22:50.226721+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:22:50.226721+0000","last_undegraded":"2023-10-09T06:22:50.226721+0000","last_fullsized":"2023-10-09T06:22:50.226721+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":53,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226417+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:22:50.226417+0000","last_peered":"2023-10-09T06:22:50.226417+0000","last_clean":"2023-10-09T06:22:50.226417+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:22:50.226417+0000","last_undegraded":"2023-10-09T06:22:50.226417+0000","last_fullsized":"2023-10-09T06:22:50.226417+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225937+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:22:50.225937+0000","last_peered":"2023-10-09T06:22:50.225937+0000","last_clean":"2023-10-09T06:22:50.225937+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:22:50.225937+0000","last_undegraded":"2023-10-09T06:22:50.225937+0000","last_fullsized":"2023-10-09T06:22:50.225937+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":205,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222592+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:22:50.222592+0000","last_peered":"2023-10-09T06:22:50.222592+0000","last_clean":"2023-10-09T06:22:50.222592+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:22:50.222592+0000","last_undegraded":"2023-10-09T06:22:50.222592+0000","last_fullsized":"2023-10-09T06:22:50.222592+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222588+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:22:50.222588+0000","last_peered":"2023-10-09T06:22:50.222588+0000","last_clean":"2023-10-09T06:22:50.222588+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:22:50.222588+0000","last_undegraded":"2023-10-09T06:22:50.222588+0000","last_fullsized":"2023-10-09T06:22:50.222588+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":218,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.062384+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:22:50.062384+0000","last_peered":"2023-10-09T06:22:50.062384+0000","last_clean":"2023-10-09T06:22:50.062384+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:22:50.062384+0000","last_undegraded":"2023-10-09T06:22:50.062384+0000","last_fullsized":"2023-10-09T06:22:50.062384+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225617+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:22:50.225617+0000","last_peered":"2023-10-09T06:22:50.225617+0000","last_clean":"2023-10-09T06:22:50.225617+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:22:50.225617+0000","last_undegraded":"2023-10-09T06:22:50.225617+0000","last_fullsized":"2023-10-09T06:22:50.225617+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3469410304,"num_objects":52939,"num_object_clones":0,"num_object_copies":158817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":52939,"num_write_kb":3388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10420748288,"data_stored":10420748288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52939,"ondisk_log_size":52939,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234308,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1767864,"kb_used_data":867632,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":91977288,"statfs":{"total":95995035648,"available":94184742912,"internally_reserved":0,"allocated":888455168,"data_stored":887746858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:40 2023","interfaces":[{"interface":"back","average":{"1min":16.815000000000001,"5min":4.4530000000000003,"15min":2.4119999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":219.41900000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.496},{"interface":"front","average":{"1min":16.806000000000001,"5min":4.6040000000000001,"15min":2.4670000000000001},"min":{"1min":0.36399999999999999,"5min":0.218,"15min":0.218},"max":{"1min":219.399,"5min":219.399,"15min":219.399},"last":0.97299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:14 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":2.8599999999999999,"15min":2.1920000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.5390000000000001,"5min":31.869,"15min":31.869},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.8979999999999999,"5min":2.3690000000000002,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1059999999999999,"5min":29.486999999999998,"15min":29.486999999999998},"last":1.893}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":11.65,"5min":3.5049999999999999,"15min":2.5230000000000001},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":215.34800000000001,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":11.436,"5min":3.5659999999999998,"15min":2.6160000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":215.869,"5min":215.869,"15min":215.869},"last":0.53400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:21:56 2023","interfaces":[{"interface":"back","average":{"1min":5.375,"5min":2.8940000000000001,"15min":1.3560000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":50.966999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":2.5760000000000001},{"interface":"front","average":{"1min":5.8470000000000004,"5min":2.988,"15min":1.4119999999999999},"min":{"1min":0.47399999999999998,"5min":0.312,"15min":0.312},"max":{"1min":51.133000000000003,"5min":51.133000000000003,"15min":51.133000000000003},"last":4.0670000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:21:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.506,"15min":1.3580000000000001},"min":{"1min":0.30599999999999999,"5min":0.28299999999999997,"15min":0.251},"max":{"1min":4.7690000000000001,"5min":25.678999999999998,"15min":25.678999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.1200000000000001,"5min":1.361,"15min":1.2270000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":7.4779999999999998,"5min":13.305,"15min":13.305},"last":1.004}]},{"osd":5,"last update":"Mon Oct 9 06:21:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.3759999999999999,"15min":1.7549999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.9649999999999999,"5min":50.304000000000002,"15min":50.304000000000002},"last":1.1899999999999999},{"interface":"front","average":{"1min":1.54,"5min":1.9590000000000001,"15min":1.3899999999999999},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":4.8319999999999999,"5min":48.774999999999999,"15min":48.774999999999999},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:21:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":1.387,"15min":1.347},"min":{"1min":0.49399999999999999,"5min":0.35999999999999999,"15min":0.29399999999999998},"max":{"1min":8.4890000000000008,"5min":8.4890000000000008,"15min":11.878},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.4159999999999999,"15min":1.3169999999999999},"min":{"1min":0.42399999999999999,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":9.4900000000000002,"5min":12.141999999999999,"15min":12.141999999999999},"last":1.03}]}]},{"osd":6,"up_from":46,"seq":197568495945,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110700,"kb_used_data":1516836,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":91634452,"statfs":{"total":95995035648,"available":93833678848,"internally_reserved":0,"allocated":1553240064,"data_stored":1552544406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":5.2110000000000003,"15min":2.8119999999999998},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.214},"max":{"1min":21.555,"5min":110.926,"15min":110.926},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.3100000000000001,"5min":5.7409999999999997,"15min":3.0920000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.24399999999999999},"max":{"1min":21.535,"5min":110.911,"15min":110.911},"last":0.53300000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":5.04,"15min":3.25},"min":{"1min":0.377,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6639999999999997,"5min":114.202,"15min":114.202},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.0669999999999999,"5min":4.6289999999999996,"15min":3.141},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.246},"max":{"1min":4.2679999999999998,"5min":114.179,"15min":114.179},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.008,"15min":2.2770000000000001},"min":{"1min":0.42799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.6120000000000001,"5min":26.306999999999999,"15min":26.306999999999999},"last":75.191999999999993},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4139999999999999,"15min":2.1640000000000001},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.1050000000000004,"5min":26.292999999999999,"15min":26.292999999999999},"last":75.120000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":3.8580000000000001,"15min":1.6459999999999999},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.387,"5min":110.358,"15min":110.358},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.95399999999999996,"5min":4.0789999999999997,"15min":1.734},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.278,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.505}]},{"osd":4,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.415,"15min":1.1870000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":7.2060000000000004,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.111,"15min":1.1279999999999999},"min":{"1min":0.318,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.30199999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.73,"15min":1.306},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":22.379999999999999,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.4590000000000001,"15min":1.3049999999999999},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":24.401,"5min":24.401,"15min":24.401},"last":0.34399999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:22:00 2023","interfaces":[{"interface":"back","average":{"1min":6.9850000000000003,"5min":2.222,"15min":1.524},"min":{"1min":0.42499999999999999,"5min":0.316,"15min":0.26200000000000001},"max":{"1min":100.55800000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":7.1360000000000001,"5min":2.3319999999999999,"15min":1.756},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":100.995,"5min":100.995,"15min":100.995},"last":0.375}]}]},{"osd":1,"up_from":110,"seq":472446402681,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1237320,"kb_used_data":836096,"kb_used_omap":4,"kb_used_meta":401211,"kb_avail":92507832,"statfs":{"total":95995035648,"available":94728019968,"internally_reserved":0,"allocated":856162304,"data_stored":855079840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":410840974},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":2.488,"15min":2.4950000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.242,"5min":148.98400000000001,"15min":148.98400000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.516,"5min":2.7829999999999999,"15min":2.871},"min":{"1min":0.26700000000000002,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":10.516,"5min":149.02699999999999,"15min":149.02699999999999},"last":0.77600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":2.0009999999999999,"15min":1.7529999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":4.2199999999999998,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.371},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.2490000000000001,"15min":2.524},"min":{"1min":0.253,"5min":0.219,"15min":0.219},"max":{"1min":10.494,"5min":49.783000000000001,"15min":49.783000000000001},"last":2.5350000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:21:59 2023","interfaces":[{"interface":"back","average":{"1min":4.2350000000000003,"5min":4.109,"15min":1.6240000000000001},"min":{"1min":0.33200000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":17.260999999999999,"5min":153.18199999999999,"15min":153.18199999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.536,"5min":3.157,"15min":1.3240000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.2270000000000003,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":3.7120000000000002,"15min":3.032},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.192,"5min":199.43799999999999,"15min":199.43799999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.339,"5min":3.258,"15min":2.9590000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.413,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.38200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.7789999999999999,"15min":2.6389999999999998},"min":{"1min":0.35599999999999998,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":4.1980000000000004,"5min":61.930999999999997,"15min":61.930999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.415,"5min":2.1440000000000001,"15min":2.786},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.462,"5min":61.947000000000003,"15min":61.947000000000003},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.1459999999999999,"15min":2.6600000000000001},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.28399999999999997},"max":{"1min":11.516999999999999,"5min":14.382,"15min":14.382},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.77,"5min":2.363,"15min":2.4529999999999998},"min":{"1min":0.41999999999999998,"5min":0.39200000000000002,"15min":0.374},"max":{"1min":21.036999999999999,"5min":21.036999999999999,"15min":21.036999999999999},"last":0.504}]},{"osd":7,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":2.2330000000000001,"15min":2.5099999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.141999999999999,"5min":17.233000000000001,"15min":17.233000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.468,"5min":1.7110000000000001,"15min":1.9690000000000001},"min":{"1min":0.46300000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":16.161000000000001,"5min":16.161000000000001,"15min":16.161000000000001},"last":0.64400000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024114,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2532492,"kb_used_data":2006852,"kb_used_omap":5,"kb_used_meta":525626,"kb_avail":91212660,"statfs":{"total":95995035648,"available":93401763840,"internally_reserved":0,"allocated":2055016448,"data_stored":2053757460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":538241658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":4.7380000000000004,"15min":2.7999999999999998},"min":{"1min":0.21099999999999999,"5min":0.193,"15min":0.19},"max":{"1min":8.8889999999999993,"5min":185.17400000000001,"15min":185.17400000000001},"last":4.1790000000000003},{"interface":"front","average":{"1min":2.2410000000000001,"5min":4.867,"15min":3.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":10.73,"5min":186.691,"15min":186.691},"last":4.2190000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.6539999999999999,"15min":2.274},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.7709999999999999,"5min":17.588999999999999,"15min":17.588999999999999},"last":4.2839999999999998},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.77,"15min":2.3140000000000001},"min":{"1min":0.23300000000000001,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":5.4210000000000003,"5min":11.162000000000001,"15min":21.198},"last":4.1989999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:21:55 2023","interfaces":[{"interface":"back","average":{"1min":6.9260000000000002,"5min":6.2670000000000003,"15min":2.3450000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":52.683,"5min":185.89400000000001,"15min":185.89400000000001},"last":4.7039999999999997},{"interface":"front","average":{"1min":7.6319999999999997,"5min":6.6950000000000003,"15min":2.4980000000000002},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":52.941000000000003,"5min":186.72900000000001,"15min":186.72900000000001},"last":5.8049999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.3560000000000001,"15min":1.5940000000000001},"min":{"1min":0.41699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9950000000000001,"5min":11.162000000000001,"15min":15.835000000000001},"last":4.407},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.6910000000000001,"15min":2.0779999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":2.456,"5min":12.752000000000001,"15min":15.023},"last":5.9889999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.1059999999999999,"15min":2.032},"min":{"1min":0.41599999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.2450000000000001,"5min":62.984999999999999,"15min":62.984999999999999},"last":4.4470000000000001},{"interface":"front","average":{"1min":1.012,"5min":2.4670000000000001,"15min":2.4550000000000001},"min":{"1min":0.39900000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.367,"5min":64.983999999999995,"15min":64.983999999999995},"last":4.7309999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.899,"15min":2.234},"min":{"1min":0.36199999999999999,"5min":0.32000000000000001,"15min":0.29299999999999998},"max":{"1min":3.1749999999999998,"5min":9.6880000000000006,"15min":32.079000000000001},"last":7.3280000000000003},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.546,"15min":1.9219999999999999},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":4.9690000000000003,"5min":11.236000000000001,"15min":15.893000000000001},"last":4.5750000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.282,"15min":2.3260000000000001},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":2.8100000000000001,"5min":62.262,"15min":62.262},"last":5.8689999999999998},{"interface":"front","average":{"1min":1.115,"5min":2.3839999999999999,"15min":2.4860000000000002},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":5.875,"5min":62.229999999999997,"15min":62.229999999999997},"last":4.7569999999999997}]}]},{"osd":2,"up_from":128,"seq":549755813983,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961908,"kb_used_data":1664236,"kb_used_omap":7,"kb_used_meta":297656,"kb_avail":91783244,"statfs":{"total":95995035648,"available":93986041856,"internally_reserved":0,"allocated":1704177664,"data_stored":1703111824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":304799825},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":5.0780000000000003,"15min":2.9580000000000002},"min":{"1min":0.183,"5min":0.157,"15min":0.157},"max":{"1min":5.2549999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.8600000000000001,"5min":5.3010000000000002,"15min":3.4830000000000001},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.448,"5min":163.363,"15min":163.363},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":2.8420000000000001,"15min":2.835},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.188},"max":{"1min":5.5030000000000001,"5min":73.483999999999995,"15min":222.50299999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.331,"5min":3.222,"15min":3.4710000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.420999999999999,"5min":75.641000000000005,"15min":222.19399999999999},"last":0.40000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.4209999999999998,"15min":1.375},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.5379999999999998,"5min":81.183999999999997,"15min":81.183999999999997},"last":0.371},{"interface":"front","average":{"1min":0.748,"5min":3.4510000000000001,"15min":1.431},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.7890000000000001,"5min":81.061999999999998,"15min":81.061999999999998},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4870000000000001,"15min":1.8520000000000001},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":2.9710000000000001,"5min":12.106999999999999,"15min":12.141999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.962,"15min":2.1989999999999998},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.7089999999999996,"5min":11.994999999999999,"15min":15.183},"last":0.42499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.304,"15min":1.5720000000000001},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":2.6259999999999999,"5min":5.7199999999999998,"15min":5.7460000000000004},"last":2.4569999999999999},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.0760000000000001,"15min":2.3380000000000001},"min":{"1min":0.41099999999999998,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":5.6200000000000001,"5min":10.548,"15min":10.548},"last":2.4750000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.7589999999999999,"15min":2.0899999999999999},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.1319999999999997,"5min":10.522,"15min":10.522},"last":2.5310000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.2969999999999999,"15min":1.5209999999999999},"min":{"1min":0.41999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.371,"5min":6.883,"15min":7.327},"last":3.4359999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.4810000000000001,"15min":1.605},"min":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.9449999999999998,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":3.3969999999999998},{"interface":"front","average":{"1min":1.569,"5min":1.5629999999999999,"15min":2.5209999999999999},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.4160000000000004,"5min":6.7779999999999996,"15min":9.3249999999999993},"last":2.4260000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127181,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1381216,"kb_used_data":1092888,"kb_used_omap":2,"kb_used_meta":288317,"kb_avail":92363936,"statfs":{"total":95995035648,"available":94580670464,"internally_reserved":0,"allocated":1119117312,"data_stored":1118135439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":295236661},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":8.2959999999999994,"5min":5.3940000000000001,"15min":2.2109999999999999},"min":{"1min":0.23699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":70.349999999999994,"5min":187.852,"15min":187.852},"last":26.516999999999999},{"interface":"front","average":{"1min":8.0169999999999995,"5min":6.6180000000000003,"15min":2.8380000000000001},"min":{"1min":0.214,"5min":0.191,"15min":0.191},"max":{"1min":69.978999999999999,"5min":187.923,"15min":187.923},"last":26.556000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":5.9710000000000001,"15min":2.6219999999999999},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":59.646999999999998,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.255},{"interface":"front","average":{"1min":4.3120000000000003,"5min":3.1829999999999998,"15min":1.4670000000000001},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":60.085999999999999,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.216}]},{"osd":2,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":5.2519999999999998,"15min":2.3799999999999999},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":62.904000000000003,"5min":187.76400000000001,"15min":187.76400000000001},"last":11.6},{"interface":"front","average":{"1min":4.4989999999999997,"5min":3.9319999999999999,"15min":1.591},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":62.719000000000001,"5min":187.79499999999999,"15min":187.79499999999999},"last":11.737}]},{"osd":4,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.4099999999999999,"15min":0.88100000000000001},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2509999999999999,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.69599999999999995,"5min":3.278,"15min":1.887},"min":{"1min":0.436,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.722,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":3.7080000000000002,"15min":2.0510000000000002},"min":{"1min":0.46500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.6749999999999998,"5min":187.833,"15min":187.833},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.81599999999999995,"5min":3.7490000000000001,"15min":2.0680000000000001},"min":{"1min":0.44800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.3820000000000001,"5min":187.952,"15min":187.952},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":3.512,"15min":1.9670000000000001},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.7490000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":3.4849999999999999,"15min":1.9590000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4279999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.39800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":3.4660000000000002,"15min":1.5820000000000001},"min":{"1min":0.39200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.434},{"interface":"front","average":{"1min":0.73099999999999998,"5min":3.431,"15min":1.954},"min":{"1min":0.41799999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.774,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.36499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019217,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3022696,"kb_used_data":2135904,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":90722456,"statfs":{"total":95995035648,"available":92899794944,"internally_reserved":0,"allocated":2187165696,"data_stored":2186465391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3069999999999999,"15min":1.476},"min":{"1min":0.315,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":1.798,"5min":19.577000000000002,"15min":19.577000000000002},"last":66.007000000000005},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.7909999999999999,"15min":1.869},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":3.1560000000000001,"5min":19.594999999999999,"15min":19.594999999999999},"last":66.030000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":3.2909999999999999,"15min":2.1629999999999998},"min":{"1min":0.23699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.0190000000000001,"5min":91.849999999999994,"15min":91.849999999999994},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.88600000000000001,"5min":3.4199999999999999,"15min":2.206},"min":{"1min":0.34799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.117,"5min":91.885000000000005,"15min":91.885000000000005},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5589999999999999,"15min":1.75},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":10.884,"5min":10.884,"15min":10.884},"last":69.640000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.821,"15min":1.8799999999999999},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.5850000000000009,"5min":15.894,"15min":15.894},"last":69.676000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":4.5049999999999999,"15min":1.877},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.4749999999999996,"5min":91.697000000000003,"15min":91.697000000000003},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":4.2279999999999998,"15min":1.8240000000000001},"min":{"1min":0.38900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1850000000000001,"5min":94.356999999999999,"15min":94.356999999999999},"last":0.36299999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.1950000000000001,"15min":1.0669999999999999},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":9.1720000000000006,"5min":10.673,"15min":10.673},"last":0.379},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.147,"15min":1.246},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.24399999999999999},"max":{"1min":9.6379999999999999,"5min":12.552,"15min":12.552},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.78800000000000003,"15min":0.84399999999999997},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.881,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.315},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.242,"15min":1.171},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.0029999999999999,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.32700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:21:58 2023","interfaces":[{"interface":"back","average":{"1min":5.1459999999999999,"5min":2.399,"15min":1.5029999999999999},"min":{"1min":0.40600000000000003,"5min":0.308,"15min":0.308},"max":{"1min":62.658000000000001,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":5.4669999999999996,"5min":2.339,"15min":1.536},"min":{"1min":0.40899999999999997,"5min":0.311,"15min":0.311},"max":{"1min":62.573,"5min":62.573,"15min":62.573},"last":0.42699999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757582,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2087124992,"data_stored":2087124992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":854065152,"data_stored":854065152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1706754048,"data_stored":1706754048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1126694912,"data_stored":1126694912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2209808384,"data_stored":2209808384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1550647296,"data_stored":1550647296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":885653504,"data_stored":885653504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:23:01.815 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:23:01.856 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:01.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:01.972 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:02.001 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:02.041 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:02.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:02.146 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:02.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:02] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T06:23:02.207 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:02.470 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:02.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:02.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:02.667 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:02.731 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 59360 59344 100.166 57.8125 0.00362033 0.00997655 2023-10-09T06:23:02.733 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[220921]: pgmap v766: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:23:02.733 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:02.733 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:02.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[212750]: pgmap v766: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:23:02.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:02.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:02 smithi150 ceph-mon[194178]: pgmap v766: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:23:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:02.915 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:02.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:02.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:02.949 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:02.998 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:03.040 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:59659 10.5 10:abbae610:::benchmark_data_smithi107_363200_object59658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:03.049402+0000", 2023-10-09T06:23:03.061 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0043531339999999998, 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043860339999999996, 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 59659 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.062 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049402+0000", 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:03.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049402+0000", 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049400+0000", 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049423+0000", 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3051999999999999e-05 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049437+0000", 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4188e-05 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049446+0000", 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9819999999999997e-06 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049498+0000", 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1267999999999998e-05 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:03.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049648+0000", 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000150214 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.049813+0000", 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016468 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.053201+0000", 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033882790000000001 2023-10-09T06:23:03.067 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45650.0:59660 10.e e248/238 10:7d56330f:::benchmark_data_smithi107_363200_object59659:head v 248'3723, mlcod=248'3721)", 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:03.052316+0000", 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0014387790000000001, 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015596399999999999, 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:03.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052316+0000", 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052316+0000", 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:03.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052315+0000", 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052348+0000", 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2496000000000002e-05 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052357+0000", 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1169999999999993e-06 2023-10-09T06:23:03.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052363+0000", 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0290000000000002e-06 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052417+0000", 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4122000000000001e-05 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:03.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:03.052440+0000", 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3166000000000001e-05 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 2 2023-10-09T06:23:03.072 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:23:03.135 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:03.731 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 60507 60491 99.4163 71.6875 0.00239225 0.0100427 2023-10-09T06:23:04.171 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:04.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:04.266 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:23:04.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:04.383 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:04.615 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:23:04.732 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 61657 61641 98.7087 71.875 0.017266 0.0101251 2023-10-09T06:23:04.738 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:04.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:04.750 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:04 smithi107 ceph-mon[212750]: pgmap v767: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:23:04.750 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:04 smithi107 ceph-mon[220921]: pgmap v767: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:23:04.851 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:04 smithi150 ceph-mon[194178]: pgmap v767: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:23:04.949 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:05.732 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:23:05.733170+0000 min lat: 0.00112444 max lat: 0.323912 avg lat: 0.010245 2023-10-09T06:23:05.732 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:23:05.733 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 62506 62490 97.5682 53.0625 0.00830685 0.010245 2023-10-09T06:23:06.059 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:06.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:06.167 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:06.255 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:06.374 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:06.530 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:06.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:06.639 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:06.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:06.732 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 63577 63561 96.8214 66.9375 0.0085595 0.0103236 2023-10-09T06:23:06.755 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:06.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:06.863 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:06.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:06 smithi107 ceph-mon[212750]: pgmap v768: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:23:06.885 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:06 smithi107 ceph-mon[220921]: pgmap v768: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:23:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:06 smithi150 ceph-mon[194178]: pgmap v768: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:23:06.938 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:07.052 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:07.463 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:07.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:07.585 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:07.672 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:07.734 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 64581 64565 96.0085 62.75 0.0238157 0.0104098 2023-10-09T06:23:07.823 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:23:07.845 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:23:08.510 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:08.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:08.616 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:08.617 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:23:08.664 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:08.707 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:08.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:08.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:08.736 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 65407 65391 94.9726 51.625 0.00976481 0.0105252 2023-10-09T06:23:08.812 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[212750]: pgmap v769: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1543209086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[220921]: pgmap v769: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1543209086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:08.838 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:08.841 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:08 smithi150 ceph-mon[194178]: pgmap v769: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-09T06:23:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1543209086' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:23:09.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.633 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.633 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.733 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":769,"stamp":"2023-10-09T06:23:07.052529+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3987347360,"num_objects":60857,"num_object_clones":0,"num_object_copies":182571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60894,"num_write_kb":3893341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":337988,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62659,"ondisk_log_size":62659,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15542300,"kb_used_data":11634596,"kb_used_omap":20,"kb_used_meta":3907627,"kb_avail":640673764,"statfs":{"total":671965249536,"available":656049934336,"internally_reserved":0,"allocated":11913826304,"data_stored":11907324628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4001410702},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,3,1,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":727515136,"num_objects":11101,"num_object_clones":0,"num_object_copies":33303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11101,"num_write_kb":710464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004232"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":227,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225796+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:22:50.225796+0000","last_peered":"2023-10-09T06:22:50.225796+0000","last_clean":"2023-10-09T06:22:50.225796+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:22:50.225796+0000","last_undegraded":"2023-10-09T06:22:50.225796+0000","last_fullsized":"2023-10-09T06:22:50.225796+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222790+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:22:50.222790+0000","last_peered":"2023-10-09T06:22:50.222790+0000","last_clean":"2023-10-09T06:22:50.222790+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:22:50.222790+0000","last_undegraded":"2023-10-09T06:22:50.222790+0000","last_fullsized":"2023-10-09T06:22:50.222790+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.216944+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:22:50.216944+0000","last_peered":"2023-10-09T06:22:50.216944+0000","last_clean":"2023-10-09T06:22:50.216944+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:22:50.216944+0000","last_undegraded":"2023-10-09T06:22:50.216944+0000","last_fullsized":"2023-10-09T06:22:50.216944+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220449+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:22:50.220449+0000","last_peered":"2023-10-09T06:22:50.220449+0000","last_clean":"2023-10-09T06:22:50.220449+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:22:50.220449+0000","last_undegraded":"2023-10-09T06:22:50.220449+0000","last_fullsized":"2023-10-09T06:22:50.220449+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222543+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:22:50.222543+0000","last_peered":"2023-10-09T06:22:50.222543+0000","last_clean":"2023-10-09T06:22:50.222543+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:22:50.222543+0000","last_undegraded":"2023-10-09T06:22:50.222543+0000","last_fullsized":"2023-10-09T06:22:50.222543+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227999+0000","last_change":"2023-10-09T06:21:45.411474+0000","last_active":"2023-10-09T06:22:50.227999+0000","last_peered":"2023-10-09T06:22:50.227999+0000","last_clean":"2023-10-09T06:22:50.227999+0000","last_became_active":"2023-10-09T06:21:45.410968+0000","last_became_peered":"2023-10-09T06:21:45.410968+0000","last_unstale":"2023-10-09T06:22:50.227999+0000","last_undegraded":"2023-10-09T06:22:50.227999+0000","last_fullsized":"2023-10-09T06:22:50.227999+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:53.196763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219387+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:22:50.219387+0000","last_peered":"2023-10-09T06:22:50.219387+0000","last_clean":"2023-10-09T06:22:50.219387+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:22:50.219387+0000","last_undegraded":"2023-10-09T06:22:50.219387+0000","last_fullsized":"2023-10-09T06:22:50.219387+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222143+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:22:50.222143+0000","last_peered":"2023-10-09T06:22:50.222143+0000","last_clean":"2023-10-09T06:22:50.222143+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:22:50.222143+0000","last_undegraded":"2023-10-09T06:22:50.222143+0000","last_fullsized":"2023-10-09T06:22:50.222143+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":176,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228800+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:22:50.228800+0000","last_peered":"2023-10-09T06:22:50.228800+0000","last_clean":"2023-10-09T06:22:50.228800+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:22:50.228800+0000","last_undegraded":"2023-10-09T06:22:50.228800+0000","last_fullsized":"2023-10-09T06:22:50.228800+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":182,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222350+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:22:50.222350+0000","last_peered":"2023-10-09T06:22:50.222350+0000","last_clean":"2023-10-09T06:22:50.222350+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:22:50.222350+0000","last_undegraded":"2023-10-09T06:22:50.222350+0000","last_fullsized":"2023-10-09T06:22:50.222350+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":97,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229093+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:22:50.229093+0000","last_peered":"2023-10-09T06:22:50.229093+0000","last_clean":"2023-10-09T06:22:50.229093+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:22:50.229093+0000","last_undegraded":"2023-10-09T06:22:50.229093+0000","last_fullsized":"2023-10-09T06:22:50.229093+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":178,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217572+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:22:50.217572+0000","last_peered":"2023-10-09T06:22:50.217572+0000","last_clean":"2023-10-09T06:22:50.217572+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:22:50.217572+0000","last_undegraded":"2023-10-09T06:22:50.217572+0000","last_fullsized":"2023-10-09T06:22:50.217572+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":254,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226156+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:22:50.226156+0000","last_peered":"2023-10-09T06:22:50.226156+0000","last_clean":"2023-10-09T06:22:50.226156+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:22:50.226156+0000","last_undegraded":"2023-10-09T06:22:50.226156+0000","last_fullsized":"2023-10-09T06:22:50.226156+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226923+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:22:50.226923+0000","last_peered":"2023-10-09T06:22:50.226923+0000","last_clean":"2023-10-09T06:22:50.226923+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:22:50.226923+0000","last_undegraded":"2023-10-09T06:22:50.226923+0000","last_fullsized":"2023-10-09T06:22:50.226923+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":97,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228251+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:22:50.228251+0000","last_peered":"2023-10-09T06:22:50.228251+0000","last_clean":"2023-10-09T06:22:50.228251+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:22:50.228251+0000","last_undegraded":"2023-10-09T06:22:50.228251+0000","last_fullsized":"2023-10-09T06:22:50.228251+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229688+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:22:50.229688+0000","last_peered":"2023-10-09T06:22:50.229688+0000","last_clean":"2023-10-09T06:22:50.229688+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:22:50.229688+0000","last_undegraded":"2023-10-09T06:22:50.229688+0000","last_fullsized":"2023-10-09T06:22:50.229688+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220388+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:22:50.220388+0000","last_peered":"2023-10-09T06:22:50.220388+0000","last_clean":"2023-10-09T06:22:50.220388+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:22:50.220388+0000","last_undegraded":"2023-10-09T06:22:50.220388+0000","last_fullsized":"2023-10-09T06:22:50.220388+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":175,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228960+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:22:50.228960+0000","last_peered":"2023-10-09T06:22:50.228960+0000","last_clean":"2023-10-09T06:22:50.228960+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:22:50.228960+0000","last_undegraded":"2023-10-09T06:22:50.228960+0000","last_fullsized":"2023-10-09T06:22:50.228960+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228248+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:22:50.228248+0000","last_peered":"2023-10-09T06:22:50.228248+0000","last_clean":"2023-10-09T06:22:50.228248+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:22:50.228248+0000","last_undegraded":"2023-10-09T06:22:50.228248+0000","last_fullsized":"2023-10-09T06:22:50.228248+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":217,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220593+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:22:50.220593+0000","last_peered":"2023-10-09T06:22:50.220593+0000","last_clean":"2023-10-09T06:22:50.220593+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:22:50.220593+0000","last_undegraded":"2023-10-09T06:22:50.220593+0000","last_fullsized":"2023-10-09T06:22:50.220593+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.224216+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:22:50.224216+0000","last_peered":"2023-10-09T06:22:50.224216+0000","last_clean":"2023-10-09T06:22:50.224216+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:22:50.224216+0000","last_undegraded":"2023-10-09T06:22:50.224216+0000","last_fullsized":"2023-10-09T06:22:50.224216+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222686+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:22:50.222686+0000","last_peered":"2023-10-09T06:22:50.222686+0000","last_clean":"2023-10-09T06:22:50.222686+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:22:50.222686+0000","last_undegraded":"2023-10-09T06:22:50.222686+0000","last_fullsized":"2023-10-09T06:22:50.222686+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"248'3819","reported_seq":3838,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.054634+0000","last_change":"2023-10-09T06:22:15.195712+0000","last_active":"2023-10-09T06:23:05.054634+0000","last_peered":"2023-10-09T06:23:05.054634+0000","last_clean":"2023-10-09T06:23:05.054634+0000","last_became_active":"2023-10-09T06:22:15.195422+0000","last_became_peered":"2023-10-09T06:22:15.195422+0000","last_unstale":"2023-10-09T06:23:05.054634+0000","last_undegraded":"2023-10-09T06:23:05.054634+0000","last_fullsized":"2023-10-09T06:23:05.054634+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3819,"log_dups_size":0,"ondisk_log_size":3819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:32.535427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250281984,"num_objects":3819,"num_object_clones":0,"num_object_copies":11457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":244416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":95,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228636+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:22:50.228636+0000","last_peered":"2023-10-09T06:22:50.228636+0000","last_clean":"2023-10-09T06:22:50.228636+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:22:50.228636+0000","last_undegraded":"2023-10-09T06:22:50.228636+0000","last_fullsized":"2023-10-09T06:22:50.228636+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":158,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227948+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:22:50.227948+0000","last_peered":"2023-10-09T06:22:50.227948+0000","last_clean":"2023-10-09T06:22:50.227948+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:22:50.227948+0000","last_undegraded":"2023-10-09T06:22:50.227948+0000","last_fullsized":"2023-10-09T06:22:50.227948+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"248'3655","reported_seq":3673,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.817202+0000","last_change":"2023-10-09T06:22:15.412855+0000","last_active":"2023-10-09T06:23:02.817202+0000","last_peered":"2023-10-09T06:23:02.817202+0000","last_clean":"2023-10-09T06:23:02.817202+0000","last_became_active":"2023-10-09T06:22:15.412520+0000","last_became_peered":"2023-10-09T06:22:15.412520+0000","last_unstale":"2023-10-09T06:23:02.817202+0000","last_undegraded":"2023-10-09T06:23:02.817202+0000","last_fullsized":"2023-10-09T06:23:02.817202+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3655,"log_dups_size":0,"ondisk_log_size":3655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:49.054096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239534080,"num_objects":3655,"num_object_clones":0,"num_object_copies":10965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3655,"num_write_kb":233920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":95,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228721+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:22:50.228721+0000","last_peered":"2023-10-09T06:22:50.228721+0000","last_clean":"2023-10-09T06:22:50.228721+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:22:50.228721+0000","last_undegraded":"2023-10-09T06:22:50.228721+0000","last_fullsized":"2023-10-09T06:22:50.228721+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225938+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:22:50.225938+0000","last_peered":"2023-10-09T06:22:50.225938+0000","last_clean":"2023-10-09T06:22:50.225938+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:22:50.225938+0000","last_undegraded":"2023-10-09T06:22:50.225938+0000","last_fullsized":"2023-10-09T06:22:50.225938+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"248'3917","reported_seq":3936,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.047179+0000","last_change":"2023-10-09T06:22:15.411974+0000","last_active":"2023-10-09T06:23:05.047179+0000","last_peered":"2023-10-09T06:23:05.047179+0000","last_clean":"2023-10-09T06:23:05.047179+0000","last_became_active":"2023-10-09T06:22:15.411649+0000","last_became_peered":"2023-10-09T06:22:15.411649+0000","last_unstale":"2023-10-09T06:23:05.047179+0000","last_undegraded":"2023-10-09T06:23:05.047179+0000","last_fullsized":"2023-10-09T06:23:05.047179+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3917,"log_dups_size":0,"ondisk_log_size":3917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:27.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256704512,"num_objects":3917,"num_object_clones":0,"num_object_copies":11751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3917,"num_write_kb":250688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":109,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228891+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:22:50.228891+0000","last_peered":"2023-10-09T06:22:50.228891+0000","last_clean":"2023-10-09T06:22:50.228891+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:22:50.228891+0000","last_undegraded":"2023-10-09T06:22:50.228891+0000","last_fullsized":"2023-10-09T06:22:50.228891+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":227,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227045+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:22:50.227045+0000","last_peered":"2023-10-09T06:22:50.227045+0000","last_clean":"2023-10-09T06:22:50.227045+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:22:50.227045+0000","last_undegraded":"2023-10-09T06:22:50.227045+0000","last_fullsized":"2023-10-09T06:22:50.227045+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"248'3923","reported_seq":3942,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.982498+0000","last_change":"2023-10-09T06:22:15.411313+0000","last_active":"2023-10-09T06:23:05.982498+0000","last_peered":"2023-10-09T06:23:05.982498+0000","last_clean":"2023-10-09T06:23:05.982498+0000","last_became_active":"2023-10-09T06:22:15.410917+0000","last_became_peered":"2023-10-09T06:22:15.410917+0000","last_unstale":"2023-10-09T06:23:05.982498+0000","last_undegraded":"2023-10-09T06:23:05.982498+0000","last_fullsized":"2023-10-09T06:23:05.982498+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3923,"log_dups_size":0,"ondisk_log_size":3923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:24.915932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257097728,"num_objects":3923,"num_object_clones":0,"num_object_copies":11769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3923,"num_write_kb":251072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229575+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:22:50.229575+0000","last_peered":"2023-10-09T06:22:50.229575+0000","last_clean":"2023-10-09T06:22:50.229575+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:22:50.229575+0000","last_undegraded":"2023-10-09T06:22:50.229575+0000","last_fullsized":"2023-10-09T06:22:50.229575+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":199,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219517+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:22:50.219517+0000","last_peered":"2023-10-09T06:22:50.219517+0000","last_clean":"2023-10-09T06:22:50.219517+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:22:50.219517+0000","last_undegraded":"2023-10-09T06:22:50.219517+0000","last_fullsized":"2023-10-09T06:22:50.219517+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"248'3911","reported_seq":3930,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:06.683785+0000","last_change":"2023-10-09T06:22:15.196669+0000","last_active":"2023-10-09T06:23:06.683785+0000","last_peered":"2023-10-09T06:23:06.683785+0000","last_clean":"2023-10-09T06:23:06.683785+0000","last_became_active":"2023-10-09T06:22:15.196399+0000","last_became_peered":"2023-10-09T06:22:15.196399+0000","last_unstale":"2023-10-09T06:23:06.683785+0000","last_undegraded":"2023-10-09T06:23:06.683785+0000","last_fullsized":"2023-10-09T06:23:06.683785+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3911,"log_dups_size":0,"ondisk_log_size":3911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.953962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256311296,"num_objects":3911,"num_object_clones":0,"num_object_copies":11733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3911,"num_write_kb":250304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":540,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228110+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:22:50.228110+0000","last_peered":"2023-10-09T06:22:50.228110+0000","last_clean":"2023-10-09T06:22:50.228110+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:22:50.228110+0000","last_undegraded":"2023-10-09T06:22:50.228110+0000","last_fullsized":"2023-10-09T06:22:50.228110+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222891+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:22:50.222891+0000","last_peered":"2023-10-09T06:22:50.222891+0000","last_clean":"2023-10-09T06:22:50.222891+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:22:50.222891+0000","last_undegraded":"2023-10-09T06:22:50.222891+0000","last_fullsized":"2023-10-09T06:22:50.222891+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226256+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:22:50.226256+0000","last_peered":"2023-10-09T06:22:50.226256+0000","last_clean":"2023-10-09T06:22:50.226256+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:22:50.226256+0000","last_undegraded":"2023-10-09T06:22:50.226256+0000","last_fullsized":"2023-10-09T06:22:50.226256+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"248'3931","reported_seq":3950,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.995525+0000","last_change":"2023-10-09T06:22:15.411262+0000","last_active":"2023-10-09T06:23:05.995525+0000","last_peered":"2023-10-09T06:23:05.995525+0000","last_clean":"2023-10-09T06:23:05.995525+0000","last_became_active":"2023-10-09T06:22:15.410925+0000","last_became_peered":"2023-10-09T06:22:15.410925+0000","last_unstale":"2023-10-09T06:23:05.995525+0000","last_undegraded":"2023-10-09T06:23:05.995525+0000","last_fullsized":"2023-10-09T06:23:05.995525+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3931,"log_dups_size":0,"ondisk_log_size":3931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:51.768992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257622016,"num_objects":3931,"num_object_clones":0,"num_object_copies":11793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3931,"num_write_kb":251584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":530,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219660+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:22:50.219660+0000","last_peered":"2023-10-09T06:22:50.219660+0000","last_clean":"2023-10-09T06:22:50.219660+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:22:50.219660+0000","last_undegraded":"2023-10-09T06:22:50.219660+0000","last_fullsized":"2023-10-09T06:22:50.219660+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226802+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:22:50.226802+0000","last_peered":"2023-10-09T06:22:50.226802+0000","last_clean":"2023-10-09T06:22:50.226802+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:22:50.226802+0000","last_undegraded":"2023-10-09T06:22:50.226802+0000","last_fullsized":"2023-10-09T06:22:50.226802+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226424+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:22:50.226424+0000","last_peered":"2023-10-09T06:22:50.226424+0000","last_clean":"2023-10-09T06:22:50.226424+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:22:50.226424+0000","last_undegraded":"2023-10-09T06:22:50.226424+0000","last_fullsized":"2023-10-09T06:22:50.226424+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"248'3727","reported_seq":3745,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.736035+0000","last_change":"2023-10-09T06:22:15.196292+0000","last_active":"2023-10-09T06:23:02.736035+0000","last_peered":"2023-10-09T06:23:02.736035+0000","last_clean":"2023-10-09T06:23:02.736035+0000","last_became_active":"2023-10-09T06:22:15.195400+0000","last_became_peered":"2023-10-09T06:22:15.195400+0000","last_unstale":"2023-10-09T06:23:02.736035+0000","last_undegraded":"2023-10-09T06:23:02.736035+0000","last_fullsized":"2023-10-09T06:23:02.736035+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3727,"log_dups_size":0,"ondisk_log_size":3727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:20.452846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244252672,"num_objects":3727,"num_object_clones":0,"num_object_copies":11181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3727,"num_write_kb":238528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225752+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:22:50.225752+0000","last_peered":"2023-10-09T06:22:50.225752+0000","last_clean":"2023-10-09T06:22:50.225752+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:22:50.225752+0000","last_undegraded":"2023-10-09T06:22:50.225752+0000","last_fullsized":"2023-10-09T06:22:50.225752+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226249+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:22:50.226249+0000","last_peered":"2023-10-09T06:22:50.226249+0000","last_clean":"2023-10-09T06:22:50.226249+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:22:50.226249+0000","last_undegraded":"2023-10-09T06:22:50.226249+0000","last_fullsized":"2023-10-09T06:22:50.226249+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"248'3812","reported_seq":3830,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.884981+0000","last_change":"2023-10-09T06:22:15.195487+0000","last_active":"2023-10-09T06:23:02.884981+0000","last_peered":"2023-10-09T06:23:02.884981+0000","last_clean":"2023-10-09T06:23:02.884981+0000","last_became_active":"2023-10-09T06:22:15.195089+0000","last_became_peered":"2023-10-09T06:22:15.195089+0000","last_unstale":"2023-10-09T06:23:02.884981+0000","last_undegraded":"2023-10-09T06:23:02.884981+0000","last_fullsized":"2023-10-09T06:23:02.884981+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3812,"log_dups_size":0,"ondisk_log_size":3812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:34.432191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249823232,"num_objects":3812,"num_object_clones":0,"num_object_copies":11436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3812,"num_write_kb":243968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":526,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220313+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:22:50.220313+0000","last_peered":"2023-10-09T06:22:50.220313+0000","last_clean":"2023-10-09T06:22:50.220313+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:22:50.220313+0000","last_undegraded":"2023-10-09T06:22:50.220313+0000","last_fullsized":"2023-10-09T06:22:50.220313+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222157+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:22:50.222157+0000","last_peered":"2023-10-09T06:22:50.222157+0000","last_clean":"2023-10-09T06:22:50.222157+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:22:50.222157+0000","last_undegraded":"2023-10-09T06:22:50.222157+0000","last_fullsized":"2023-10-09T06:22:50.222157+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":178,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217365+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:22:50.217365+0000","last_peered":"2023-10-09T06:22:50.217365+0000","last_clean":"2023-10-09T06:22:50.217365+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:22:50.217365+0000","last_undegraded":"2023-10-09T06:22:50.217365+0000","last_fullsized":"2023-10-09T06:22:50.217365+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"248'3607","reported_seq":3625,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:03.071035+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:23:03.071035+0000","last_peered":"2023-10-09T06:23:03.071035+0000","last_clean":"2023-10-09T06:23:03.071035+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:23:03.071035+0000","last_undegraded":"2023-10-09T06:23:03.071035+0000","last_fullsized":"2023-10-09T06:23:03.071035+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3607,"log_dups_size":0,"ondisk_log_size":3607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236388352,"num_objects":3607,"num_object_clones":0,"num_object_copies":10821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3607,"num_write_kb":230848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228084+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:22:50.228084+0000","last_peered":"2023-10-09T06:22:50.228084+0000","last_clean":"2023-10-09T06:22:50.228084+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:22:50.228084+0000","last_undegraded":"2023-10-09T06:22:50.228084+0000","last_fullsized":"2023-10-09T06:22:50.228084+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217333+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:22:50.217333+0000","last_peered":"2023-10-09T06:22:50.217333+0000","last_clean":"2023-10-09T06:22:50.217333+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:22:50.217333+0000","last_undegraded":"2023-10-09T06:22:50.217333+0000","last_fullsized":"2023-10-09T06:22:50.217333+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222874+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:22:50.222874+0000","last_peered":"2023-10-09T06:22:50.222874+0000","last_clean":"2023-10-09T06:22:50.222874+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:22:50.222874+0000","last_undegraded":"2023-10-09T06:22:50.222874+0000","last_fullsized":"2023-10-09T06:22:50.222874+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":172,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217172+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:22:50.217172+0000","last_peered":"2023-10-09T06:22:50.217172+0000","last_clean":"2023-10-09T06:22:50.217172+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:22:50.217172+0000","last_undegraded":"2023-10-09T06:22:50.217172+0000","last_fullsized":"2023-10-09T06:22:50.217172+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"248'3755","reported_seq":3773,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.842885+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:23:02.842885+0000","last_peered":"2023-10-09T06:23:02.842885+0000","last_clean":"2023-10-09T06:23:02.842885+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:23:02.842885+0000","last_undegraded":"2023-10-09T06:23:02.842885+0000","last_fullsized":"2023-10-09T06:23:02.842885+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3755,"log_dups_size":0,"ondisk_log_size":3755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246087680,"num_objects":3755,"num_object_clones":0,"num_object_copies":11265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3755,"num_write_kb":240320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":531,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.218634+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:22:50.218634+0000","last_peered":"2023-10-09T06:22:50.218634+0000","last_clean":"2023-10-09T06:22:50.218634+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:22:50.218634+0000","last_undegraded":"2023-10-09T06:22:50.218634+0000","last_fullsized":"2023-10-09T06:22:50.218634+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217314+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:22:50.217314+0000","last_peered":"2023-10-09T06:22:50.217314+0000","last_clean":"2023-10-09T06:22:50.217314+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:22:50.217314+0000","last_undegraded":"2023-10-09T06:22:50.217314+0000","last_fullsized":"2023-10-09T06:22:50.217314+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226806+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:22:50.226806+0000","last_peered":"2023-10-09T06:22:50.226806+0000","last_clean":"2023-10-09T06:22:50.226806+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:22:50.226806+0000","last_undegraded":"2023-10-09T06:22:50.226806+0000","last_fullsized":"2023-10-09T06:22:50.226806+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":193,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217250+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:22:50.217250+0000","last_peered":"2023-10-09T06:22:50.217250+0000","last_clean":"2023-10-09T06:22:50.217250+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:22:50.217250+0000","last_undegraded":"2023-10-09T06:22:50.217250+0000","last_fullsized":"2023-10-09T06:22:50.217250+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"248'3959","reported_seq":3978,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.995590+0000","last_change":"2023-10-09T06:22:15.195697+0000","last_active":"2023-10-09T06:23:05.995590+0000","last_peered":"2023-10-09T06:23:05.995590+0000","last_clean":"2023-10-09T06:23:05.995590+0000","last_became_active":"2023-10-09T06:22:15.195391+0000","last_became_peered":"2023-10-09T06:22:15.195391+0000","last_unstale":"2023-10-09T06:23:05.995590+0000","last_undegraded":"2023-10-09T06:23:05.995590+0000","last_fullsized":"2023-10-09T06:23:05.995590+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3959,"log_dups_size":0,"ondisk_log_size":3959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:00.178768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259457024,"num_objects":3959,"num_object_clones":0,"num_object_copies":11877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3959,"num_write_kb":253376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":540,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228134+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:22:50.228134+0000","last_peered":"2023-10-09T06:22:50.228134+0000","last_clean":"2023-10-09T06:22:50.228134+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:22:50.228134+0000","last_undegraded":"2023-10-09T06:22:50.228134+0000","last_fullsized":"2023-10-09T06:22:50.228134+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222732+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:22:50.222732+0000","last_peered":"2023-10-09T06:22:50.222732+0000","last_clean":"2023-10-09T06:22:50.222732+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:22:50.222732+0000","last_undegraded":"2023-10-09T06:22:50.222732+0000","last_fullsized":"2023-10-09T06:22:50.222732+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277382+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:22:57.277382+0000","last_peered":"2023-10-09T06:22:57.277382+0000","last_clean":"2023-10-09T06:22:57.277382+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:22:57.277382+0000","last_undegraded":"2023-10-09T06:22:57.277382+0000","last_fullsized":"2023-10-09T06:22:57.277382+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217487+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:22:50.217487+0000","last_peered":"2023-10-09T06:22:50.217487+0000","last_clean":"2023-10-09T06:22:50.217487+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:22:50.217487+0000","last_undegraded":"2023-10-09T06:22:50.217487+0000","last_fullsized":"2023-10-09T06:22:50.217487+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"248'3720","reported_seq":3738,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.827842+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:23:02.827842+0000","last_peered":"2023-10-09T06:23:02.827842+0000","last_clean":"2023-10-09T06:23:02.827842+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:23:02.827842+0000","last_undegraded":"2023-10-09T06:23:02.827842+0000","last_fullsized":"2023-10-09T06:23:02.827842+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3720,"log_dups_size":0,"ondisk_log_size":3720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243793920,"num_objects":3720,"num_object_clones":0,"num_object_copies":11160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3720,"num_write_kb":238080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":179,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.221706+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:22:50.221706+0000","last_peered":"2023-10-09T06:22:50.221706+0000","last_clean":"2023-10-09T06:22:50.221706+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:22:50.221706+0000","last_undegraded":"2023-10-09T06:22:50.221706+0000","last_fullsized":"2023-10-09T06:22:50.221706+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":521,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228007+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:22:50.228007+0000","last_peered":"2023-10-09T06:22:50.228007+0000","last_clean":"2023-10-09T06:22:50.228007+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:22:50.228007+0000","last_undegraded":"2023-10-09T06:22:50.228007+0000","last_fullsized":"2023-10-09T06:22:50.228007+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":174,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294971+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:22:57.294971+0000","last_peered":"2023-10-09T06:22:57.294971+0000","last_clean":"2023-10-09T06:22:57.294971+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:22:57.294971+0000","last_undegraded":"2023-10-09T06:22:57.294971+0000","last_fullsized":"2023-10-09T06:22:57.294971+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":210,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227559+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:22:50.227559+0000","last_peered":"2023-10-09T06:22:50.227559+0000","last_clean":"2023-10-09T06:22:50.227559+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:22:50.227559+0000","last_undegraded":"2023-10-09T06:22:50.227559+0000","last_fullsized":"2023-10-09T06:22:50.227559+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"248'3859","reported_seq":3878,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:05.066575+0000","last_change":"2023-10-09T06:22:15.192849+0000","last_active":"2023-10-09T06:23:05.066575+0000","last_peered":"2023-10-09T06:23:05.066575+0000","last_clean":"2023-10-09T06:23:05.066575+0000","last_became_active":"2023-10-09T06:22:15.192540+0000","last_became_peered":"2023-10-09T06:22:15.192540+0000","last_unstale":"2023-10-09T06:23:05.066575+0000","last_undegraded":"2023-10-09T06:23:05.066575+0000","last_fullsized":"2023-10-09T06:23:05.066575+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3859,"log_dups_size":0,"ondisk_log_size":3859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:21.157073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252903424,"num_objects":3859,"num_object_clones":0,"num_object_copies":11577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3859,"num_write_kb":246976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":529,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217223+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:22:50.217223+0000","last_peered":"2023-10-09T06:22:50.217223+0000","last_clean":"2023-10-09T06:22:50.217223+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:22:50.217223+0000","last_undegraded":"2023-10-09T06:22:50.217223+0000","last_fullsized":"2023-10-09T06:22:50.217223+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227894+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:22:50.227894+0000","last_peered":"2023-10-09T06:22:50.227894+0000","last_clean":"2023-10-09T06:22:50.227894+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:22:50.227894+0000","last_undegraded":"2023-10-09T06:22:50.227894+0000","last_fullsized":"2023-10-09T06:22:50.227894+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":200,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.218725+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:22:50.218725+0000","last_peered":"2023-10-09T06:22:50.218725+0000","last_clean":"2023-10-09T06:22:50.218725+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:22:50.218725+0000","last_undegraded":"2023-10-09T06:22:50.218725+0000","last_fullsized":"2023-10-09T06:22:50.218725+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":222,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220524+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:22:50.220524+0000","last_peered":"2023-10-09T06:22:50.220524+0000","last_clean":"2023-10-09T06:22:50.220524+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:22:50.220524+0000","last_undegraded":"2023-10-09T06:22:50.220524+0000","last_fullsized":"2023-10-09T06:22:50.220524+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"248'3761","reported_seq":3779,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.750195+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:23:02.750195+0000","last_peered":"2023-10-09T06:23:02.750195+0000","last_clean":"2023-10-09T06:23:02.750195+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:23:02.750195+0000","last_undegraded":"2023-10-09T06:23:02.750195+0000","last_fullsized":"2023-10-09T06:23:02.750195+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3761,"log_dups_size":0,"ondisk_log_size":3761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246480896,"num_objects":3761,"num_object_clones":0,"num_object_copies":11283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3761,"num_write_kb":240704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.227884+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:22:50.227884+0000","last_peered":"2023-10-09T06:22:50.227884+0000","last_clean":"2023-10-09T06:22:50.227884+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:22:50.227884+0000","last_undegraded":"2023-10-09T06:22:50.227884+0000","last_fullsized":"2023-10-09T06:22:50.227884+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229022+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:22:50.229022+0000","last_peered":"2023-10-09T06:22:50.229022+0000","last_clean":"2023-10-09T06:22:50.229022+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:22:50.229022+0000","last_undegraded":"2023-10-09T06:22:50.229022+0000","last_fullsized":"2023-10-09T06:22:50.229022+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":228,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217170+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:22:50.217170+0000","last_peered":"2023-10-09T06:22:50.217170+0000","last_clean":"2023-10-09T06:22:50.217170+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:22:50.217170+0000","last_undegraded":"2023-10-09T06:22:50.217170+0000","last_fullsized":"2023-10-09T06:22:50.217170+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219722+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:22:50.219722+0000","last_peered":"2023-10-09T06:22:50.219722+0000","last_clean":"2023-10-09T06:22:50.219722+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:22:50.219722+0000","last_undegraded":"2023-10-09T06:22:50.219722+0000","last_fullsized":"2023-10-09T06:22:50.219722+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"248'3708","reported_seq":3726,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.819342+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:23:02.819342+0000","last_peered":"2023-10-09T06:23:02.819342+0000","last_clean":"2023-10-09T06:23:02.819342+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:23:02.819342+0000","last_undegraded":"2023-10-09T06:23:02.819342+0000","last_fullsized":"2023-10-09T06:23:02.819342+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3708,"log_dups_size":0,"ondisk_log_size":3708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243007488,"num_objects":3708,"num_object_clones":0,"num_object_copies":11124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3708,"num_write_kb":237312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":531,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226275+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:22:50.226275+0000","last_peered":"2023-10-09T06:22:50.226275+0000","last_clean":"2023-10-09T06:22:50.226275+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:22:50.226275+0000","last_undegraded":"2023-10-09T06:22:50.226275+0000","last_fullsized":"2023-10-09T06:22:50.226275+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222690+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:22:50.222690+0000","last_peered":"2023-10-09T06:22:50.222690+0000","last_clean":"2023-10-09T06:22:50.222690+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:22:50.222690+0000","last_undegraded":"2023-10-09T06:22:50.222690+0000","last_fullsized":"2023-10-09T06:22:50.222690+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":205,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.217719+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:22:50.217719+0000","last_peered":"2023-10-09T06:22:50.217719+0000","last_clean":"2023-10-09T06:22:50.217719+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:22:50.217719+0000","last_undegraded":"2023-10-09T06:22:50.217719+0000","last_fullsized":"2023-10-09T06:22:50.217719+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222018+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:22:50.222018+0000","last_peered":"2023-10-09T06:22:50.222018+0000","last_clean":"2023-10-09T06:22:50.222018+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:22:50.222018+0000","last_undegraded":"2023-10-09T06:22:50.222018+0000","last_fullsized":"2023-10-09T06:22:50.222018+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"248'3769","reported_seq":3787,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:02.807526+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:23:02.807526+0000","last_peered":"2023-10-09T06:23:02.807526+0000","last_clean":"2023-10-09T06:23:02.807526+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:23:02.807526+0000","last_undegraded":"2023-10-09T06:23:02.807526+0000","last_fullsized":"2023-10-09T06:23:02.807526+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":3769,"log_dups_size":0,"ondisk_log_size":3769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247005184,"num_objects":3769,"num_object_clones":0,"num_object_copies":11307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3769,"num_write_kb":241216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":550,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.229808+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:22:50.229808+0000","last_peered":"2023-10-09T06:22:50.229808+0000","last_clean":"2023-10-09T06:22:50.229808+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:22:50.229808+0000","last_undegraded":"2023-10-09T06:22:50.229808+0000","last_fullsized":"2023-10-09T06:22:50.229808+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":232,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.228395+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:22:50.228395+0000","last_peered":"2023-10-09T06:22:50.228395+0000","last_clean":"2023-10-09T06:22:50.228395+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:22:50.228395+0000","last_undegraded":"2023-10-09T06:22:50.228395+0000","last_fullsized":"2023-10-09T06:22:50.228395+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":203,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.220170+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:22:50.220170+0000","last_peered":"2023-10-09T06:22:50.220170+0000","last_clean":"2023-10-09T06:22:50.220170+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:22:50.220170+0000","last_undegraded":"2023-10-09T06:22:50.220170+0000","last_fullsized":"2023-10-09T06:22:50.220170+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":201,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225857+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:22:50.225857+0000","last_peered":"2023-10-09T06:22:50.225857+0000","last_clean":"2023-10-09T06:22:50.225857+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:22:50.225857+0000","last_undegraded":"2023-10-09T06:22:50.225857+0000","last_fullsized":"2023-10-09T06:22:50.225857+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":205,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219658+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:22:50.219658+0000","last_peered":"2023-10-09T06:22:50.219658+0000","last_clean":"2023-10-09T06:22:50.219658+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:22:50.219658+0000","last_undegraded":"2023-10-09T06:22:50.219658+0000","last_fullsized":"2023-10-09T06:22:50.219658+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219742+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:22:50.219742+0000","last_peered":"2023-10-09T06:22:50.219742+0000","last_clean":"2023-10-09T06:22:50.219742+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:22:50.219742+0000","last_undegraded":"2023-10-09T06:22:50.219742+0000","last_fullsized":"2023-10-09T06:22:50.219742+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":53,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226055+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:22:50.226055+0000","last_peered":"2023-10-09T06:22:50.226055+0000","last_clean":"2023-10-09T06:22:50.226055+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:22:50.226055+0000","last_undegraded":"2023-10-09T06:22:50.226055+0000","last_fullsized":"2023-10-09T06:22:50.226055+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226721+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:22:50.226721+0000","last_peered":"2023-10-09T06:22:50.226721+0000","last_clean":"2023-10-09T06:22:50.226721+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:22:50.226721+0000","last_undegraded":"2023-10-09T06:22:50.226721+0000","last_fullsized":"2023-10-09T06:22:50.226721+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":53,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.226417+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:22:50.226417+0000","last_peered":"2023-10-09T06:22:50.226417+0000","last_clean":"2023-10-09T06:22:50.226417+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:22:50.226417+0000","last_undegraded":"2023-10-09T06:22:50.226417+0000","last_fullsized":"2023-10-09T06:22:50.226417+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225937+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:22:50.225937+0000","last_peered":"2023-10-09T06:22:50.225937+0000","last_clean":"2023-10-09T06:22:50.225937+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:22:50.225937+0000","last_undegraded":"2023-10-09T06:22:50.225937+0000","last_fullsized":"2023-10-09T06:22:50.225937+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":205,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222592+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:22:50.222592+0000","last_peered":"2023-10-09T06:22:50.222592+0000","last_clean":"2023-10-09T06:22:50.222592+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:22:50.222592+0000","last_undegraded":"2023-10-09T06:22:50.222592+0000","last_fullsized":"2023-10-09T06:22:50.222592+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.222588+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:22:50.222588+0000","last_peered":"2023-10-09T06:22:50.222588+0000","last_clean":"2023-10-09T06:22:50.222588+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:22:50.222588+0000","last_undegraded":"2023-10-09T06:22:50.222588+0000","last_fullsized":"2023-10-09T06:22:50.222588+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":219,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.219287+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:22:50.219287+0000","last_peered":"2023-10-09T06:22:50.219287+0000","last_clean":"2023-10-09T06:22:50.219287+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:22:50.219287+0000","last_undegraded":"2023-10-09T06:22:50.219287+0000","last_fullsized":"2023-10-09T06:22:50.219287+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-10-09T06:22:50.225617+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:22:50.225617+0000","last_peered":"2023-10-09T06:22:50.225617+0000","last_clean":"2023-10-09T06:22:50.225617+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:22:50.225617+0000","last_undegraded":"2023-10-09T06:22:50.225617+0000","last_fullsized":"2023-10-09T06:22:50.225617+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3986751488,"num_objects":60833,"num_object_clones":0,"num_object_copies":182499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60833,"num_write_kb":3893312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12003966976,"data_stored":12003966976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":60833,"ondisk_log_size":60833,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234309,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1845816,"kb_used_data":941488,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":91899336,"statfs":{"total":95995035648,"available":94104920064,"internally_reserved":0,"allocated":964083712,"data_stored":963375402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:40 2023","interfaces":[{"interface":"back","average":{"1min":16.815000000000001,"5min":4.4530000000000003,"15min":2.4119999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":219.41900000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":2.0129999999999999},{"interface":"front","average":{"1min":16.806000000000001,"5min":4.6040000000000001,"15min":2.4670000000000001},"min":{"1min":0.36399999999999999,"5min":0.218,"15min":0.218},"max":{"1min":219.399,"5min":219.399,"15min":219.399},"last":2.0219999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:14 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":2.8599999999999999,"15min":2.1920000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.5390000000000001,"5min":31.869,"15min":31.869},"last":1.419},{"interface":"front","average":{"1min":1.8979999999999999,"5min":2.3690000000000002,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1059999999999999,"5min":29.486999999999998,"15min":29.486999999999998},"last":1.706}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":11.65,"5min":3.5049999999999999,"15min":2.5230000000000001},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":215.34800000000001,"5min":215.34800000000001,"15min":215.34800000000001},"last":3.5649999999999999},{"interface":"front","average":{"1min":11.436,"5min":3.5659999999999998,"15min":2.6160000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":215.869,"5min":215.869,"15min":215.869},"last":1.772}]},{"osd":3,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":2.956,"15min":1.3759999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0899999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":2.0720000000000001},{"interface":"front","average":{"1min":1.2370000000000001,"5min":3.1110000000000002,"15min":1.4530000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.7619999999999996,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.49,"15min":1.276},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.827,"5min":25.678999999999998,"15min":25.678999999999998},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.337,"15min":1.115},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":13.305,"15min":13.305},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.379,"15min":1.6830000000000001},"min":{"1min":0.33300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2690000000000001,"5min":50.304000000000002,"15min":50.304000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.9259999999999999,"15min":1.3260000000000001},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":1.5,"5min":48.774999999999999,"15min":48.774999999999999},"last":0.38100000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.357,"15min":1.254},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":1.5780000000000001,"5min":8.4890000000000008,"15min":11.878},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.63,"5min":1.4039999999999999,"15min":1.2130000000000001},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":1.601,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.46600000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495946,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258988,"kb_used_data":1673316,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":91486164,"statfs":{"total":95995035648,"available":93681831936,"internally_reserved":0,"allocated":1713475584,"data_stored":1712779926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":5.2110000000000003,"15min":2.8119999999999998},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.214},"max":{"1min":21.555,"5min":110.926,"15min":110.926},"last":58.270000000000003},{"interface":"front","average":{"1min":2.3100000000000001,"5min":5.7409999999999997,"15min":3.0920000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.24399999999999999},"max":{"1min":21.535,"5min":110.911,"15min":110.911},"last":58.253999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":5.04,"15min":3.25},"min":{"1min":0.377,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6639999999999997,"5min":114.202,"15min":114.202},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.0669999999999999,"5min":4.6289999999999996,"15min":3.141},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.246},"max":{"1min":4.2679999999999998,"5min":114.179,"15min":114.179},"last":0.47199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.008,"15min":2.2770000000000001},"min":{"1min":0.42799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.6120000000000001,"5min":26.306999999999999,"15min":26.306999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4139999999999999,"15min":2.1640000000000001},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.1050000000000004,"5min":26.292999999999999,"15min":26.292999999999999},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":3.8580000000000001,"15min":1.6459999999999999},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.387,"5min":110.358,"15min":110.358},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":4.0789999999999997,"15min":1.734},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.278,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.415,"15min":1.1870000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":7.2060000000000004,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.111,"15min":1.1279999999999999},"min":{"1min":0.318,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.73,"15min":1.306},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":22.379999999999999,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.626},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.4590000000000001,"15min":1.3049999999999999},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":24.401,"5min":24.401,"15min":24.401},"last":0.54900000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:23:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":2.2029999999999998,"15min":1.3979999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.26200000000000001},"max":{"1min":1.3180000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.343,"15min":1.647},"min":{"1min":0.375,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":3.21,"5min":100.995,"15min":100.995},"last":0.502}]}]},{"osd":1,"up_from":110,"seq":472446402683,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1406612,"kb_used_data":1009484,"kb_used_omap":4,"kb_used_meta":397115,"kb_avail":92338540,"statfs":{"total":95995035648,"available":94554664960,"internally_reserved":0,"allocated":1033711616,"data_stored":1032625181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":406646670},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":2.488,"15min":2.4950000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.242,"5min":148.98400000000001,"15min":148.98400000000001},"last":4.4729999999999999},{"interface":"front","average":{"1min":2.516,"5min":2.7829999999999999,"15min":2.871},"min":{"1min":0.26700000000000002,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":10.516,"5min":149.02699999999999,"15min":149.02699999999999},"last":0.73499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":2.0009999999999999,"15min":1.7529999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":4.2199999999999998,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.2490000000000001,"15min":2.524},"min":{"1min":0.253,"5min":0.219,"15min":0.219},"max":{"1min":10.494,"5min":49.783000000000001,"15min":49.783000000000001},"last":3.4039999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":4.2839999999999998,"15min":1.6819999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2800000000000002,"5min":153.18199999999999,"15min":153.18199999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.2170000000000001,"15min":1.3440000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.5259999999999998,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":3.7120000000000002,"15min":3.032},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.192,"5min":199.43799999999999,"15min":199.43799999999999},"last":3.4620000000000002},{"interface":"front","average":{"1min":2.339,"5min":3.258,"15min":2.9590000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.413,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.313}]},{"osd":5,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.7789999999999999,"15min":2.6389999999999998},"min":{"1min":0.35599999999999998,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":4.1980000000000004,"5min":61.930999999999997,"15min":61.930999999999997},"last":0.441},{"interface":"front","average":{"1min":2.415,"5min":2.1440000000000001,"15min":2.786},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.462,"5min":61.947000000000003,"15min":61.947000000000003},"last":0.374}]},{"osd":6,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.1459999999999999,"15min":2.6600000000000001},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.28399999999999997},"max":{"1min":11.516999999999999,"5min":14.382,"15min":14.382},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.77,"5min":2.363,"15min":2.4529999999999998},"min":{"1min":0.41999999999999998,"5min":0.39200000000000002,"15min":0.374},"max":{"1min":21.036999999999999,"5min":21.036999999999999,"15min":21.036999999999999},"last":1.089}]},{"osd":7,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":2.2330000000000001,"15min":2.5099999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.141999999999999,"5min":17.233000000000001,"15min":17.233000000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":2.468,"5min":1.7110000000000001,"15min":1.9690000000000001},"min":{"1min":0.46300000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":16.161000000000001,"5min":16.161000000000001,"15min":16.161000000000001},"last":0.41599999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024116,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2996440,"kb_used_data":2448656,"kb_used_omap":5,"kb_used_meta":547770,"kb_avail":90748712,"statfs":{"total":95995035648,"available":92926681088,"internally_reserved":0,"allocated":2507423744,"data_stored":2506160785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":560917114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":4.7380000000000004,"15min":2.7999999999999998},"min":{"1min":0.21099999999999999,"5min":0.193,"15min":0.19},"max":{"1min":8.8889999999999993,"5min":185.17400000000001,"15min":185.17400000000001},"last":1.0920000000000001},{"interface":"front","average":{"1min":2.2410000000000001,"5min":4.867,"15min":3.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":10.73,"5min":186.691,"15min":186.691},"last":1.488}]},{"osd":2,"last update":"Mon Oct 9 06:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.6539999999999999,"15min":2.274},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.7709999999999999,"5min":17.588999999999999,"15min":17.588999999999999},"last":1.762},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.77,"15min":2.3140000000000001},"min":{"1min":0.23300000000000001,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":5.4210000000000003,"5min":11.162000000000001,"15min":21.198},"last":0.71299999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":6.4169999999999998,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.1070000000000002,"5min":185.89400000000001,"15min":185.89400000000001},"last":2.0790000000000002},{"interface":"front","average":{"1min":1.234,"5min":6.8620000000000001,"15min":2.5539999999999998},"min":{"1min":0.32600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":6.298,"5min":186.72900000000001,"15min":186.72900000000001},"last":2.383}]},{"osd":4,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.3560000000000001,"15min":1.5940000000000001},"min":{"1min":0.41699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9950000000000001,"5min":11.162000000000001,"15min":15.835000000000001},"last":1.135},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.6910000000000001,"15min":2.0779999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":2.456,"5min":12.752000000000001,"15min":15.023},"last":1.595}]},{"osd":5,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.1059999999999999,"15min":2.032},"min":{"1min":0.41599999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.2450000000000001,"5min":62.984999999999999,"15min":62.984999999999999},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.012,"5min":2.4670000000000001,"15min":2.4550000000000001},"min":{"1min":0.39900000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.367,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.64900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.899,"15min":2.234},"min":{"1min":0.36199999999999999,"5min":0.32000000000000001,"15min":0.29299999999999998},"max":{"1min":3.1749999999999998,"5min":9.6880000000000006,"15min":32.079000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.546,"15min":1.9219999999999999},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":4.9690000000000003,"5min":11.236000000000001,"15min":15.893000000000001},"last":1.1970000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.282,"15min":2.3260000000000001},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":2.8100000000000001,"5min":62.262,"15min":62.262},"last":1.5629999999999999},{"interface":"front","average":{"1min":1.115,"5min":2.3839999999999999,"15min":2.4860000000000002},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":5.875,"5min":62.229999999999997,"15min":62.229999999999997},"last":0.621}]}]},{"osd":2,"up_from":128,"seq":549755813985,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301756,"kb_used_data":2004084,"kb_used_omap":7,"kb_used_meta":297656,"kb_avail":91443396,"statfs":{"total":95995035648,"available":93638037504,"internally_reserved":0,"allocated":2052182016,"data_stored":2051116028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":304799825},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":5.0780000000000003,"15min":2.9580000000000002},"min":{"1min":0.183,"5min":0.157,"15min":0.157},"max":{"1min":5.2549999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":2.1909999999999998},{"interface":"front","average":{"1min":1.8600000000000001,"5min":5.3010000000000002,"15min":3.4830000000000001},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.448,"5min":163.363,"15min":163.363},"last":0.182}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":2.8420000000000001,"15min":2.835},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.188},"max":{"1min":5.5030000000000001,"5min":73.483999999999995,"15min":222.50299999999999},"last":2.1059999999999999},{"interface":"front","average":{"1min":2.331,"5min":3.222,"15min":3.4710000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.420999999999999,"5min":75.641000000000005,"15min":222.19399999999999},"last":3.5670000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.4209999999999998,"15min":1.375},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.5379999999999998,"5min":81.183999999999997,"15min":81.183999999999997},"last":3.532},{"interface":"front","average":{"1min":0.748,"5min":3.4510000000000001,"15min":1.431},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.7890000000000001,"5min":81.061999999999998,"15min":81.061999999999998},"last":1.139}]},{"osd":4,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4870000000000001,"15min":1.8520000000000001},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":2.9710000000000001,"5min":12.106999999999999,"15min":12.141999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.962,"15min":2.1989999999999998},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.7089999999999996,"5min":11.994999999999999,"15min":15.183},"last":1.6399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.304,"15min":1.5720000000000001},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":2.6259999999999999,"5min":5.7199999999999998,"15min":5.7460000000000004},"last":0.318},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.0760000000000001,"15min":2.3380000000000001},"min":{"1min":0.41099999999999998,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":5.6200000000000001,"5min":10.548,"15min":10.548},"last":1.679}]},{"osd":6,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.7589999999999999,"15min":2.0899999999999999},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.1319999999999997,"5min":10.522,"15min":10.522},"last":2.4569999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.2969999999999999,"15min":1.5209999999999999},"min":{"1min":0.41999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.371,"5min":6.883,"15min":7.327},"last":2.1349999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.4810000000000001,"15min":1.605},"min":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.9449999999999998,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":2.1619999999999999},{"interface":"front","average":{"1min":1.569,"5min":1.5629999999999999,"15min":2.5209999999999999},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.4160000000000004,"5min":6.7779999999999996,"15min":9.3249999999999993},"last":0.36499999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127182,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1481632,"kb_used_data":1205592,"kb_used_omap":2,"kb_used_meta":276029,"kb_avail":92263520,"statfs":{"total":95995035648,"available":94477844480,"internally_reserved":0,"allocated":1234526208,"data_stored":1233544335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":282653749},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":8.2959999999999994,"5min":5.3940000000000001,"15min":2.2109999999999999},"min":{"1min":0.23699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":70.349999999999994,"5min":187.852,"15min":187.852},"last":10.336},{"interface":"front","average":{"1min":8.0169999999999995,"5min":6.6180000000000003,"15min":2.8380000000000001},"min":{"1min":0.214,"5min":0.191,"15min":0.191},"max":{"1min":69.978999999999999,"5min":187.923,"15min":187.923},"last":6.1520000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":5.9710000000000001,"15min":2.6219999999999999},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":59.646999999999998,"5min":187.89099999999999,"15min":187.89099999999999},"last":2.4060000000000001},{"interface":"front","average":{"1min":4.3120000000000003,"5min":3.1829999999999998,"15min":1.4670000000000001},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":60.085999999999999,"5min":67.028000000000006,"15min":67.028000000000006},"last":6.8280000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":5.2519999999999998,"15min":2.3799999999999999},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":62.904000000000003,"5min":187.76400000000001,"15min":187.76400000000001},"last":2.4209999999999998},{"interface":"front","average":{"1min":4.4989999999999997,"5min":3.9319999999999999,"15min":1.591},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":62.719000000000001,"5min":187.79499999999999,"15min":187.79499999999999},"last":7.0410000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.4099999999999999,"15min":0.88100000000000001},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2509999999999999,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":1.3240000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":3.278,"15min":1.887},"min":{"1min":0.436,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.722,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":3.7080000000000002,"15min":2.0510000000000002},"min":{"1min":0.46500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.6749999999999998,"5min":187.833,"15min":187.833},"last":1.905},{"interface":"front","average":{"1min":0.81599999999999995,"5min":3.7490000000000001,"15min":2.0680000000000001},"min":{"1min":0.44800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.3820000000000001,"5min":187.952,"15min":187.952},"last":2.3809999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":3.512,"15min":1.9670000000000001},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.7490000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":3.4849999999999999,"15min":1.9590000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4279999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":2.3410000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":3.4660000000000002,"15min":1.5820000000000001},"min":{"1min":0.39200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3,"5min":192.20400000000001,"15min":192.20400000000001},"last":1.052},{"interface":"front","average":{"1min":0.73099999999999998,"5min":3.431,"15min":1.954},"min":{"1min":0.41799999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.774,"5min":187.86500000000001,"15min":187.86500000000001},"last":1.121}]}]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3251056,"kb_used_data":2351976,"kb_used_omap":0,"kb_used_meta":899072,"kb_avail":90494096,"statfs":{"total":95995035648,"available":92665954304,"internally_reserved":0,"allocated":2408423424,"data_stored":2407722971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920649728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3069999999999999,"15min":1.476},"min":{"1min":0.315,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":1.798,"5min":19.577000000000002,"15min":19.577000000000002},"last":5.1139999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.7909999999999999,"15min":1.869},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":3.1560000000000001,"5min":19.594999999999999,"15min":19.594999999999999},"last":5.0999999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":3.2909999999999999,"15min":2.1629999999999998},"min":{"1min":0.23699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.0190000000000001,"5min":91.849999999999994,"15min":91.849999999999994},"last":1.365},{"interface":"front","average":{"1min":0.88600000000000001,"5min":3.4199999999999999,"15min":2.206},"min":{"1min":0.34799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.117,"5min":91.885000000000005,"15min":91.885000000000005},"last":1.4950000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5589999999999999,"15min":1.75},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":10.884,"5min":10.884,"15min":10.884},"last":0.315},{"interface":"front","average":{"1min":1.026,"5min":1.821,"15min":1.8799999999999999},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.5850000000000009,"5min":15.894,"15min":15.894},"last":4.1429999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":4.5049999999999999,"15min":1.877},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.4749999999999996,"5min":91.697000000000003,"15min":91.697000000000003},"last":8.0530000000000008},{"interface":"front","average":{"1min":0.69999999999999996,"5min":4.2279999999999998,"15min":1.8240000000000001},"min":{"1min":0.38900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1850000000000001,"5min":94.356999999999999,"15min":94.356999999999999},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.1950000000000001,"15min":1.0669999999999999},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":9.1720000000000006,"5min":10.673,"15min":10.673},"last":0.38},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.147,"15min":1.246},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.24399999999999999},"max":{"1min":9.6379999999999999,"5min":12.552,"15min":12.552},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.78800000000000003,"15min":0.84399999999999997},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.881,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.242,"15min":1.171},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.0029999999999999,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.443}]},{"osd":7,"last update":"Mon Oct 9 06:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":2.3900000000000001,"15min":1.427},"min":{"1min":0.36199999999999999,"5min":0.308,"15min":0.308},"max":{"1min":1.7370000000000001,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":2.3210000000000002,"15min":1.4650000000000001},"min":{"1min":0.42699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.704,"5min":62.573,"15min":62.573},"last":0.42799999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757583,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2537029632,"data_stored":2537029632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1036517376,"data_stored":1036517376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2061828096,"data_stored":2061828096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1243283456,"data_stored":1243283456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2437218304,"data_stored":2437218304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1710555136,"data_stored":1710555136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":977534976,"data_stored":977534976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:23:09.740 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:23:09.740 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:23:09.740 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T06:23:09.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:23:09.742 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 66493 66477 94.357 67.875 0.00872311 0.0105934 2023-10-09T06:23:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[220921]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: osdmap e249: 8 total, 8 up, 7 in 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:09 smithi150 ceph-mon[194178]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:09.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:09 smithi107 ceph-mon[212750]: failed to encode map e249 with expected crc 2023-10-09T06:23:10.024 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:10.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:10.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:10.143 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:10.175 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:10.239 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:10.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:10.349 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:10.423 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:10.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[212750]: pgmap v771: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:23:10.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[220921]: pgmap v771: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:23:10.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.736 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 67526 67510 93.6942 64.5625 0.00891114 0.0106685 2023-10-09T06:23:10.792 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:10.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:10.903 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:10 smithi150 ceph-mon[194178]: pgmap v771: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:23:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:10.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:11.003 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:11.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:11.111 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:11.175 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:11.237 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:11.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:11.378 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:23:11.379 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:23:11.379 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:23:11.379 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:23:11.379 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:23:11.444 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:11.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:11.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:11.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:11.712 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:11.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:11.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:11.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:11.736 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 68546 68530 93.0435 63.75 0.0229818 0.0107432 2023-10-09T06:23:11.751 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:11.796 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:11.834 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:11.872 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:23:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:23:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:11.908 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:11.948 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:11.984 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:11.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:11.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:11.992 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:12.083 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:23:12.092 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:12.118 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:12.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:12.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:23:12.162 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:12.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:12.268 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:12.365 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:12] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T06:23:12.372 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:12.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:23:12.736 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 69481 69465 92.3076 58.4375 0.0107774 0.0108281 2023-10-09T06:23:13.103 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:12 smithi107 ceph-mon[220921]: pgmap v772: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:23:13.103 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2673838573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:13.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:12 smithi107 ceph-mon[212750]: pgmap v772: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:23:13.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2673838573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:12 smithi150 ceph-mon[194178]: pgmap v772: 99 pgs: 99 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:23:13.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2673838573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:13.252 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:13.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:13.359 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:13.392 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:13.470 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:13.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:13.575 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:13.619 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:13.662 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:13.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:13.736 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 70324 70308 91.4825 52.6875 0.00566996 0.0109012 2023-10-09T06:23:13.768 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:13.796 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:14.086 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:14.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:14.195 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:14.254 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:14.301 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:14.366 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:14.506 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:14.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:14.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:14.619 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:14.701 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:14.737 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 71304 71288 90.8647 61.25 0.0131463 0.011001 2023-10-09T06:23:14.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:14 smithi107 ceph-mon[220921]: pgmap v773: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:23:14.797 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:14 smithi107 ceph-mon[212750]: pgmap v773: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:23:15.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:14 smithi150 ceph-mon[194178]: pgmap v773: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T06:23:15.237 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:23:15.254 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:23:15.304 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:15.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:15.390 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:23:15.412 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:15.443 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:15.513 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:15.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:15.622 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:15.667 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:15.737 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:15.737 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 72262 72246 90.2452 59.875 0.0166537 0.0110767 2023-10-09T06:23:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:15.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:15.847 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:15.875 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:16.737 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 73047 73031 89.4381 49.0625 0.180742 0.0111625 2023-10-09T06:23:16.802 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: pgmap v774: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: pgmap v774: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T06:23:16.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:23:16.804 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:23:16.804 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:16.920 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:16.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:17.033 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:17.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: pgmap v774: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:23:17.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: from='client.45704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:23:17.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-10-09T06:23:17.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-09T06:23:17.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-09T06:23:17.069 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:17.078 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:17.148 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:17.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:17.261 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:17.339 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:17.574 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:17.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:17.616 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":774,"stamp":"2023-10-09T06:23:15.054530+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4500494240,"num_objects":68687,"num_object_clones":0,"num_object_copies":206061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":68724,"num_write_kb":4394461,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":337988,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70489,"ondisk_log_size":70489,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17029792,"kb_used_data":13114216,"kb_used_omap":20,"kb_used_meta":3915499,"kb_avail":639186272,"statfs":{"total":671965249536,"available":654526742528,"internally_reserved":0,"allocated":13428957184,"data_stored":13422443003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4009471630},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,8,1,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":665976832,"num_objects":10162,"num_object_clones":0,"num_object_copies":30486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10162,"num_write_kb":650368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002488"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280161+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:22:57.280161+0000","last_peered":"2023-10-09T06:22:57.280161+0000","last_clean":"2023-10-09T06:22:57.280161+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:22:57.280161+0000","last_undegraded":"2023-10-09T06:22:57.280161+0000","last_fullsized":"2023-10-09T06:22:57.280161+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":185,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294967+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:22:57.294967+0000","last_peered":"2023-10-09T06:22:57.294967+0000","last_clean":"2023-10-09T06:22:57.294967+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:22:57.294967+0000","last_undegraded":"2023-10-09T06:22:57.294967+0000","last_fullsized":"2023-10-09T06:22:57.294967+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277512+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:22:57.277512+0000","last_peered":"2023-10-09T06:22:57.277512+0000","last_clean":"2023-10-09T06:22:57.277512+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:22:57.277512+0000","last_undegraded":"2023-10-09T06:22:57.277512+0000","last_fullsized":"2023-10-09T06:22:57.277512+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":219,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280981+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:22:57.280981+0000","last_peered":"2023-10-09T06:22:57.280981+0000","last_clean":"2023-10-09T06:22:57.280981+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:22:57.280981+0000","last_undegraded":"2023-10-09T06:22:57.280981+0000","last_fullsized":"2023-10-09T06:22:57.280981+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":203,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294675+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:22:57.294675+0000","last_peered":"2023-10-09T06:22:57.294675+0000","last_clean":"2023-10-09T06:22:57.294675+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:22:57.294675+0000","last_undegraded":"2023-10-09T06:22:57.294675+0000","last_fullsized":"2023-10-09T06:22:57.294675+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":218,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278285+0000","last_change":"2023-10-09T06:21:45.411474+0000","last_active":"2023-10-09T06:22:57.278285+0000","last_peered":"2023-10-09T06:22:57.278285+0000","last_clean":"2023-10-09T06:22:57.278285+0000","last_became_active":"2023-10-09T06:21:45.410968+0000","last_became_peered":"2023-10-09T06:21:45.410968+0000","last_unstale":"2023-10-09T06:22:57.278285+0000","last_undegraded":"2023-10-09T06:22:57.278285+0000","last_fullsized":"2023-10-09T06:22:57.278285+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:53.196763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280653+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:22:57.280653+0000","last_peered":"2023-10-09T06:22:57.280653+0000","last_clean":"2023-10-09T06:22:57.280653+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:22:57.280653+0000","last_undegraded":"2023-10-09T06:22:57.280653+0000","last_fullsized":"2023-10-09T06:22:57.280653+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":185,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294657+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:22:57.294657+0000","last_peered":"2023-10-09T06:22:57.294657+0000","last_clean":"2023-10-09T06:22:57.294657+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:22:57.294657+0000","last_undegraded":"2023-10-09T06:22:57.294657+0000","last_fullsized":"2023-10-09T06:22:57.294657+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.306242+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:22:57.306242+0000","last_peered":"2023-10-09T06:22:57.306242+0000","last_clean":"2023-10-09T06:22:57.306242+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:22:57.306242+0000","last_undegraded":"2023-10-09T06:22:57.306242+0000","last_fullsized":"2023-10-09T06:22:57.306242+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":183,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294649+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:22:57.294649+0000","last_peered":"2023-10-09T06:22:57.294649+0000","last_clean":"2023-10-09T06:22:57.294649+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:22:57.294649+0000","last_undegraded":"2023-10-09T06:22:57.294649+0000","last_fullsized":"2023-10-09T06:22:57.294649+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":98,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.306131+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:22:57.306131+0000","last_peered":"2023-10-09T06:22:57.306131+0000","last_clean":"2023-10-09T06:22:57.306131+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:22:57.306131+0000","last_undegraded":"2023-10-09T06:22:57.306131+0000","last_fullsized":"2023-10-09T06:22:57.306131+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":179,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278587+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:22:57.278587+0000","last_peered":"2023-10-09T06:22:57.278587+0000","last_clean":"2023-10-09T06:22:57.278587+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:22:57.278587+0000","last_undegraded":"2023-10-09T06:22:57.278587+0000","last_fullsized":"2023-10-09T06:22:57.278587+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":255,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280576+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:22:57.280576+0000","last_peered":"2023-10-09T06:22:57.280576+0000","last_clean":"2023-10-09T06:22:57.280576+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:22:57.280576+0000","last_undegraded":"2023-10-09T06:22:57.280576+0000","last_fullsized":"2023-10-09T06:22:57.280576+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280607+0000","last_change":"2023-10-09T06:13:02.834858+0000","last_active":"2023-10-09T06:22:57.280607+0000","last_peered":"2023-10-09T06:22:57.280607+0000","last_clean":"2023-10-09T06:22:57.280607+0000","last_became_active":"2023-10-09T06:13:02.834569+0000","last_became_peered":"2023-10-09T06:13:02.834569+0000","last_unstale":"2023-10-09T06:22:57.280607+0000","last_undegraded":"2023-10-09T06:22:57.280607+0000","last_fullsized":"2023-10-09T06:22:57.280607+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:10.946236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":98,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305798+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:22:57.305798+0000","last_peered":"2023-10-09T06:22:57.305798+0000","last_clean":"2023-10-09T06:22:57.305798+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:22:57.305798+0000","last_undegraded":"2023-10-09T06:22:57.305798+0000","last_fullsized":"2023-10-09T06:22:57.305798+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":203,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.296634+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:22:57.296634+0000","last_peered":"2023-10-09T06:22:57.296634+0000","last_clean":"2023-10-09T06:22:57.296634+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:22:57.296634+0000","last_undegraded":"2023-10-09T06:22:57.296634+0000","last_fullsized":"2023-10-09T06:22:57.296634+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":218,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.281416+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:22:57.281416+0000","last_peered":"2023-10-09T06:22:57.281416+0000","last_clean":"2023-10-09T06:22:57.281416+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:22:57.281416+0000","last_undegraded":"2023-10-09T06:22:57.281416+0000","last_fullsized":"2023-10-09T06:22:57.281416+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":176,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.306055+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:22:57.306055+0000","last_peered":"2023-10-09T06:22:57.306055+0000","last_clean":"2023-10-09T06:22:57.306055+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:22:57.306055+0000","last_undegraded":"2023-10-09T06:22:57.306055+0000","last_fullsized":"2023-10-09T06:22:57.306055+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305614+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:22:57.305614+0000","last_peered":"2023-10-09T06:22:57.305614+0000","last_clean":"2023-10-09T06:22:57.305614+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:22:57.305614+0000","last_undegraded":"2023-10-09T06:22:57.305614+0000","last_fullsized":"2023-10-09T06:22:57.305614+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":218,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.281067+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:22:57.281067+0000","last_peered":"2023-10-09T06:22:57.281067+0000","last_clean":"2023-10-09T06:22:57.281067+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:22:57.281067+0000","last_undegraded":"2023-10-09T06:22:57.281067+0000","last_fullsized":"2023-10-09T06:22:57.281067+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":237,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277839+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:22:57.277839+0000","last_peered":"2023-10-09T06:22:57.277839+0000","last_clean":"2023-10-09T06:22:57.277839+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:22:57.277839+0000","last_undegraded":"2023-10-09T06:22:57.277839+0000","last_fullsized":"2023-10-09T06:22:57.277839+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":203,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.297648+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:22:57.297648+0000","last_peered":"2023-10-09T06:22:57.297648+0000","last_clean":"2023-10-09T06:22:57.297648+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:22:57.297648+0000","last_undegraded":"2023-10-09T06:22:57.297648+0000","last_fullsized":"2023-10-09T06:22:57.297648+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"249'4131","reported_seq":4151,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.058029+0000","last_change":"2023-10-09T06:22:15.195712+0000","last_active":"2023-10-09T06:23:10.058029+0000","last_peered":"2023-10-09T06:23:10.058029+0000","last_clean":"2023-10-09T06:23:10.058029+0000","last_became_active":"2023-10-09T06:22:15.195422+0000","last_became_peered":"2023-10-09T06:22:15.195422+0000","last_unstale":"2023-10-09T06:23:10.058029+0000","last_undegraded":"2023-10-09T06:23:10.058029+0000","last_fullsized":"2023-10-09T06:23:10.058029+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4131,"log_dups_size":0,"ondisk_log_size":4131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:32.535427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270729216,"num_objects":4131,"num_object_clones":0,"num_object_copies":12393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4131,"num_write_kb":264384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":96,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.307002+0000","last_change":"2023-10-09T06:19:06.791173+0000","last_active":"2023-10-09T06:22:57.307002+0000","last_peered":"2023-10-09T06:22:57.307002+0000","last_clean":"2023-10-09T06:22:57.307002+0000","last_became_active":"2023-10-09T06:19:06.757340+0000","last_became_peered":"2023-10-09T06:19:06.757340+0000","last_unstale":"2023-10-09T06:22:57.307002+0000","last_undegraded":"2023-10-09T06:22:57.307002+0000","last_fullsized":"2023-10-09T06:22:57.307002+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:36:41.454642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305497+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:22:57.305497+0000","last_peered":"2023-10-09T06:22:57.305497+0000","last_clean":"2023-10-09T06:22:57.305497+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:22:57.305497+0000","last_undegraded":"2023-10-09T06:22:57.305497+0000","last_fullsized":"2023-10-09T06:22:57.305497+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"249'4298","reported_seq":4317,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.798287+0000","last_change":"2023-10-09T06:22:15.412855+0000","last_active":"2023-10-09T06:23:12.798287+0000","last_peered":"2023-10-09T06:23:12.798287+0000","last_clean":"2023-10-09T06:23:12.798287+0000","last_became_active":"2023-10-09T06:22:15.412520+0000","last_became_peered":"2023-10-09T06:22:15.412520+0000","last_unstale":"2023-10-09T06:23:12.798287+0000","last_undegraded":"2023-10-09T06:23:12.798287+0000","last_fullsized":"2023-10-09T06:23:12.798287+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4298,"log_dups_size":0,"ondisk_log_size":4298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:49.054096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281673728,"num_objects":4298,"num_object_clones":0,"num_object_copies":12894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4298,"num_write_kb":275072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":96,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.307177+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:22:57.307177+0000","last_peered":"2023-10-09T06:22:57.307177+0000","last_clean":"2023-10-09T06:22:57.307177+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:22:57.307177+0000","last_undegraded":"2023-10-09T06:22:57.307177+0000","last_fullsized":"2023-10-09T06:22:57.307177+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":226,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280265+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:22:57.280265+0000","last_peered":"2023-10-09T06:22:57.280265+0000","last_clean":"2023-10-09T06:22:57.280265+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:22:57.280265+0000","last_undegraded":"2023-10-09T06:22:57.280265+0000","last_fullsized":"2023-10-09T06:22:57.280265+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"249'4245","reported_seq":4265,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.058184+0000","last_change":"2023-10-09T06:22:15.411974+0000","last_active":"2023-10-09T06:23:10.058184+0000","last_peered":"2023-10-09T06:23:10.058184+0000","last_clean":"2023-10-09T06:23:10.058184+0000","last_became_active":"2023-10-09T06:22:15.411649+0000","last_became_peered":"2023-10-09T06:22:15.411649+0000","last_unstale":"2023-10-09T06:23:10.058184+0000","last_undegraded":"2023-10-09T06:23:10.058184+0000","last_fullsized":"2023-10-09T06:23:10.058184+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4245,"log_dups_size":0,"ondisk_log_size":4245,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:27.266507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278200320,"num_objects":4245,"num_object_clones":0,"num_object_copies":12735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4245,"num_write_kb":271680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":110,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.306022+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:22:57.306022+0000","last_peered":"2023-10-09T06:22:57.306022+0000","last_clean":"2023-10-09T06:22:57.306022+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:22:57.306022+0000","last_undegraded":"2023-10-09T06:22:57.306022+0000","last_fullsized":"2023-10-09T06:22:57.306022+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278600+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:22:57.278600+0000","last_peered":"2023-10-09T06:22:57.278600+0000","last_clean":"2023-10-09T06:22:57.278600+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:22:57.278600+0000","last_undegraded":"2023-10-09T06:22:57.278600+0000","last_fullsized":"2023-10-09T06:22:57.278600+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"249'4208","reported_seq":4228,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.988425+0000","last_change":"2023-10-09T06:22:15.411313+0000","last_active":"2023-10-09T06:23:10.988425+0000","last_peered":"2023-10-09T06:23:10.988425+0000","last_clean":"2023-10-09T06:23:10.988425+0000","last_became_active":"2023-10-09T06:22:15.410917+0000","last_became_peered":"2023-10-09T06:22:15.410917+0000","last_unstale":"2023-10-09T06:23:10.988425+0000","last_undegraded":"2023-10-09T06:23:10.988425+0000","last_fullsized":"2023-10-09T06:23:10.988425+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4208,"log_dups_size":0,"ondisk_log_size":4208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:24.915932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4208,"num_write_kb":269312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":139,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.295737+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:22:57.295737+0000","last_peered":"2023-10-09T06:22:57.295737+0000","last_clean":"2023-10-09T06:22:57.295737+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:22:57.295737+0000","last_undegraded":"2023-10-09T06:22:57.295737+0000","last_fullsized":"2023-10-09T06:22:57.295737+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":200,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280286+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:22:57.280286+0000","last_peered":"2023-10-09T06:22:57.280286+0000","last_clean":"2023-10-09T06:22:57.280286+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:22:57.280286+0000","last_undegraded":"2023-10-09T06:22:57.280286+0000","last_fullsized":"2023-10-09T06:22:57.280286+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"249'4242","reported_seq":4262,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:11.702307+0000","last_change":"2023-10-09T06:22:15.196669+0000","last_active":"2023-10-09T06:23:11.702307+0000","last_peered":"2023-10-09T06:23:11.702307+0000","last_clean":"2023-10-09T06:23:11.702307+0000","last_became_active":"2023-10-09T06:22:15.196399+0000","last_became_peered":"2023-10-09T06:22:15.196399+0000","last_unstale":"2023-10-09T06:23:11.702307+0000","last_undegraded":"2023-10-09T06:23:11.702307+0000","last_fullsized":"2023-10-09T06:23:11.702307+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4242,"log_dups_size":0,"ondisk_log_size":4242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.953962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003712,"num_objects":4242,"num_object_clones":0,"num_object_copies":12726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4242,"num_write_kb":271488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":541,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305537+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:22:57.305537+0000","last_peered":"2023-10-09T06:22:57.305537+0000","last_clean":"2023-10-09T06:22:57.305537+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:22:57.305537+0000","last_undegraded":"2023-10-09T06:22:57.305537+0000","last_fullsized":"2023-10-09T06:22:57.305537+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":211,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.295518+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:22:57.295518+0000","last_peered":"2023-10-09T06:22:57.295518+0000","last_clean":"2023-10-09T06:22:57.295518+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:22:57.295518+0000","last_undegraded":"2023-10-09T06:22:57.295518+0000","last_fullsized":"2023-10-09T06:22:57.295518+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280653+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:22:57.280653+0000","last_peered":"2023-10-09T06:22:57.280653+0000","last_clean":"2023-10-09T06:22:57.280653+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:22:57.280653+0000","last_undegraded":"2023-10-09T06:22:57.280653+0000","last_fullsized":"2023-10-09T06:22:57.280653+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"249'4244","reported_seq":4264,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.950786+0000","last_change":"2023-10-09T06:22:15.411262+0000","last_active":"2023-10-09T06:23:10.950786+0000","last_peered":"2023-10-09T06:23:10.950786+0000","last_clean":"2023-10-09T06:23:10.950786+0000","last_became_active":"2023-10-09T06:22:15.410925+0000","last_became_peered":"2023-10-09T06:22:15.410925+0000","last_unstale":"2023-10-09T06:23:10.950786+0000","last_undegraded":"2023-10-09T06:23:10.950786+0000","last_fullsized":"2023-10-09T06:23:10.950786+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4244,"log_dups_size":0,"ondisk_log_size":4244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:51.768992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278134784,"num_objects":4244,"num_object_clones":0,"num_object_copies":12732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4244,"num_write_kb":271616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":531,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.300040+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:22:57.300040+0000","last_peered":"2023-10-09T06:22:57.300040+0000","last_clean":"2023-10-09T06:22:57.300040+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:22:57.300040+0000","last_undegraded":"2023-10-09T06:22:57.300040+0000","last_fullsized":"2023-10-09T06:22:57.300040+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278129+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:22:57.278129+0000","last_peered":"2023-10-09T06:22:57.278129+0000","last_clean":"2023-10-09T06:22:57.278129+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:22:57.278129+0000","last_undegraded":"2023-10-09T06:22:57.278129+0000","last_fullsized":"2023-10-09T06:22:57.278129+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":226,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280848+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:22:57.280848+0000","last_peered":"2023-10-09T06:22:57.280848+0000","last_clean":"2023-10-09T06:22:57.280848+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:22:57.280848+0000","last_undegraded":"2023-10-09T06:22:57.280848+0000","last_fullsized":"2023-10-09T06:22:57.280848+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"249'4339","reported_seq":4358,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.766572+0000","last_change":"2023-10-09T06:22:15.196292+0000","last_active":"2023-10-09T06:23:12.766572+0000","last_peered":"2023-10-09T06:23:12.766572+0000","last_clean":"2023-10-09T06:23:12.766572+0000","last_became_active":"2023-10-09T06:22:15.195400+0000","last_became_peered":"2023-10-09T06:22:15.195400+0000","last_unstale":"2023-10-09T06:23:12.766572+0000","last_undegraded":"2023-10-09T06:23:12.766572+0000","last_fullsized":"2023-10-09T06:23:12.766572+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4339,"log_dups_size":0,"ondisk_log_size":4339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:20.452846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284360704,"num_objects":4339,"num_object_clones":0,"num_object_copies":13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4339,"num_write_kb":277696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280172+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:22:57.280172+0000","last_peered":"2023-10-09T06:22:57.280172+0000","last_clean":"2023-10-09T06:22:57.280172+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:22:57.280172+0000","last_undegraded":"2023-10-09T06:22:57.280172+0000","last_fullsized":"2023-10-09T06:22:57.280172+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.281785+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:22:57.281785+0000","last_peered":"2023-10-09T06:22:57.281785+0000","last_clean":"2023-10-09T06:22:57.281785+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:22:57.281785+0000","last_undegraded":"2023-10-09T06:22:57.281785+0000","last_fullsized":"2023-10-09T06:22:57.281785+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"249'4432","reported_seq":4451,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.773999+0000","last_change":"2023-10-09T06:22:15.195487+0000","last_active":"2023-10-09T06:23:12.773999+0000","last_peered":"2023-10-09T06:23:12.773999+0000","last_clean":"2023-10-09T06:23:12.773999+0000","last_became_active":"2023-10-09T06:22:15.195089+0000","last_became_peered":"2023-10-09T06:22:15.195089+0000","last_unstale":"2023-10-09T06:23:12.773999+0000","last_undegraded":"2023-10-09T06:23:12.773999+0000","last_fullsized":"2023-10-09T06:23:12.773999+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4432,"log_dups_size":0,"ondisk_log_size":4432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:34.432191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290455552,"num_objects":4432,"num_object_clones":0,"num_object_copies":13296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4432,"num_write_kb":283648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":527,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.302458+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:22:57.302458+0000","last_peered":"2023-10-09T06:22:57.302458+0000","last_clean":"2023-10-09T06:22:57.302458+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:22:57.302458+0000","last_undegraded":"2023-10-09T06:22:57.302458+0000","last_fullsized":"2023-10-09T06:22:57.302458+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":205,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:08.730931+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:23:08.730931+0000","last_peered":"2023-10-09T06:23:08.730931+0000","last_clean":"2023-10-09T06:23:08.730931+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:23:08.730931+0000","last_undegraded":"2023-10-09T06:23:08.730931+0000","last_fullsized":"2023-10-09T06:23:08.730931+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":179,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277645+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:22:57.277645+0000","last_peered":"2023-10-09T06:22:57.277645+0000","last_clean":"2023-10-09T06:22:57.277645+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:22:57.277645+0000","last_undegraded":"2023-10-09T06:22:57.277645+0000","last_fullsized":"2023-10-09T06:22:57.277645+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"249'4220","reported_seq":4239,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:13.094270+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:23:13.094270+0000","last_peered":"2023-10-09T06:23:13.094270+0000","last_clean":"2023-10-09T06:23:13.094270+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:23:13.094270+0000","last_undegraded":"2023-10-09T06:23:13.094270+0000","last_fullsized":"2023-10-09T06:23:13.094270+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4220,"log_dups_size":0,"ondisk_log_size":4220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4220,"num_write_kb":270080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":471,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278651+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:22:57.278651+0000","last_peered":"2023-10-09T06:22:57.278651+0000","last_clean":"2023-10-09T06:22:57.278651+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:22:57.278651+0000","last_undegraded":"2023-10-09T06:22:57.278651+0000","last_fullsized":"2023-10-09T06:22:57.278651+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":230,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277789+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:22:57.277789+0000","last_peered":"2023-10-09T06:22:57.277789+0000","last_clean":"2023-10-09T06:22:57.277789+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:22:57.277789+0000","last_undegraded":"2023-10-09T06:22:57.277789+0000","last_fullsized":"2023-10-09T06:22:57.277789+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":156,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.295108+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:22:57.295108+0000","last_peered":"2023-10-09T06:22:57.295108+0000","last_clean":"2023-10-09T06:22:57.295108+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:22:57.295108+0000","last_undegraded":"2023-10-09T06:22:57.295108+0000","last_fullsized":"2023-10-09T06:22:57.295108+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":173,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278165+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:22:57.278165+0000","last_peered":"2023-10-09T06:22:57.278165+0000","last_clean":"2023-10-09T06:22:57.278165+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:22:57.278165+0000","last_undegraded":"2023-10-09T06:22:57.278165+0000","last_fullsized":"2023-10-09T06:22:57.278165+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"249'4388","reported_seq":4407,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.769042+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:23:12.769042+0000","last_peered":"2023-10-09T06:23:12.769042+0000","last_clean":"2023-10-09T06:23:12.769042+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:23:12.769042+0000","last_undegraded":"2023-10-09T06:23:12.769042+0000","last_fullsized":"2023-10-09T06:23:12.769042+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4388,"log_dups_size":0,"ondisk_log_size":4388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287571968,"num_objects":4388,"num_object_clones":0,"num_object_copies":13164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4388,"num_write_kb":280832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":532,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.283818+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:22:57.283818+0000","last_peered":"2023-10-09T06:22:57.283818+0000","last_clean":"2023-10-09T06:22:57.283818+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:22:57.283818+0000","last_undegraded":"2023-10-09T06:22:57.283818+0000","last_fullsized":"2023-10-09T06:22:57.283818+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":104,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278311+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:22:57.278311+0000","last_peered":"2023-10-09T06:22:57.278311+0000","last_clean":"2023-10-09T06:22:57.278311+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:22:57.278311+0000","last_undegraded":"2023-10-09T06:22:57.278311+0000","last_fullsized":"2023-10-09T06:22:57.278311+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":210,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280484+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:22:57.280484+0000","last_peered":"2023-10-09T06:22:57.280484+0000","last_clean":"2023-10-09T06:22:57.280484+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:22:57.280484+0000","last_undegraded":"2023-10-09T06:22:57.280484+0000","last_fullsized":"2023-10-09T06:22:57.280484+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":194,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277731+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:22:57.277731+0000","last_peered":"2023-10-09T06:22:57.277731+0000","last_clean":"2023-10-09T06:22:57.277731+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:22:57.277731+0000","last_undegraded":"2023-10-09T06:22:57.277731+0000","last_fullsized":"2023-10-09T06:22:57.277731+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"249'4253","reported_seq":4273,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.979599+0000","last_change":"2023-10-09T06:22:15.195697+0000","last_active":"2023-10-09T06:23:10.979599+0000","last_peered":"2023-10-09T06:23:10.979599+0000","last_clean":"2023-10-09T06:23:10.979599+0000","last_became_active":"2023-10-09T06:22:15.195391+0000","last_became_peered":"2023-10-09T06:22:15.195391+0000","last_unstale":"2023-10-09T06:23:10.979599+0000","last_undegraded":"2023-10-09T06:23:10.979599+0000","last_fullsized":"2023-10-09T06:23:10.979599+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4253,"log_dups_size":0,"ondisk_log_size":4253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:00.178768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278724608,"num_objects":4253,"num_object_clones":0,"num_object_copies":12759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4253,"num_write_kb":272192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":541,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305611+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:22:57.305611+0000","last_peered":"2023-10-09T06:22:57.305611+0000","last_clean":"2023-10-09T06:22:57.305611+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:22:57.305611+0000","last_undegraded":"2023-10-09T06:22:57.305611+0000","last_fullsized":"2023-10-09T06:22:57.305611+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":213,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.300512+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:22:57.300512+0000","last_peered":"2023-10-09T06:22:57.300512+0000","last_clean":"2023-10-09T06:22:57.300512+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:22:57.300512+0000","last_undegraded":"2023-10-09T06:22:57.300512+0000","last_fullsized":"2023-10-09T06:22:57.300512+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":221,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:23:01.722060+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:23:01.722060+0000","last_peered":"2023-10-09T06:23:01.722060+0000","last_clean":"2023-10-09T06:23:01.722060+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:23:01.722060+0000","last_undegraded":"2023-10-09T06:23:01.722060+0000","last_fullsized":"2023-10-09T06:23:01.722060+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":216,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277964+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:22:57.277964+0000","last_peered":"2023-10-09T06:22:57.277964+0000","last_clean":"2023-10-09T06:22:57.277964+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:22:57.277964+0000","last_undegraded":"2023-10-09T06:22:57.277964+0000","last_fullsized":"2023-10-09T06:22:57.277964+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"249'4374","reported_seq":4393,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.745689+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:23:12.745689+0000","last_peered":"2023-10-09T06:23:12.745689+0000","last_clean":"2023-10-09T06:23:12.745689+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:23:12.745689+0000","last_undegraded":"2023-10-09T06:23:12.745689+0000","last_fullsized":"2023-10-09T06:23:12.745689+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4374,"log_dups_size":0,"ondisk_log_size":4374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4374,"num_write_kb":279936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":180,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.302708+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:22:57.302708+0000","last_peered":"2023-10-09T06:22:57.302708+0000","last_clean":"2023-10-09T06:22:57.302708+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:22:57.302708+0000","last_undegraded":"2023-10-09T06:22:57.302708+0000","last_fullsized":"2023-10-09T06:22:57.302708+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":522,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305508+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:22:57.305508+0000","last_peered":"2023-10-09T06:22:57.305508+0000","last_clean":"2023-10-09T06:22:57.305508+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:22:57.305508+0000","last_undegraded":"2023-10-09T06:22:57.305508+0000","last_fullsized":"2023-10-09T06:22:57.305508+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:58.752596+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:22:58.752596+0000","last_peered":"2023-10-09T06:22:58.752596+0000","last_clean":"2023-10-09T06:22:58.752596+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:22:58.752596+0000","last_undegraded":"2023-10-09T06:22:58.752596+0000","last_fullsized":"2023-10-09T06:22:58.752596+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":211,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278395+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:22:57.278395+0000","last_peered":"2023-10-09T06:22:57.278395+0000","last_clean":"2023-10-09T06:22:57.278395+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:22:57.278395+0000","last_undegraded":"2023-10-09T06:22:57.278395+0000","last_fullsized":"2023-10-09T06:22:57.278395+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"249'4168","reported_seq":4188,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:10.040270+0000","last_change":"2023-10-09T06:22:15.192849+0000","last_active":"2023-10-09T06:23:10.040270+0000","last_peered":"2023-10-09T06:23:10.040270+0000","last_clean":"2023-10-09T06:23:10.040270+0000","last_became_active":"2023-10-09T06:22:15.192540+0000","last_became_peered":"2023-10-09T06:22:15.192540+0000","last_unstale":"2023-10-09T06:23:10.040270+0000","last_undegraded":"2023-10-09T06:23:10.040270+0000","last_fullsized":"2023-10-09T06:23:10.040270+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4168,"log_dups_size":0,"ondisk_log_size":4168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:21.157073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273154048,"num_objects":4168,"num_object_clones":0,"num_object_copies":12504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4168,"num_write_kb":266752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":530,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278054+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:22:57.278054+0000","last_peered":"2023-10-09T06:22:57.278054+0000","last_clean":"2023-10-09T06:22:57.278054+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:22:57.278054+0000","last_undegraded":"2023-10-09T06:22:57.278054+0000","last_fullsized":"2023-10-09T06:22:57.278054+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":92,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.305418+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:22:57.305418+0000","last_peered":"2023-10-09T06:22:57.305418+0000","last_clean":"2023-10-09T06:22:57.305418+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:22:57.305418+0000","last_undegraded":"2023-10-09T06:22:57.305418+0000","last_fullsized":"2023-10-09T06:22:57.305418+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.283901+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:22:57.283901+0000","last_peered":"2023-10-09T06:22:57.283901+0000","last_clean":"2023-10-09T06:22:57.283901+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:22:57.283901+0000","last_undegraded":"2023-10-09T06:22:57.283901+0000","last_fullsized":"2023-10-09T06:22:57.283901+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":223,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280979+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:22:57.280979+0000","last_peered":"2023-10-09T06:22:57.280979+0000","last_clean":"2023-10-09T06:22:57.280979+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:22:57.280979+0000","last_undegraded":"2023-10-09T06:22:57.280979+0000","last_fullsized":"2023-10-09T06:22:57.280979+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"249'4402","reported_seq":4421,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.787958+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:23:12.787958+0000","last_peered":"2023-10-09T06:23:12.787958+0000","last_clean":"2023-10-09T06:23:12.787958+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:23:12.787958+0000","last_undegraded":"2023-10-09T06:23:12.787958+0000","last_fullsized":"2023-10-09T06:23:12.787958+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4402,"log_dups_size":0,"ondisk_log_size":4402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288489472,"num_objects":4402,"num_object_clones":0,"num_object_copies":13206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4402,"num_write_kb":281728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":471,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278285+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:22:57.278285+0000","last_peered":"2023-10-09T06:22:57.278285+0000","last_clean":"2023-10-09T06:22:57.278285+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:22:57.278285+0000","last_undegraded":"2023-10-09T06:22:57.278285+0000","last_fullsized":"2023-10-09T06:22:57.278285+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":92,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.306094+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:22:57.306094+0000","last_peered":"2023-10-09T06:22:57.306094+0000","last_clean":"2023-10-09T06:22:57.306094+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:22:57.306094+0000","last_undegraded":"2023-10-09T06:22:57.306094+0000","last_fullsized":"2023-10-09T06:22:57.306094+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.277713+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:22:57.277713+0000","last_peered":"2023-10-09T06:22:57.277713+0000","last_clean":"2023-10-09T06:22:57.277713+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:22:57.277713+0000","last_undegraded":"2023-10-09T06:22:57.277713+0000","last_fullsized":"2023-10-09T06:22:57.277713+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280519+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:22:57.280519+0000","last_peered":"2023-10-09T06:22:57.280519+0000","last_clean":"2023-10-09T06:22:57.280519+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:22:57.280519+0000","last_undegraded":"2023-10-09T06:22:57.280519+0000","last_fullsized":"2023-10-09T06:22:57.280519+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"249'4311","reported_seq":4330,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.770243+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:23:12.770243+0000","last_peered":"2023-10-09T06:23:12.770243+0000","last_clean":"2023-10-09T06:23:12.770243+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:23:12.770243+0000","last_undegraded":"2023-10-09T06:23:12.770243+0000","last_fullsized":"2023-10-09T06:23:12.770243+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4311,"log_dups_size":0,"ondisk_log_size":4311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282525696,"num_objects":4311,"num_object_clones":0,"num_object_copies":12933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4311,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":532,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280646+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:22:57.280646+0000","last_peered":"2023-10-09T06:22:57.280646+0000","last_clean":"2023-10-09T06:22:57.280646+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:22:57.280646+0000","last_undegraded":"2023-10-09T06:22:57.280646+0000","last_fullsized":"2023-10-09T06:22:57.280646+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":238,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:08.732004+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:23:08.732004+0000","last_peered":"2023-10-09T06:23:08.732004+0000","last_clean":"2023-10-09T06:23:08.732004+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:23:08.732004+0000","last_undegraded":"2023-10-09T06:23:08.732004+0000","last_fullsized":"2023-10-09T06:23:08.732004+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":206,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278611+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:22:57.278611+0000","last_peered":"2023-10-09T06:22:57.278611+0000","last_clean":"2023-10-09T06:22:57.278611+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:22:57.278611+0000","last_undegraded":"2023-10-09T06:22:57.278611+0000","last_fullsized":"2023-10-09T06:22:57.278611+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":227,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294570+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:22:57.294570+0000","last_peered":"2023-10-09T06:22:57.294570+0000","last_clean":"2023-10-09T06:22:57.294570+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:22:57.294570+0000","last_undegraded":"2023-10-09T06:22:57.294570+0000","last_fullsized":"2023-10-09T06:22:57.294570+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"249'4408","reported_seq":4427,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-09T06:23:12.807734+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:23:12.807734+0000","last_peered":"2023-10-09T06:23:12.807734+0000","last_clean":"2023-10-09T06:23:12.807734+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:23:12.807734+0000","last_undegraded":"2023-10-09T06:23:12.807734+0000","last_fullsized":"2023-10-09T06:23:12.807734+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":4408,"log_dups_size":0,"ondisk_log_size":4408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288882688,"num_objects":4408,"num_object_clones":0,"num_object_copies":13224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4408,"num_write_kb":282112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":551,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.296769+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:22:57.296769+0000","last_peered":"2023-10-09T06:22:57.296769+0000","last_clean":"2023-10-09T06:22:57.296769+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:22:57.296769+0000","last_undegraded":"2023-10-09T06:22:57.296769+0000","last_fullsized":"2023-10-09T06:22:57.296769+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":233,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.278807+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:22:57.278807+0000","last_peered":"2023-10-09T06:22:57.278807+0000","last_clean":"2023-10-09T06:22:57.278807+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:22:57.278807+0000","last_undegraded":"2023-10-09T06:22:57.278807+0000","last_fullsized":"2023-10-09T06:22:57.278807+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":204,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280862+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:22:57.280862+0000","last_peered":"2023-10-09T06:22:57.280862+0000","last_clean":"2023-10-09T06:22:57.280862+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:22:57.280862+0000","last_undegraded":"2023-10-09T06:22:57.280862+0000","last_fullsized":"2023-10-09T06:22:57.280862+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":202,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280353+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:22:57.280353+0000","last_peered":"2023-10-09T06:22:57.280353+0000","last_clean":"2023-10-09T06:22:57.280353+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:22:57.280353+0000","last_undegraded":"2023-10-09T06:22:57.280353+0000","last_fullsized":"2023-10-09T06:22:57.280353+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":206,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280445+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:22:57.280445+0000","last_peered":"2023-10-09T06:22:57.280445+0000","last_clean":"2023-10-09T06:22:57.280445+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:22:57.280445+0000","last_undegraded":"2023-10-09T06:22:57.280445+0000","last_fullsized":"2023-10-09T06:22:57.280445+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":230,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.302691+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:22:57.302691+0000","last_peered":"2023-10-09T06:22:57.302691+0000","last_clean":"2023-10-09T06:22:57.302691+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:22:57.302691+0000","last_undegraded":"2023-10-09T06:22:57.302691+0000","last_fullsized":"2023-10-09T06:22:57.302691+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":54,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280489+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:22:57.280489+0000","last_peered":"2023-10-09T06:22:57.280489+0000","last_clean":"2023-10-09T06:22:57.280489+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:22:57.280489+0000","last_undegraded":"2023-10-09T06:22:57.280489+0000","last_fullsized":"2023-10-09T06:22:57.280489+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":236,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280412+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:22:57.280412+0000","last_peered":"2023-10-09T06:22:57.280412+0000","last_clean":"2023-10-09T06:22:57.280412+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:22:57.280412+0000","last_undegraded":"2023-10-09T06:22:57.280412+0000","last_fullsized":"2023-10-09T06:22:57.280412+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280937+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:22:57.280937+0000","last_peered":"2023-10-09T06:22:57.280937+0000","last_clean":"2023-10-09T06:22:57.280937+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:22:57.280937+0000","last_undegraded":"2023-10-09T06:22:57.280937+0000","last_fullsized":"2023-10-09T06:22:57.280937+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":226,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280252+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:22:57.280252+0000","last_peered":"2023-10-09T06:22:57.280252+0000","last_clean":"2023-10-09T06:22:57.280252+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:22:57.280252+0000","last_undegraded":"2023-10-09T06:22:57.280252+0000","last_fullsized":"2023-10-09T06:22:57.280252+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":206,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.295277+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:22:57.295277+0000","last_peered":"2023-10-09T06:22:57.295277+0000","last_clean":"2023-10-09T06:22:57.295277+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:22:57.295277+0000","last_undegraded":"2023-10-09T06:22:57.295277+0000","last_fullsized":"2023-10-09T06:22:57.295277+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.294886+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:22:57.294886+0000","last_peered":"2023-10-09T06:22:57.294886+0000","last_clean":"2023-10-09T06:22:57.294886+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:22:57.294886+0000","last_undegraded":"2023-10-09T06:22:57.294886+0000","last_fullsized":"2023-10-09T06:22:57.294886+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280549+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:22:57.280549+0000","last_peered":"2023-10-09T06:22:57.280549+0000","last_clean":"2023-10-09T06:22:57.280549+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:22:57.280549+0000","last_undegraded":"2023-10-09T06:22:57.280549+0000","last_fullsized":"2023-10-09T06:22:57.280549+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-09T06:22:57.280036+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:22:57.280036+0000","last_peered":"2023-10-09T06:22:57.280036+0000","last_clean":"2023-10-09T06:22:57.280036+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:22:57.280036+0000","last_undegraded":"2023-10-09T06:22:57.280036+0000","last_fullsized":"2023-10-09T06:22:57.280036+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4499898368,"num_objects":68663,"num_object_clones":0,"num_object_copies":205989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":68663,"num_write_kb":4394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13508608000,"data_stored":13508608000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68663,"ondisk_log_size":68663,"up":48,"acting":48,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":10308,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234311,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2001984,"kb_used_data":1101752,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":91743168,"statfs":{"total":95995035648,"available":93945004032,"internally_reserved":0,"allocated":1128194048,"data_stored":1127485590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:40 2023","interfaces":[{"interface":"back","average":{"1min":16.815000000000001,"5min":4.4530000000000003,"15min":2.4119999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":219.41900000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":16.806000000000001,"5min":4.6040000000000001,"15min":2.4670000000000001},"min":{"1min":0.36399999999999999,"5min":0.218,"15min":0.218},"max":{"1min":219.399,"5min":219.399,"15min":219.399},"last":3.6880000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:14 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":2.8599999999999999,"15min":2.1920000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.5390000000000001,"5min":31.869,"15min":31.869},"last":3.5600000000000001},{"interface":"front","average":{"1min":1.8979999999999999,"5min":2.3690000000000002,"15min":1.859},"min":{"1min":0.42599999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1059999999999999,"5min":29.486999999999998,"15min":29.486999999999998},"last":4.6980000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:22:26 2023","interfaces":[{"interface":"back","average":{"1min":11.65,"5min":3.5049999999999999,"15min":2.5230000000000001},"min":{"1min":0.33200000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":215.34800000000001,"5min":215.34800000000001,"15min":215.34800000000001},"last":2.5409999999999999},{"interface":"front","average":{"1min":11.436,"5min":3.5659999999999998,"15min":2.6160000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":215.869,"5min":215.869,"15min":215.869},"last":4.6520000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":2.956,"15min":1.3759999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0899999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":1.077},{"interface":"front","average":{"1min":1.2370000000000001,"5min":3.1110000000000002,"15min":1.4530000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.7619999999999996,"5min":51.133000000000003,"15min":51.133000000000003},"last":3.0449999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.49,"15min":1.276},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.827,"5min":25.678999999999998,"15min":25.678999999999998},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.337,"15min":1.115},"min":{"1min":0.32200000000000001,"5min":0.28899999999999998,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":13.305,"15min":13.305},"last":1.008}]},{"osd":5,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":2.379,"15min":1.6830000000000001},"min":{"1min":0.33300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2690000000000001,"5min":50.304000000000002,"15min":50.304000000000002},"last":1.026},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.9259999999999999,"15min":1.3260000000000001},"min":{"1min":0.34399999999999997,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":1.5,"5min":48.774999999999999,"15min":48.774999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:22:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.357,"15min":1.254},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":1.5780000000000001,"5min":8.4890000000000008,"15min":11.878},"last":0.433},{"interface":"front","average":{"1min":0.63,"5min":1.4039999999999999,"15min":1.2130000000000001},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":1.601,"5min":12.141999999999999,"15min":12.141999999999999},"last":1.048}]}]},{"osd":6,"up_from":46,"seq":197568495948,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2532084,"kb_used_data":1934124,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":91213068,"statfs":{"total":95995035648,"available":93402181632,"internally_reserved":0,"allocated":1980542976,"data_stored":1979847170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":5.2110000000000003,"15min":2.8119999999999998},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.214},"max":{"1min":21.555,"5min":110.926,"15min":110.926},"last":4.6479999999999997},{"interface":"front","average":{"1min":2.3100000000000001,"5min":5.7409999999999997,"15min":3.0920000000000001},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.24399999999999999},"max":{"1min":21.535,"5min":110.911,"15min":110.911},"last":1.579}]},{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":5.04,"15min":3.25},"min":{"1min":0.377,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6639999999999997,"5min":114.202,"15min":114.202},"last":0.311},{"interface":"front","average":{"1min":1.0669999999999999,"5min":4.6289999999999996,"15min":3.141},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.246},"max":{"1min":4.2679999999999998,"5min":114.179,"15min":114.179},"last":2.7029999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.008,"15min":2.2770000000000001},"min":{"1min":0.42799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.6120000000000001,"5min":26.306999999999999,"15min":26.306999999999999},"last":2.2370000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.4139999999999999,"15min":2.1640000000000001},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.1050000000000004,"5min":26.292999999999999,"15min":26.292999999999999},"last":2.581}]},{"osd":3,"last update":"Mon Oct 9 06:22:48 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":3.8580000000000001,"15min":1.6459999999999999},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.387,"5min":110.358,"15min":110.358},"last":2.6269999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":4.0789999999999997,"15min":1.734},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.278,"5min":110.43899999999999,"15min":110.43899999999999},"last":4.3780000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.415,"15min":1.1870000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.245},"max":{"1min":7.2060000000000004,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.111,"15min":1.1279999999999999},"min":{"1min":0.318,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":2.6480000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:32 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.73,"15min":1.306},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":22.379999999999999,"5min":22.379999999999999,"15min":22.379999999999999},"last":1.0640000000000001},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.4590000000000001,"15min":1.3049999999999999},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":24.401,"5min":24.401,"15min":24.401},"last":2.609}]},{"osd":7,"last update":"Mon Oct 9 06:23:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":2.2029999999999998,"15min":1.3979999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.26200000000000001},"max":{"1min":1.3180000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":3.0880000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":2.343,"15min":1.647},"min":{"1min":0.375,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":3.21,"5min":100.995,"15min":100.995},"last":3.1000000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402684,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493788,"kb_used_data":1092564,"kb_used_omap":4,"kb_used_meta":401211,"kb_avail":92251364,"statfs":{"total":95995035648,"available":94465396736,"internally_reserved":0,"allocated":1118785536,"data_stored":1117698953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":410840974},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":2.488,"15min":2.4950000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.242,"5min":148.98400000000001,"15min":148.98400000000001},"last":4.5359999999999996},{"interface":"front","average":{"1min":2.516,"5min":2.7829999999999999,"15min":2.871},"min":{"1min":0.26700000000000002,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":10.516,"5min":149.02699999999999,"15min":149.02699999999999},"last":2.5129999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":2.0009999999999999,"15min":1.7529999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":4.2199999999999998,"5min":50.411000000000001,"15min":50.411000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.2490000000000001,"15min":2.524},"min":{"1min":0.253,"5min":0.219,"15min":0.219},"max":{"1min":10.494,"5min":49.783000000000001,"15min":49.783000000000001},"last":3.02}]},{"osd":3,"last update":"Mon Oct 9 06:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":4.2839999999999998,"15min":1.6819999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2800000000000002,"5min":153.18199999999999,"15min":153.18199999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.2170000000000001,"15min":1.3440000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":3.5259999999999998,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":3.7120000000000002,"15min":3.032},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.192,"5min":199.43799999999999,"15min":199.43799999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":2.339,"5min":3.258,"15min":2.9590000000000001},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.413,"5min":199.32900000000001,"15min":199.32900000000001},"last":2.5499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.7789999999999999,"15min":2.6389999999999998},"min":{"1min":0.35599999999999998,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":4.1980000000000004,"5min":61.930999999999997,"15min":61.930999999999997},"last":0.621},{"interface":"front","average":{"1min":2.415,"5min":2.1440000000000001,"15min":2.786},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.462,"5min":61.947000000000003,"15min":61.947000000000003},"last":2.6139999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3319999999999999,"5min":2.1459999999999999,"15min":2.6600000000000001},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.28399999999999997},"max":{"1min":11.516999999999999,"5min":14.382,"15min":14.382},"last":2.589},{"interface":"front","average":{"1min":2.77,"5min":2.363,"15min":2.4529999999999998},"min":{"1min":0.41999999999999998,"5min":0.39200000000000002,"15min":0.374},"max":{"1min":21.036999999999999,"5min":21.036999999999999,"15min":21.036999999999999},"last":0.71399999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:22:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":2.2330000000000001,"15min":2.5099999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.141999999999999,"5min":17.233000000000001,"15min":17.233000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":2.468,"5min":1.7110000000000001,"15min":1.9690000000000001},"min":{"1min":0.46300000000000002,"5min":0.40200000000000002,"15min":0.378},"max":{"1min":16.161000000000001,"5min":16.161000000000001,"15min":16.161000000000001},"last":0.54500000000000004}]}]},{"osd":0,"up_from":91,"seq":390842024117,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3191396,"kb_used_data":2651804,"kb_used_omap":5,"kb_used_meta":539578,"kb_avail":90553756,"statfs":{"total":95995035648,"available":92727046144,"internally_reserved":0,"allocated":2715447296,"data_stored":2714180366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":552528506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,7,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:22:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":4.7380000000000004,"15min":2.7999999999999998},"min":{"1min":0.21099999999999999,"5min":0.193,"15min":0.19},"max":{"1min":8.8889999999999993,"5min":185.17400000000001,"15min":185.17400000000001},"last":5.109},{"interface":"front","average":{"1min":2.2410000000000001,"5min":4.867,"15min":3.4119999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":10.73,"5min":186.691,"15min":186.691},"last":2.7440000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:22:24 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.6539999999999999,"15min":2.274},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.17799999999999999},"max":{"1min":7.7709999999999999,"5min":17.588999999999999,"15min":17.588999999999999},"last":2.6539999999999999},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.77,"15min":2.3140000000000001},"min":{"1min":0.23300000000000001,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":5.4210000000000003,"5min":11.162000000000001,"15min":21.198},"last":13.081}]},{"osd":3,"last update":"Mon Oct 9 06:22:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":6.4169999999999998,"15min":2.395},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.1070000000000002,"5min":185.89400000000001,"15min":185.89400000000001},"last":12.269},{"interface":"front","average":{"1min":1.234,"5min":6.8620000000000001,"15min":2.5539999999999998},"min":{"1min":0.32600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":6.298,"5min":186.72900000000001,"15min":186.72900000000001},"last":2.677}]},{"osd":4,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.3560000000000001,"15min":1.5940000000000001},"min":{"1min":0.41699999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9950000000000001,"5min":11.162000000000001,"15min":15.835000000000001},"last":1.905},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.6910000000000001,"15min":2.0779999999999998},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.25700000000000001},"max":{"1min":2.456,"5min":12.752000000000001,"15min":15.023},"last":2.4860000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":2.1059999999999999,"15min":2.032},"min":{"1min":0.41599999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.2450000000000001,"5min":62.984999999999999,"15min":62.984999999999999},"last":5.0499999999999998},{"interface":"front","average":{"1min":1.012,"5min":2.4670000000000001,"15min":2.4550000000000001},"min":{"1min":0.39900000000000002,"5min":0.30299999999999999,"15min":0.26700000000000002},"max":{"1min":4.367,"5min":64.983999999999995,"15min":64.983999999999995},"last":8.7050000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.899,"15min":2.234},"min":{"1min":0.36199999999999999,"5min":0.32000000000000001,"15min":0.29299999999999998},"max":{"1min":3.1749999999999998,"5min":9.6880000000000006,"15min":32.079000000000001},"last":8.6389999999999993},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.546,"15min":1.9219999999999999},"min":{"1min":0.44500000000000001,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":4.9690000000000003,"5min":11.236000000000001,"15min":15.893000000000001},"last":5.085}]},{"osd":7,"last update":"Mon Oct 9 06:22:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.282,"15min":2.3260000000000001},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":2.8100000000000001,"5min":62.262,"15min":62.262},"last":2.6299999999999999},{"interface":"front","average":{"1min":1.115,"5min":2.3839999999999999,"15min":2.4860000000000002},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":5.875,"5min":62.229999999999997,"15min":62.229999999999997},"last":8.6809999999999992}]}]},{"osd":2,"up_from":128,"seq":549755813986,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2454984,"kb_used_data":2165504,"kb_used_omap":7,"kb_used_meta":289464,"kb_avail":91290168,"statfs":{"total":95995035648,"available":93481132032,"internally_reserved":0,"allocated":2217476096,"data_stored":2216406137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":296411217},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":5.0780000000000003,"15min":2.9580000000000002},"min":{"1min":0.183,"5min":0.157,"15min":0.157},"max":{"1min":5.2549999999999999,"5min":163.40100000000001,"15min":163.40100000000001},"last":2.77},{"interface":"front","average":{"1min":1.8600000000000001,"5min":5.3010000000000002,"15min":3.4830000000000001},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.448,"5min":163.363,"15min":163.363},"last":2.7200000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":2.8420000000000001,"15min":2.835},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.188},"max":{"1min":5.5030000000000001,"5min":73.483999999999995,"15min":222.50299999999999},"last":2.6379999999999999},{"interface":"front","average":{"1min":2.331,"5min":3.222,"15min":3.4710000000000001},"min":{"1min":0.26900000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":10.420999999999999,"5min":75.641000000000005,"15min":222.19399999999999},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.4209999999999998,"15min":1.375},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.5379999999999998,"5min":81.183999999999997,"15min":81.183999999999997},"last":2.532},{"interface":"front","average":{"1min":0.748,"5min":3.4510000000000001,"15min":1.431},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.7890000000000001,"5min":81.061999999999998,"15min":81.061999999999998},"last":2.1709999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4870000000000001,"15min":1.8520000000000001},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":2.9710000000000001,"5min":12.106999999999999,"15min":12.141999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.962,"15min":2.1989999999999998},"min":{"1min":0.374,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.7089999999999996,"5min":11.994999999999999,"15min":15.183},"last":1.0920000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.304,"15min":1.5720000000000001},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":2.6259999999999999,"5min":5.7199999999999998,"15min":5.7460000000000004},"last":5.2640000000000002},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.0760000000000001,"15min":2.3380000000000001},"min":{"1min":0.41099999999999998,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":5.6200000000000001,"5min":10.548,"15min":10.548},"last":4.6360000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":1.7589999999999999,"15min":2.0899999999999999},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.1319999999999997,"5min":10.522,"15min":10.522},"last":1.218},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.2969999999999999,"15min":1.5209999999999999},"min":{"1min":0.41999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.371,"5min":6.883,"15min":7.327},"last":4.7359999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.4810000000000001,"15min":1.605},"min":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.9449999999999998,"5min":7.6660000000000004,"15min":7.6660000000000004},"last":5.5869999999999997},{"interface":"front","average":{"1min":1.569,"5min":1.5629999999999999,"15min":2.5209999999999999},"min":{"1min":0.40100000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":4.4160000000000004,"5min":6.7779999999999996,"15min":9.3249999999999993},"last":2.6190000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127184,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1704620,"kb_used_data":1404324,"kb_used_omap":2,"kb_used_meta":300285,"kb_avail":92040532,"statfs":{"total":95995035648,"available":94249504768,"internally_reserved":0,"allocated":1438027776,"data_stored":1437041932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":307491893},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":8.2959999999999994,"5min":5.3940000000000001,"15min":2.2109999999999999},"min":{"1min":0.23699999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":70.349999999999994,"5min":187.852,"15min":187.852},"last":63.384999999999998},{"interface":"front","average":{"1min":8.0169999999999995,"5min":6.6180000000000003,"15min":2.8380000000000001},"min":{"1min":0.214,"5min":0.191,"15min":0.191},"max":{"1min":69.978999999999999,"5min":187.923,"15min":187.923},"last":63.319000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.0229999999999997,"5min":5.9710000000000001,"15min":2.6219999999999999},"min":{"1min":0.25600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":59.646999999999998,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":4.3120000000000003,"5min":3.1829999999999998,"15min":1.4670000000000001},"min":{"1min":0.245,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":60.085999999999999,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":4.4729999999999999,"5min":5.2519999999999998,"15min":2.3799999999999999},"min":{"1min":0.28199999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":62.904000000000003,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":4.4989999999999997,"5min":3.9319999999999999,"15min":1.591},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":62.719000000000001,"5min":187.79499999999999,"15min":187.79499999999999},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.4099999999999999,"15min":0.88100000000000001},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.2509999999999999,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.69599999999999995,"5min":3.278,"15min":1.887},"min":{"1min":0.436,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.722,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":3.7080000000000002,"15min":2.0510000000000002},"min":{"1min":0.46500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.6749999999999998,"5min":187.833,"15min":187.833},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":3.7490000000000001,"15min":2.0680000000000001},"min":{"1min":0.44800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.3820000000000001,"5min":187.952,"15min":187.952},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":3.512,"15min":1.9670000000000001},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.7490000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":3.4849999999999999,"15min":1.9590000000000001},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.4279999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.41599999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":3.4660000000000002,"15min":1.5820000000000001},"min":{"1min":0.39200000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":3.431,"15min":1.954},"min":{"1min":0.41799999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":2.774,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.52200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019220,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3650936,"kb_used_data":2764144,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":90094216,"statfs":{"total":95995035648,"available":92256477184,"internally_reserved":0,"allocated":2830483456,"data_stored":2829782855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.3069999999999999,"15min":1.476},"min":{"1min":0.315,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":1.798,"5min":19.577000000000002,"15min":19.577000000000002},"last":3.4820000000000002},{"interface":"front","average":{"1min":0.82299999999999995,"5min":1.7909999999999999,"15min":1.869},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":3.1560000000000001,"5min":19.594999999999999,"15min":19.594999999999999},"last":3.5600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":3.2909999999999999,"15min":2.1629999999999998},"min":{"1min":0.23699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.0190000000000001,"5min":91.849999999999994,"15min":91.849999999999994},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.88600000000000001,"5min":3.4199999999999999,"15min":2.206},"min":{"1min":0.34799999999999998,"5min":0.245,"15min":0.245},"max":{"1min":5.117,"5min":91.885000000000005,"15min":91.885000000000005},"last":4.5049999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.5589999999999999,"15min":1.75},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":10.884,"5min":10.884,"15min":10.884},"last":2.637},{"interface":"front","average":{"1min":1.026,"5min":1.821,"15min":1.8799999999999999},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.5850000000000009,"5min":15.894,"15min":15.894},"last":2.4750000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":4.5049999999999999,"15min":1.877},"min":{"1min":0.38100000000000001,"5min":0.309,"15min":0.309},"max":{"1min":6.4749999999999996,"5min":91.697000000000003,"15min":91.697000000000003},"last":2.431},{"interface":"front","average":{"1min":0.69999999999999996,"5min":4.2279999999999998,"15min":1.8240000000000001},"min":{"1min":0.38900000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1850000000000001,"5min":94.356999999999999,"15min":94.356999999999999},"last":0.88700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.1950000000000001,"15min":1.0669999999999999},"min":{"1min":0.308,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":9.1720000000000006,"5min":10.673,"15min":10.673},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.147,"15min":1.246},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.24399999999999999},"max":{"1min":9.6379999999999999,"5min":12.552,"15min":12.552},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:22:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.78800000000000003,"15min":0.84399999999999997},"min":{"1min":0.33700000000000002,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.881,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.242,"15min":1.171},"min":{"1min":0.32700000000000001,"5min":0.32400000000000001,"15min":0.29499999999999998},"max":{"1min":1.0029999999999999,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:22:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":2.3900000000000001,"15min":1.427},"min":{"1min":0.36199999999999999,"5min":0.308,"15min":0.308},"max":{"1min":1.7370000000000001,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":2.3210000000000002,"15min":1.4650000000000001},"min":{"1min":0.42699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.704,"5min":62.573,"15min":62.573},"last":0.504}]}]},{"osd":5,"up_from":0,"seq":163208757585,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2739994624,"data_stored":2739994624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1118699520,"data_stored":1118699520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2221539328,"data_stored":2221539328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1442775040,"data_stored":1442775040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2846425088,"data_stored":2846425088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2000551936,"data_stored":2000551936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1138622464,"data_stored":1138622464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:23:17.624 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:23:17.624 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-10-09T06:23:17.695 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:17.732 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:17.738 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 74157 74141 89.0523 69.375 0.0101952 0.0112242 2023-10-09T06:23:17.788 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:17.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:17.855 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.855 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[220921]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:23:17.856 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:23:17.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:17 smithi107 ceph-mon[212750]: failed to encode map e250 with expected crc 2023-10-09T06:23:17.896 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:17.998 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:18.103 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:18.108 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:18.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:18.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: failed to encode map e250 with expected crc 2023-10-09T06:23:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-10-09T06:23:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-09T06:23:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-09T06:23:18.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: osdmap e250: 8 total, 8 up, 7 in 2023-10-09T06:23:18.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: failed to encode map e250 with expected crc 2023-10-09T06:23:18.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:17 smithi150 ceph-mon[194178]: failed to encode map e250 with expected crc 2023-10-09T06:23:18.202 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:23:18.203 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:23:18.218 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:18.246 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:18.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:18.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:18.418 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:18.493 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:18.737 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 75001 74985 88.3676 52.75 0.0163302 0.0113115 2023-10-09T06:23:18.751 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:23:18.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:23:18.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:23:18.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: failed to encode map e251 with expected crc 2023-10-09T06:23:18.790 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: pgmap v776: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: failed to encode map e251 with expected crc 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: osdmap e251: 8 total, 8 up, 7 in 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: failed to encode map e251 with expected crc 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: failed to encode map e251 with expected crc 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: failed to encode map e251 with expected crc 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: failed to encode map e251 with expected crc 2023-10-09T06:23:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:19.738 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 76131 76115 88.0391 70.625 0.0218826 0.0113534 2023-10-09T06:23:19.878 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:19.878 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[212750]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:19.879 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T06:23:19.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.880 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:19 smithi107 ceph-mon[220921]: failed to encode map e252 with expected crc 2023-10-09T06:23:19.906 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:23:20.062 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:20.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: failed to encode map e252 with expected crc 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: failed to encode map e252 with expected crc 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2008861628' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: osdmap e252: 8 total, 8 up, 7 in 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: failed to encode map e252 with expected crc 2023-10-09T06:23:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:19 smithi150 ceph-mon[194178]: failed to encode map e252 with expected crc 2023-10-09T06:23:20.167 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:20.194 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:20.622 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:20.627 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:20.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:20.735 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:20.748 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 77144 77128 87.5879 63.3125 0.0133298 0.0114125 2023-10-09T06:23:20.818 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:20.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:20 smithi107 ceph-mon[212750]: pgmap v779: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 828 op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:20.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:20 smithi107 ceph-mon[220921]: pgmap v779: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 828 op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:21.006 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:21.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:21.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:20 smithi150 ceph-mon[194178]: pgmap v779: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 828 op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:21.173 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:21.212 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:21.449 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:21.461 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:21.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:21.483 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:21.528 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:21.566 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:21.577 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:21.615 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:21.649 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:21.658 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:21.692 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:21.740 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:21.740 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 78124 78108 87.1149 61.25 0.0171012 0.0114742 2023-10-09T06:23:21.748 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.748 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:23:21.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:23:22.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/752633664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:22.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/752633664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:22.016 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:22] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-10-09T06:23:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/752633664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:22.252 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:22.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:22.361 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:22.387 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:22.427 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:22.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:22.535 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:22.569 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:22.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:22.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:22.670 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:23:22.741 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 79029 79013 86.5792 56.5625 0.0165478 0.0115455 2023-10-09T06:23:22.746 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:22.778 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:22.840 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:22.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:22.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:22 smithi107 ceph-mon[212750]: pgmap v780: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:22.891 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:22 smithi107 ceph-mon[220921]: pgmap v780: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:22 smithi150 ceph-mon[194178]: pgmap v780: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:22.948 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:23.060 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:23.205 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:23.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:23.313 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:23.393 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:23.742 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 80017 80001 86.1491 61.75 0.0196964 0.0116029 2023-10-09T06:23:24.056 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:24.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:24.163 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:24.240 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:24.384 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:24.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:24.441 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:24.496 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:24.530 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:24.601 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:24.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:24.710 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:24.742 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 80996 80980 85.7261 61.1875 0.0120572 0.0116593 2023-10-09T06:23:24.795 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 10 pg_num to 26 2023-10-09T06:23:24.809 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:24 smithi150 ceph-mon[194178]: pgmap v781: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3471804546' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:24.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[212750]: pgmap v781: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:24.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3471804546' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:24.964 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:24.964 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[220921]: pgmap v781: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 3 B/s, 0 objects/s recovering 2023-10-09T06:23:24.964 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3471804546' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:24.964 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:23:25.016 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:23:25.609 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:25.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:25.721 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:25.745 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:23:25.746263+0000 min lat: 0.00112444 max lat: 0.323912 avg lat: 0.0117111 2023-10-09T06:23:25.745 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:23:25.746 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 82014 81998 85.3537 63.625 0.0162483 0.0117111 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: failed to encode map e253 with expected crc 2023-10-09T06:23:25.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:23:25.795 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:23:26.126 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:23:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:23:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:23:26.126 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:23:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:23:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: osdmap e253: 8 total, 8 up, 7 in 2023-10-09T06:23:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: failed to encode map e253 with expected crc 2023-10-09T06:23:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:23:26.262 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:26.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:26.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:26.451 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:26.746 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 82076 82060 84.0187 3.875 0.00698399 0.0117358 2023-10-09T06:23:27.134 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:27.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: pgmap v783: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 2 B/s, 0 objects/s recovering 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[212750]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: pgmap v783: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 2 B/s, 0 objects/s recovering 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:26 smithi150 ceph-mon[194178]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: pgmap v783: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 2 B/s, 0 objects/s recovering 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: osdmap e254: 8 total, 8 up, 7 in 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:26 smithi107 ceph-mon[220921]: failed to encode map e254 with expected crc 2023-10-09T06:23:27.169 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:27.176 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:23:27.247 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:27.343 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:27.525 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:27.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:27.634 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:27.697 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:27.746 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 83178 83162 83.7735 68.875 0.0114225 0.0119308 2023-10-09T06:23:27.759 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:27.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:27.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:27.947 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:28.076 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:28.086 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.086 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[212750]: 10.0 scrub starts 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:27 smithi107 ceph-mon[220921]: 10.0 scrub starts 2023-10-09T06:23:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:28.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: osdmap e255: 8 total, 8 up, 7 in 2023-10-09T06:23:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: failed to encode map e255 with expected crc 2023-10-09T06:23:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:27 smithi150 ceph-mon[194178]: 10.0 scrub starts 2023-10-09T06:23:28.190 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:28.228 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:28.302 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:28.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:28.408 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:28.434 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:28.476 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:28.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:28.579 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:28.611 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:28.698 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:28.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:28.746 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 84088 84072 83.3467 56.875 0.012217 0.0119934 2023-10-09T06:23:28.805 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:28.869 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:23:28.891 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: pgmap v786: 109 pgs: 4 peering, 10 unknown, 95 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:23:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: 10.0 scrub ok 2023-10-09T06:23:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: 10.9 scrub starts 2023-10-09T06:23:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: 10.2 scrub starts 2023-10-09T06:23:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3970711290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:29.120 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: pgmap v786: 109 pgs: 4 peering, 10 unknown, 95 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: 10.0 scrub ok 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: 10.9 scrub starts 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: 10.2 scrub starts 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3970711290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: pgmap v786: 109 pgs: 4 peering, 10 unknown, 95 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 851 op/s 2023-10-09T06:23:29.121 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: 10.0 scrub ok 2023-10-09T06:23:29.122 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: 10.9 scrub starts 2023-10-09T06:23:29.122 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: 10.2 scrub starts 2023-10-09T06:23:29.122 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3970711290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:29.122 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:29.130 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:29.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:29.237 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:29.277 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:29.324 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:29.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:29.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:29.433 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:29.473 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:29.549 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:29.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:29.658 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:29.735 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:29.748 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 85287 85271 83.2137 74.9375 0.0231699 0.0120118 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: 10.9 scrub ok 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: 10.8 scrub starts 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:23:29.968 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[212750]: pgmap v788: 109 pgs: 4 peering, 7 unknown, 98 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s 2023-10-09T06:23:29.969 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: 10.9 scrub ok 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: 10.8 scrub starts 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.970 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: failed to encode map e256 with expected crc 2023-10-09T06:23:29.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:29 smithi107 ceph-mon[220921]: pgmap v788: 109 pgs: 4 peering, 7 unknown, 98 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s 2023-10-09T06:23:29.980 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:29.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:30.088 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:30.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: 10.9 scrub ok 2023-10-09T06:23:30.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: 10.8 scrub starts 2023-10-09T06:23:30.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:23:30.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:23:30.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: osdmap e256: 8 total, 8 up, 7 in 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: failed to encode map e256 with expected crc 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: failed to encode map e256 with expected crc 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: failed to encode map e256 with expected crc 2023-10-09T06:23:30.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:29 smithi150 ceph-mon[194178]: pgmap v788: 109 pgs: 4 peering, 7 unknown, 98 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s 2023-10-09T06:23:30.141 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:30.201 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:30.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:30.298 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:30.307 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:30.347 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:30.409 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:30.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:30.475 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:30.510 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:30.517 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:30.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:30.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:30.596 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:30.634 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:30.677 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:30.720 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:30.732 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:30.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:30.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 86389 86373 82.9917 68.875 0.0140785 0.012043 2023-10-09T06:23:30.757 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:30.758 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.758 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.758 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.758 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.759 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.759 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.759 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.759 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:23:30.841 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:30.851 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[212750]: 10.8 scrub ok 2023-10-09T06:23:30.881 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:30.951 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:30.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:31.061 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:31.100 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[212750]: 10.7 deep-scrub starts 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[212750]: failed to encode map e256 with expected crc 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1586134792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[220921]: 10.8 scrub ok 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[220921]: 10.7 deep-scrub starts 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[220921]: failed to encode map e256 with expected crc 2023-10-09T06:23:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1586134792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:30 smithi150 ceph-mon[194178]: 10.8 scrub ok 2023-10-09T06:23:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:30 smithi150 ceph-mon[194178]: 10.7 deep-scrub starts 2023-10-09T06:23:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:30 smithi150 ceph-mon[194178]: failed to encode map e256 with expected crc 2023-10-09T06:23:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1586134792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:31.160 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:31.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:31.268 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:31.310 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:31.379 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:31.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:31.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:31.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:31.497 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:23:31.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:23:31.537 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:31.579 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:31.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:31.686 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:31.716 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:31.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 87239 87223 82.5394 53.125 0.00383774 0.0121083 2023-10-09T06:23:31.774 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:31.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:31.879 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:31.915 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:32.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:23:32.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:32] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[212750]: 10.6 scrub starts 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[212750]: pgmap v789: 109 pgs: 4 peering, 4 unknown, 101 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[220921]: 10.6 scrub starts 2023-10-09T06:23:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[220921]: pgmap v789: 109 pgs: 4 peering, 4 unknown, 101 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail 2023-10-09T06:23:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:32.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:31 smithi150 ceph-mon[194178]: 10.6 scrub starts 2023-10-09T06:23:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:31 smithi150 ceph-mon[194178]: pgmap v789: 109 pgs: 4 peering, 4 unknown, 101 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail 2023-10-09T06:23:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:32.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 87785 87769 81.8171 34.125 0.0984062 0.0122124 2023-10-09T06:23:32.888 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:32.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:33.006 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:33.038 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:33.100 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:33.103 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:33.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[220921]: 10.6 scrub ok 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[220921]: 10.2 scrub starts 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[220921]: 10.2 scrub ok 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[212750]: 10.6 scrub ok 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[212750]: 10.2 scrub starts 2023-10-09T06:23:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:32 smithi107 ceph-mon[212750]: 10.2 scrub ok 2023-10-09T06:23:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:32 smithi150 ceph-mon[194178]: 10.6 scrub ok 2023-10-09T06:23:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:32 smithi150 ceph-mon[194178]: 10.2 scrub starts 2023-10-09T06:23:33.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:32 smithi150 ceph-mon[194178]: 10.2 scrub ok 2023-10-09T06:23:33.203 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:33.206 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:33.245 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:33.322 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:33.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:33.438 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:33.491 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:33.583 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:33.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:33.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:33.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 88766 88750 81.5156 61.3125 0.0118885 0.0122621 2023-10-09T06:23:33.758 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:33.783 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:23:33.783 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:23:33.841 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:33.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:33.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:33.949 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[220921]: pgmap v790: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 8.8 KiB/s, 0 objects/s recovering 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[220921]: 10.1 scrub starts 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[212750]: pgmap v790: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 8.8 KiB/s, 0 objects/s recovering 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:23:34.004 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:33 smithi107 ceph-mon[212750]: 10.1 scrub starts 2023-10-09T06:23:34.008 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:34.141 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:34.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:34.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:33 smithi150 ceph-mon[194178]: pgmap v790: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 8.8 KiB/s, 0 objects/s recovering 2023-10-09T06:23:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:33 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:23:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:33 smithi150 ceph-mon[194178]: 10.1 scrub starts 2023-10-09T06:23:34.248 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:34.284 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:34.339 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:34.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:34.444 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:34.477 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:34.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 89357 89341 80.8698 36.9375 0.00315134 0.0122541 2023-10-09T06:23:34.929 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: 10.1 scrub ok 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[212750]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: 10.1 scrub ok 2023-10-09T06:23:35.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:23:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T06:23:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:34 smithi107 ceph-mon[220921]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: 10.1 scrub ok 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: osdmap e257: 8 total, 8 up, 7 in 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:34 smithi150 ceph-mon[194178]: failed to encode map e257 with expected crc 2023-10-09T06:23:35.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 89357 89341 79.7152 0 - 0.0122541 2023-10-09T06:23:36.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: 10.12 deep-scrub starts 2023-10-09T06:23:36.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T06:23:36.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.113 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.113 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: pgmap v793: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:23:36.113 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:23:36.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: 10.12 deep-scrub starts 2023-10-09T06:23:36.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T06:23:36.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: pgmap v793: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:23:36.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:35 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:23:36.148 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: 10.12 deep-scrub starts 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: osdmap e258: 8 total, 8 up, 7 in 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: failed to encode map e258 with expected crc 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: pgmap v793: 109 pgs: 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:23:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:35 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:23:36.307 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:36.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:36.419 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:36.455 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:36.519 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:36.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:36.630 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:36.664 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:36.726 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:36.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:36.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:36.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 89756 89740 78.9441 12.4688 0.00386595 0.0126208 2023-10-09T06:23:36.781 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:36.832 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:36.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:23:36.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[212750]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T06:23:36.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[212750]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[220921]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:23:36.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[220921]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T06:23:36.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[220921]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:36 smithi107 ceph-mon[220921]: failed to encode map e259 with expected crc 2023-10-09T06:23:36.908 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:37.008 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:37.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:37.122 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:37.174 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:36 smithi150 ceph-mon[194178]: failed to encode map e259 with expected crc 2023-10-09T06:23:37.174 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:23:37.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:36 smithi150 ceph-mon[194178]: osdmap e259: 8 total, 8 up, 7 in 2023-10-09T06:23:37.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:36 smithi150 ceph-mon[194178]: failed to encode map e259 with expected crc 2023-10-09T06:23:37.175 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:36 smithi150 ceph-mon[194178]: failed to encode map e259 with expected crc 2023-10-09T06:23:37.207 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:37.225 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:37.262 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:37.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:37.368 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:37.402 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:37.562 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:37.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:37.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:37.695 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:37.739 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:37.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:37.749 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 89886 89870 77.9611 8.125 0.00215449 0.0126224 2023-10-09T06:23:37.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:37.868 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:37.907 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:23:37.909 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:23:37.913 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:37.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:38.018 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:38.044 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:38.086 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:38.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T06:23:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: pgmap v796: 109 pgs: 2 active+clean+scrubbing+deep, 107 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: 10.7 deep-scrub ok 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1730372893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1663371518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[212750]: 10.3 deep-scrub starts 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: pgmap v796: 109 pgs: 2 active+clean+scrubbing+deep, 107 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: 10.7 deep-scrub ok 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1730372893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1663371518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:37 smithi107 ceph-mon[220921]: 10.3 deep-scrub starts 2023-10-09T06:23:38.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: osdmap e260: 8 total, 8 up, 7 in 2023-10-09T06:23:38.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: failed to encode map e260 with expected crc 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: pgmap v796: 109 pgs: 2 active+clean+scrubbing+deep, 107 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: 10.7 deep-scrub ok 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1730372893' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:23:38.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1663371518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:23:38.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:37 smithi150 ceph-mon[194178]: 10.3 deep-scrub starts 2023-10-09T06:23:38.189 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:38.237 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:38.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 90342 90326 77.2839 28.5 0.00314499 0.0128563 2023-10-09T06:23:39.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:39.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: failed to encode map e261 with expected crc 2023-10-09T06:23:39.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[212750]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: failed to encode map e261 with expected crc 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:23:39.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:23:39.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:39.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:38 smithi107 ceph-mon[220921]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: failed to encode map e261 with expected crc 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:23:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:38 smithi150 ceph-mon[194178]: osdmap e261: 8 total, 8 up, 7 in 2023-10-09T06:23:39.374 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:39.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:39.477 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:39.596 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:39.609 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:23:39.678 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:39.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:39.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 90343 90327 76.2409 0.0625 0.135159 0.0128576 2023-10-09T06:23:39.783 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:39.810 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:39.855 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:39.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:39.964 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:39.994 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:40.042 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:40.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:40.151 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T06:23:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: pgmap v799: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2496/269307 objects misplaced (0.927%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:23:40.194 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:40.240 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:40.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:40.346 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: pgmap v799: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2496/269307 objects misplaced (0.927%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:23:40.405 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:40.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: osdmap e262: 8 total, 8 up, 7 in 2023-10-09T06:23:40.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.422 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: pgmap v799: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 107 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2496/269307 objects misplaced (0.927%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:40.423 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:23:40.516 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:40.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:40.565 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:40.592 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:40.619 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:40.629 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:40.643 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:40.665 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:40.680 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:40.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:40.717 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:40.721 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:40.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:40.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:40.742 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.744 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:23:40.751 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 90817 90801 75.6196 29.625 0.00264814 0.01317 2023-10-09T06:23:40.776 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:40.824 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:40.857 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:40.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: failed to encode map e262 with expected crc 2023-10-09T06:23:40.916 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: failed to encode map e263 with expected crc 2023-10-09T06:23:40.917 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3539690358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:41.030 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:41.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:41.137 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: failed to encode map e262 with expected crc 2023-10-09T06:23:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:23:41.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:23:41.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3539690358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:41.171 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:41.219 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:41.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:41.326 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:41.360 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: failed to encode map e262 with expected crc 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: osdmap e263: 8 total, 8 up, 7 in 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: failed to encode map e263 with expected crc 2023-10-09T06:23:41.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3539690358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:41.403 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:41.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:41.508 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:41.540 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:41.588 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:41.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:41.647 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:23:41.694 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:41.726 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:41.750 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 90817 90801 74.6252 0 - 0.01317 2023-10-09T06:23:41.927 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:41.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:41.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:42.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:23:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[220921]: pgmap v802: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2805/270075 objects degraded (1.039%); 2496/270075 objects misplaced (0.924%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:23:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:41 smithi107 ceph-mon[212750]: pgmap v802: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2805/270075 objects degraded (1.039%); 2496/270075 objects misplaced (0.924%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:42.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64723 9.2 9:4bf0c3c2:::benchmark_data_smithi107_300016_object64722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:23:42.036 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.95476567100002, 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.362261308, 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64723 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.037 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078541+0000", 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078567+0000", 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6197e-05 2023-10-09T06:23:42.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078577+0000", 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078583+0000", 2023-10-09T06:23:42.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4130000000000003e-06 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079048+0000", 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046495299999999999 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079196+0000", 2023-10-09T06:23:42.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014766599999999999 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079363+0000", 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016698200000000001 2023-10-09T06:23:42.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.084453+0000", 2023-10-09T06:23:42.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050901360000000003 2023-10-09T06:23:42.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440670+0000", 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.356217269 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440709+0000", 2023-10-09T06:23:42.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9119999999999998e-05 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440762+0000", 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2203000000000001e-05 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440804+0000", 2023-10-09T06:23:42.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1916999999999999e-05 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64724 9.7 e163/160 9:e68efaaf:::benchmark_data_smithi107_300016_object64723:head v 163'4118, mlcod=0'0)", 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.95466688699997, 2023-10-09T06:23:42.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35938271500000002, 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078682+0000", 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0822999999999999e-05 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078689+0000", 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078694+0000", 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0590000000000002e-06 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081003+0000", 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023091069999999999 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081031+0000", 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8022e-05 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437591+0000", 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35656025299999999 2023-10-09T06:23:42.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438024+0000", 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043272699999999999 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.052 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64725 9.f e163/152 9:f1989478:::benchmark_data_smithi107_300016_object64724:head v 163'3999, mlcod=163'3997)", 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.95460121000002, 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35934786400000002, 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967296 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078736+0000", 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9133e-05 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078741+0000", 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5060000000000003e-06 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078745+0000", 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1790000000000001e-06 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083200+0000", 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044543509999999996 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083218+0000", 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8397999999999999e-05 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438028+0000", 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35480992500000003 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438055+0000", 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6554e-05 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64727 9.8 9:1ddec4f7:::benchmark_data_smithi107_300016_object64726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.95171842299999, 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35651537999999999, 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64727 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.059 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081588+0000", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081613+0000", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5406e-05 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081626+0000", 2023-10-09T06:23:42.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2520999999999999e-05 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081634+0000", 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081691+0000", 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6190999999999998e-05 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081807+0000", 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011626299999999999 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081946+0000", 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013932899999999999 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087000+0000", 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050537489999999997 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437968+0000", 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35096775800000002 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438021+0000", 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3038999999999998e-05 2023-10-09T06:23:42.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438065+0000", 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4045999999999998e-05 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438105+0000", 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0014e-05 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64729 9.3 9:cba02ed8:::benchmark_data_smithi107_300016_object64728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.951656232, 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37691757999999997, 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64729 2023-10-09T06:23:42.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081651+0000", 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081673+0000", 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2178999999999999e-05 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081681+0000", 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9219999999999997e-06 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081688+0000", 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2469999999999997e-06 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085025+0000", 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033377429999999998 2023-10-09T06:23:42.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085147+0000", 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000122089 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085280+0000", 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013249099999999999 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444158+0000", 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35887852100000001 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444204+0000", 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5355999999999997e-05 2023-10-09T06:23:42.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456528+0000", 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01232391 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456591+0000", 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.458569+0000", 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019784849999999999 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.076 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64730 9.f e163/152 9:f3222a22:::benchmark_data_smithi107_300016_object64729:head v 163'4000, mlcod=163'3997)", 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.95157986800001, 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35634943299999999, 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:23:42.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081727+0000", 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081887+0000", 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016013400000000001 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081898+0000", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0875e-05 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081916+0000", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7195000000000001e-05 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085279+0000", 2023-10-09T06:23:42.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033639469999999999 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085297+0000", 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7669e-05 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438058+0000", 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35276105099999999 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438077+0000", 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9317000000000002e-05 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64733 9.2 9:4941674d:::benchmark_data_smithi107_300016_object64732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.94657189700001, 2023-10-09T06:23:42.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.360635498, 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64733 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086735+0000", 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086763+0000", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7846000000000001e-05 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086772+0000", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9039999999999998e-06 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086778+0000", 2023-10-09T06:23:42.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2759999999999997e-06 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086950+0000", 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172048 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087082+0000", 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013214800000000001 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087233+0000", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000151213 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437967+0000", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35073413799999997 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440845+0000", 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028772379999999998 2023-10-09T06:23:42.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447284+0000", 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064393209999999996 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447333+0000", 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9299999999999999e-05 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447372+0000", 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8277e-05 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64729 9.3 e163/160 ondisk, result = 0)", 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.94448414599998, 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.355337442, 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.091 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088823+0000", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088829+0000", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5189999999999998e-06 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088840+0000", 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:23:42.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088848+0000", 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.391e-06 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444142+0000", 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35529475900000002 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444151+0000", 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444161+0000", 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0239e-05 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64727 9.8 e163/152 ondisk, result = 0)", 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.94336280300001, 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.348172755, 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089944+0000", 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089951+0000", 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3820000000000001e-06 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437572+0000", 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34762106599999998 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437921+0000", 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034914000000000001 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438007+0000", 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6438000000000001e-05 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438015+0000", 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4359999999999996e-06 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438118+0000", 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103149 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64737 9.2 9:42e1c82a:::benchmark_data_smithi107_300016_object64736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.94295265699998, 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.359336867, 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.100 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64737 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090354+0000", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090375+0000", 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1294e-05 2023-10-09T06:23:42.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437567+0000", 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34719175099999999 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437894+0000", 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032692099999999998 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437999+0000", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010511299999999999 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438153+0000", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015426700000000001 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438341+0000", 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018829099999999999 2023-10-09T06:23:42.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447252+0000", 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0089106849999999998 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447436+0000", 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018348799999999999 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449593+0000", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002157413 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449650+0000", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7450000000000001e-05 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449692+0000", 2023-10-09T06:23:42.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1686e-05 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64736 9.a e163/160 9:553011b2:::benchmark_data_smithi107_300016_object64735:head v 163'4090, mlcod=0'0)", 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 293.94258761100002, 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35900552499999999, 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090719+0000", 2023-10-09T06:23:42.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090765+0000", 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5793e-05 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437575+0000", 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34680993900000001 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437924+0000", 2023-10-09T06:23:42.111 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:42.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034926999999999999 2023-10-09T06:23:42.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437970+0000", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5179999999999998e-05 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438005+0000", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5361000000000002e-05 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449697+0000", 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011691906 2023-10-09T06:23:42.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449726+0000", 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8932999999999999e-05 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:112415 9.3 9:c7576938:::benchmark_data_smithi107_300016_object14843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:23:42.113 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 224.43687580100001, 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29174138700000002, 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 112415 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:23:42.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596434+0000", 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7590000000000002e-06 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596440+0000", 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3129999999999998e-06 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596445+0000", 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0579999999999997e-06 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601164+0000", 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047186659999999998 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601299+0000", 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013497699999999999 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601421+0000", 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012268199999999999 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.607663+0000", 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0062417699999999998 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.608492+0000", 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082930000000000005 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887363+0000", 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27887095200000001 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887422+0000", 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.889e-05 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.888174+0000", 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075141200000000002 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141939 9.7 e201/188 9:ee97ad70:::benchmark_data_smithi107_300016_object44367:head v 201'9015, mlcod=201'9011)", 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 181.10241692700001, 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29274838399999997, 2023-10-09T06:23:42.120 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930890+0000", 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930897+0000", 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930907+0000", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0074e-05 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930913+0000", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3160000000000002e-06 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935291+0000", 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043775979999999999 2023-10-09T06:23:42.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935312+0000", 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1523e-05 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223606+0000", 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28829408499999998 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223639+0000", 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3176999999999998e-05 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141944 9.f e201/152 9:f293b650:::benchmark_data_smithi107_300016_object44372:head v 201'8823, mlcod=201'8821)", 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 181.093923824, 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30728261400000001, 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.125 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939383+0000", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939389+0000", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939397+0000", 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2600000000000005e-06 2023-10-09T06:23:42.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939404+0000", 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8419999999999999e-06 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939455+0000", 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1220000000000001e-05 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939471+0000", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5988e-05 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246587+0000", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.307116161 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246667+0000", 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9617999999999996e-05 2023-10-09T06:23:42.129 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141947 9.2 9:429ff582:::benchmark_data_smithi107_300016_object44375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 181.09290213200001, 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29032069100000002, 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.130 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141947 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940405+0000", 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940410+0000", 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8690000000000001e-06 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940422+0000", 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1296e-05 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940429+0000", 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1130000000000003e-06 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940473+0000", 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4391999999999998e-05 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219069+0000", 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27859536699999998 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219175+0000", 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106272 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.226985+0000", 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0078100460000000002 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230576+0000", 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035914990000000002 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230642+0000", 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5840000000000007e-05 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230689+0000", 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6653999999999998e-05 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230726+0000", 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7589000000000002e-05 2023-10-09T06:23:42.136 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141951 9.1 9:831efef4:::benchmark_data_smithi107_300016_object44379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 181.08911718499999, 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.286240618, 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.137 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141951 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:23:42.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944189+0000", 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944195+0000", 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8109999999999999e-06 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219289+0000", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.275094218 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219299+0000", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219377+0000", 2023-10-09T06:23:42.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7982e-05 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219481+0000", 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103836 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219580+0000", 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9399000000000002e-05 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227278+0000", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0076973400000000004 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227425+0000", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014719900000000001 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230360+0000", 2023-10-09T06:23:42.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029345669999999999 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230398+0000", 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8917e-05 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230431+0000", 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2916000000000001e-05 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.143 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141948 9.f e201/152 9:f4f21468:::benchmark_data_smithi107_300016_object44376:head v 201'8824, mlcod=201'8821)", 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 181.08881391899999, 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30220260900000001, 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944493+0000", 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944500+0000", 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218928+0000", 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27442793999999998 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218942+0000", 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.412e-05 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218983+0000", 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0599999999999998e-05 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219005+0000", 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2347000000000001e-05 2023-10-09T06:23:42.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246673+0000", 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027668321999999999 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246697+0000", 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3408999999999999e-05 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189419 9.8 9:1506b9c6:::benchmark_data_smithi107_300016_object91847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 111.495693339, 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99043692000000005, 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189419 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.149 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537613+0000", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537620+0000", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2089999999999999e-06 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537637+0000", 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7238e-05 2023-10-09T06:23:42.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537645+0000", 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8110000000000003e-06 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537696+0000", 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0652000000000002e-05 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:23:42.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537708+0000", 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2262999999999999e-05 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508194+0000", 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97048550899999997 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508310+0000", 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116286 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508418+0000", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010815999999999999 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.518815+0000", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010397269000000001 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524555+0000", 2023-10-09T06:23:42.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057400180000000004 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.527965+0000", 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034098969999999998 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528013+0000", 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8173000000000001e-05 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528052+0000", 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8163e-05 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189440 9.8 9:186cfebb:::benchmark_data_smithi107_300016_object91868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 111.450949749, 2023-10-09T06:23:42.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94578946799999997, 2023-10-09T06:23:42.157 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.157 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.157 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189440 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582357+0000", 2023-10-09T06:23:42.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582364+0000", 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5750000000000003e-06 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582377+0000", 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3012e-05 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582385+0000", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0059999999999992e-06 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582445+0000", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0266000000000002e-05 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582457+0000", 2023-10-09T06:23:42.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1567e-05 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511852+0000", 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.92939540200000004 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511956+0000", 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103732 2023-10-09T06:23:42.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.512066+0000", 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109555 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.520344+0000", 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0082784669999999994 2023-10-09T06:23:42.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524583+0000", 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004239306 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528084+0000", 2023-10-09T06:23:42.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035007110000000001 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528117+0000", 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3117999999999997e-05 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528148+0000", 2023-10-09T06:23:42.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0361e-05 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189461 9.8 9:171409ee:::benchmark_data_smithi107_300016_object91889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 111.406833265, 2023-10-09T06:23:42.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.90177728999999995, 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189461 2023-10-09T06:23:42.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:23:42.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626473+0000", 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626479+0000", 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8989999999999998e-06 2023-10-09T06:23:42.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626490+0000", 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0424999999999999e-05 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626495+0000", 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2089999999999996e-06 2023-10-09T06:23:42.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626547+0000", 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1792000000000002e-05 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626557+0000", 2023-10-09T06:23:42.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0407e-05 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515348+0000", 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.88879119399999995 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515455+0000", 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106475 2023-10-09T06:23:42.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515561+0000", 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010639199999999999 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524611+0000", 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090500259999999992 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.526227+0000", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016156779999999999 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528177+0000", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001950212 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528210+0000", 2023-10-09T06:23:42.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2511000000000001e-05 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528252+0000", 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2354000000000001e-05 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:23:42.175 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:23:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:42.176 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:42.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:42.278 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:42.302 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:42] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-10-09T06:23:42.305 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: osdmap e264: 8 total, 8 up, 7 in 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: failed to encode map e264 with expected crc 2023-10-09T06:23:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:41 smithi150 ceph-mon[194178]: pgmap v802: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 106 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2805/270075 objects degraded (1.039%); 2496/270075 objects misplaced (0.924%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:23:42.751 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 91261 91245 74.0167 13.875 0.0108052 0.0135065 2023-10-09T06:23:42.848 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:42.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:42.955 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:43.027 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 2805/270075 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:23:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[212750]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:23:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 2805/270075 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:23:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[220921]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[220921]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[220921]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:23:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:42 smithi107 ceph-mon[220921]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:42 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 2805/270075 objects degraded (1.039%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:23:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:42 smithi150 ceph-mon[194178]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:42 smithi150 ceph-mon[194178]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:42 smithi150 ceph-mon[194178]: osdmap e265: 8 total, 8 up, 7 in 2023-10-09T06:23:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:42 smithi150 ceph-mon[194178]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.466 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:43.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:43.485 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:23:43.577 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:43.632 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:43.650 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:23:43.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 92370 92354 73.9546 69.3125 0.00992233 0.0135173 2023-10-09T06:23:43.800 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:43.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:43.906 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:43.907 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:43.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:43 smithi107 ceph-mon[212750]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:43 smithi107 ceph-mon[212750]: pgmap v804: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 8377/273216 objects degraded (3.066%); 4529/273216 objects misplaced (1.658%); 11 MiB/s, 181 objects/s recovering 2023-10-09T06:23:43.989 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:43 smithi107 ceph-mon[220921]: failed to encode map e265 with expected crc 2023-10-09T06:23:43.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:43 smithi107 ceph-mon[220921]: pgmap v804: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 8377/273216 objects degraded (3.066%); 4529/273216 objects misplaced (1.658%); 11 MiB/s, 181 objects/s recovering 2023-10-09T06:23:43.994 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:44.130 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:44.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:44.237 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:44.327 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:43 smithi150 ceph-mon[194178]: failed to encode map e265 with expected crc 2023-10-09T06:23:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:43 smithi150 ceph-mon[194178]: pgmap v804: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 8377/273216 objects degraded (3.066%); 4529/273216 objects misplaced (1.658%); 11 MiB/s, 181 objects/s recovering 2023-10-09T06:23:44.467 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:44.558 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:44.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:44.662 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:44.728 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:44.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 93316 93300 73.7653 59.125 0.0220764 0.0135515 2023-10-09T06:23:44.824 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:44.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:44.940 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:44.977 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:23:45.008 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/292500606' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:44 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.256 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:45.373 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:45.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:45.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:45.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/292500606' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:44 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.378 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/292500606' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.378 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:44 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:23:45.479 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:45.539 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:45.632 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:45.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:45.742 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:45.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:23:45.754947+0000 min lat: 0.00112444 max lat: 2.25565 avg lat: 0.0135577 2023-10-09T06:23:45.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:23:45.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 94457 94441 73.7346 71.3125 0.0273659 0.0135577 2023-10-09T06:23:45.789 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[212750]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[212750]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[212750]: pgmap v806: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 274 op/s; 8377/273561 objects degraded (3.062%); 4529/273561 objects misplaced (1.656%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[220921]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:23:46.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[220921]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:23:46.082 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[220921]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.082 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:45 smithi107 ceph-mon[220921]: pgmap v806: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 274 op/s; 8377/273561 objects degraded (3.062%); 4529/273561 objects misplaced (1.656%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:23:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:45 smithi150 ceph-mon[194178]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:45 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:23:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:45 smithi150 ceph-mon[194178]: osdmap e266: 8 total, 8 up, 7 in 2023-10-09T06:23:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:45 smithi150 ceph-mon[194178]: failed to encode map e266 with expected crc 2023-10-09T06:23:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:45 smithi150 ceph-mon[194178]: pgmap v806: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 274 op/s; 8377/273561 objects degraded (3.062%); 4529/273561 objects misplaced (1.656%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T06:23:46.353 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:23:46.384 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:46.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:46.493 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:46.606 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:46.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 95332 95316 73.4995 54.6875 0.00293409 0.0135984 2023-10-09T06:23:47.269 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:47.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:47.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:47.374 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:47.449 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:47.560 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:47.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:47.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:47.670 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:47.728 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:47.728 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:47.728 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8377/273561 objects degraded (3.062%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:47.728 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3200643567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.728 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.733 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:47.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 96429 96413 73.4393 68.5625 0.00717566 0.013612 2023-10-09T06:23:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8377/273561 objects degraded (3.062%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:47.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3200643567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8377/273561 objects degraded (3.062%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3200643567' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:23:47.918 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:47.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:47.931 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:48.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:48.114 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:48.143 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:23:48.209 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:23:48.209 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:23:48.209 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.209 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45650.0:96883 10.c e267/238 10:35686190:::benchmark_data_smithi107_363200_object96882:head v 267'5891, mlcod=267'5889)", 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:48.205820+0000", 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00036998100000000002, 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00039136299999999999, 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205820+0000", 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.210 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205820+0000", 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205818+0000", 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:23:48.211 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205848+0000", 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9390000000000002e-05 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205866+0000", 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8547e-05 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205875+0000", 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8829999999999995e-06 2023-10-09T06:23:48.212 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205927+0000", 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2426000000000003e-05 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205951+0000", 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3475000000000001e-05 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:23:48.213 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45650.0:96884 10.c e267/238)", 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:48.205893+0000", 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00029672500000000002, 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00038382200000000003, 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "queued for pg", 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.214 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205893+0000", 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205893+0000", 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:23:48.215 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205893+0000", 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205919+0000", 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6679e-05 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.216 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205926+0000", 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6420000000000001e-06 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:48.205930+0000", 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5249999999999999e-06 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: ], 2023-10-09T06:23:48.217 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 2 2023-10-09T06:23:48.218 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:23:48.222 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:48.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:48.334 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:48.446 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:48.577 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:48.611 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:48.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:48.623 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: pgmap v807: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s; 13997/278925 objects degraded (5.018%); 4529/278925 objects misplaced (1.624%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/661779409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:48.676 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:48.711 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:48.724 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:48.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: pgmap v807: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s; 13997/278925 objects degraded (5.018%); 4529/278925 objects misplaced (1.624%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T06:23:48.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/661779409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: pgmap v807: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 30 MiB/s wr, 479 op/s; 13997/278925 objects degraded (5.018%); 4529/278925 objects misplaced (1.624%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:23:48.736 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: osdmap e267: 8 total, 8 up, 7 in 2023-10-09T06:23:48.737 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.737 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: failed to encode map e267 with expected crc 2023-10-09T06:23:48.737 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/661779409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:48.752 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:48.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 97445 97429 73.3195 63.5 0.0118802 0.0136343 2023-10-09T06:23:48.788 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:48.788 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:48.828 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:48.868 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:48.870 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.870 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.870 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.871 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.871 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.871 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.871 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.871 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:23:48.881 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:48.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:48.989 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:49.043 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:49.136 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:49.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:49.240 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:49.313 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:49.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 98515 98499 73.2428 66.875 0.00395808 0.0136469 2023-10-09T06:23:49.778 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:23:49.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[212750]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[220921]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[220921]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:49 smithi107 ceph-mon[220921]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:49 smithi150 ceph-mon[194178]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:49 smithi150 ceph-mon[194178]: failed to encode map e267 with expected crc 2023-10-09T06:23:49.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:49 smithi150 ceph-mon[194178]: failed to encode map e267 with expected crc 2023-10-09T06:23:50.402 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:50.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 99430 99414 73.054 57.1875 0.0160725 0.0136842 2023-10-09T06:23:50.865 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:50.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:50.896 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:50 smithi150 ceph-mon[194178]: pgmap v809: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 13997/287118 objects degraded (4.875%); 4025/287118 objects misplaced (1.402%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:23:50.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:50 smithi107 ceph-mon[212750]: pgmap v809: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 13997/287118 objects degraded (4.875%); 4025/287118 objects misplaced (1.402%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:23:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:50 smithi107 ceph-mon[220921]: pgmap v809: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 13997/287118 objects degraded (4.875%); 4025/287118 objects misplaced (1.402%); 11 MiB/s, 176 objects/s recovering 2023-10-09T06:23:50.973 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:51.021 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:51.156 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:51.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:51.262 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:51.325 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:51.458 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:51.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:51.569 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:51.609 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:51.653 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:51.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:51.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 100340 100324 72.8658 56.875 0.00443199 0.0136892 2023-10-09T06:23:51.756 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:51.783 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:51.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:51.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:51.991 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:52.043 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:52.274 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:52.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:52.384 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:23:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:23:52] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T06:23:52.443 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:52.626 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:52.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:52.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:52.644 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:23:52.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:52 smithi150 ceph-mon[194178]: pgmap v810: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 13997/290400 objects degraded (4.820%); 4025/290400 objects misplaced (1.386%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:23:52.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:23:52.729 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13997/290400 objects degraded (4.820%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:52.739 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:23:52.754 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 101466 101450 72.8371 70.375 0.0137722 0.0137251 2023-10-09T06:23:52.806 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[212750]: pgmap v810: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 13997/290400 objects degraded (4.820%); 4025/290400 objects misplaced (1.386%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:23:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:23:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13997/290400 objects degraded (4.820%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[220921]: pgmap v810: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.9 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 45 MiB/s wr, 715 op/s; 13997/290400 objects degraded (4.820%); 4025/290400 objects misplaced (1.386%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:23:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:23:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13997/290400 objects degraded (4.820%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:52.916 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:23:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:52.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:23:52.977 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:23:52.983 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:23:52.983 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:23:53.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:23:53.019 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:53.081 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:53.202 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:53.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:53.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:53.308 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:53.337 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:53.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[212750]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:23:53.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[212750]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:23:53.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 102393 102377 72.6678 57.9375 0.00268372 0.0137561 2023-10-09T06:23:53.758 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:53.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:53.863 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:53.872 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:23:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[220921]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[220921]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:23:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:53 smithi107 ceph-mon[220921]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:23:53.904 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:53 smithi150 ceph-mon[194178]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:53 smithi150 ceph-mon[194178]: failed to encode map e268 with expected crc 2023-10-09T06:23:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4173542432' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:23:53.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:53 smithi150 ceph-mon[194178]: osdmap e268: 8 total, 8 up, 7 in 2023-10-09T06:23:53.978 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:53.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:54.084 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:54.096 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:54.132 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:54.152 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:54.367 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:54.369 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:23:54.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:54.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:54.476 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:54.542 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:54.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 103278 103262 72.4729 55.3125 0.00537583 0.0137944 2023-10-09T06:23:54.863 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:54 smithi107 ceph-mon[212750]: pgmap v812: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 13997/301383 objects degraded (4.644%); 3472/301383 objects misplaced (1.152%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:54.863 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:54 smithi107 ceph-mon[220921]: pgmap v812: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 13997/301383 objects degraded (4.644%); 3472/301383 objects misplaced (1.152%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:54.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:54 smithi150 ceph-mon[194178]: pgmap v812: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 13997/301383 objects degraded (4.644%); 3472/301383 objects misplaced (1.152%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:54.998 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:55.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:55.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:55.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:55.186 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:55.683 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:55.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:55.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0109 2023-10-09T06:23:55.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 104089 2023-10-09T06:23:55.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:23:55.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:23:55.755 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 72.2753 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 37.3195 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 229.875 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1156 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 597.121 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3678 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0138324 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0493629 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.25565 2023-10-09T06:23:55.756 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00112444 2023-10-09T06:23:55.760 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:23:55.792 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:55.859 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:56.131 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:56.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:56.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:56.242 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:23:56.269 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:56.312 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:23:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:56.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:23:56.416 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:23:56.444 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:56.688 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:23:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:56.703 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:56 smithi107 ceph-mon[212750]: pgmap v813: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 13997/302361 objects degraded (4.629%); 3472/302361 objects misplaced (1.148%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:56.704 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:56 smithi107 ceph-mon[220921]: pgmap v813: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 13997/302361 objects degraded (4.629%); 3472/302361 objects misplaced (1.148%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:56.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:23:56.715 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:23:56.806 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:23:56.876 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:57.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:56 smithi150 ceph-mon[194178]: pgmap v813: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 13997/302361 objects degraded (4.629%); 3472/302361 objects misplaced (1.148%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T06:23:57.171 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:23:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:57.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:23:57.277 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:23:57.304 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:23:57.360 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:23:57.363 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:57.401 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:23:57.413 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:23:57.413 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:23:57.435 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:23:57.518 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:23:57.518 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:23:57.518 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:23:57.546 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:23:57.559 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.559 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.560 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:23:57.671 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:23:57.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:57.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:23:57.779 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:57.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13997/302361 objects degraded (4.629%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:57.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:23:57.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.806 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3411027569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:57.808 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:57.854 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:57.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:57.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13997/302361 objects degraded (4.629%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3411027569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13997/302361 objects degraded (4.629%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: osdmap e269: 8 total, 8 up, 7 in 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:57.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3411027569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:23:57.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:57.985 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:58.389 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:23:58.802 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: pgmap v814: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 848 op/s; 13919/308157 objects degraded (4.517%); 3442/308157 objects misplaced (1.117%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: failed to encode map e269 with expected crc 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: pgmap v814: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 848 op/s; 13919/308157 objects degraded (4.517%); 3442/308157 objects misplaced (1.117%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: failed to encode map e269 with expected crc 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:58.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:23:58.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:58 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: pgmap v814: 109 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 52 MiB/s wr, 848 op/s; 13919/308157 objects degraded (4.517%); 3442/308157 objects misplaced (1.117%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:23:59.075 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: failed to encode map e269 with expected crc 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3961755991' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: osdmap e270: 8 total, 8 up, 7 in 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:58 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.198 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:23:59.342 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:23:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:59.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:23:59.446 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:23:59.506 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:59.580 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:23:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:59.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:23:59.687 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:23:59.711 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:59 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.711 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:23:59 smithi150 ceph-mon[194178]: failed to encode map e270 with expected crc 2023-10-09T06:23:59.714 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:23:59.755 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:23:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:23:59.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:23:59.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:23:59.933 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:00.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:59 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:24:00.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:23:59 smithi107 ceph-mon[220921]: failed to encode map e270 with expected crc 2023-10-09T06:24:00.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:59 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:24:00.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:23:59 smithi107 ceph-mon[212750]: failed to encode map e270 with expected crc 2023-10-09T06:24:00.206 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:00.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:00.316 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:00.343 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:00.405 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:00.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:00.509 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:00.575 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:00.712 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:00.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:00.818 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:00.818 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:24:00.818 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:24:00.818 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:00.819 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:00.842 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:00 smithi150 ceph-mon[194178]: pgmap v817: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 159 B/s rd, 59 MiB/s wr, 1.09k op/s; 13919/309543 objects degraded (4.497%); 2756/309543 objects misplaced (0.890%); 12 MiB/s, 198 objects/s recovering 2023-10-09T06:24:00.903 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:00.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:00 smithi107 ceph-mon[212750]: pgmap v817: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 159 B/s rd, 59 MiB/s wr, 1.09k op/s; 13919/309543 objects degraded (4.497%); 2756/309543 objects misplaced (0.890%); 12 MiB/s, 198 objects/s recovering 2023-10-09T06:24:00.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:00 smithi107 ceph-mon[220921]: pgmap v817: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 159 B/s rd, 59 MiB/s wr, 1.09k op/s; 13919/309543 objects degraded (4.497%); 2756/309543 objects misplaced (0.890%); 12 MiB/s, 198 objects/s recovering 2023-10-09T06:24:01.073 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:01.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:01.179 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:01.210 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:01.268 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:01.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:01.372 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:01.462 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:01.584 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:01.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:01.694 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:01.739 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:24:01.750 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:01.829 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:01.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:01.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:01.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:01.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:01.946 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:02.006 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:02.020 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:02.122 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:02.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:02.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:02.233 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:02.296 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:02.316 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:02] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:24:02.368 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:02.399 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:02.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:02.509 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.600 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:02.734 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:02.760 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:24:02.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:02.873 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:24:02.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[212750]: pgmap v818: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 30 MiB/s wr, 682 op/s; 13919/307695 objects degraded (4.524%); 2756/307695 objects misplaced (0.896%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:24:02.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3342050534' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13919/307695 objects degraded (4.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[220921]: pgmap v818: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 30 MiB/s wr, 682 op/s; 13919/307695 objects degraded (4.524%); 2756/307695 objects misplaced (0.896%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3342050534' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13919/307695 objects degraded (4.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:02.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:02 smithi150 ceph-mon[194178]: pgmap v818: 109 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 30 MiB/s wr, 682 op/s; 13919/307695 objects degraded (4.524%); 2756/307695 objects misplaced (0.896%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T06:24:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3342050534' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:24:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13919/307695 objects degraded (4.524%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:02.940 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:03.029 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:24:03.066 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:03.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:03.179 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:03.256 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:03.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:03.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:03.445 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:03.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:24:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[212750]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T06:24:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T06:24:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T06:24:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[212750]: failed to encode map e271 with expected crc 2023-10-09T06:24:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:24:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[220921]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T06:24:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[220921]: failed to encode map e271 with expected crc 2023-10-09T06:24:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[220921]: failed to encode map e271 with expected crc 2023-10-09T06:24:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:03 smithi107 ceph-mon[220921]: failed to encode map e271 with expected crc 2023-10-09T06:24:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:24:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:03 smithi150 ceph-mon[194178]: osdmap e271: 8 total, 8 up, 7 in 2023-10-09T06:24:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:03 smithi150 ceph-mon[194178]: failed to encode map e271 with expected crc 2023-10-09T06:24:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:03 smithi150 ceph-mon[194178]: failed to encode map e271 with expected crc 2023-10-09T06:24:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:03 smithi150 ceph-mon[194178]: failed to encode map e271 with expected crc 2023-10-09T06:24:04.424 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:04.424 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:24:04.424 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:24:04.424 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:24:04.425 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:04.809 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:04.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:04.920 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:04.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:24:04.949 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:04.992 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:04.992 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:04.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:05.095 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:05.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:04 smithi107 ceph-mon[212750]: pgmap v820: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 15 MiB/s wr, 984 op/s; 8068/298800 objects degraded (2.700%); 7877/298800 objects misplaced (2.636%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:24:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:04 smithi150 ceph-mon[194178]: pgmap v820: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 15 MiB/s wr, 984 op/s; 8068/298800 objects degraded (2.700%); 7877/298800 objects misplaced (2.636%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:24:05.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:04 smithi107 ceph-mon[220921]: pgmap v820: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 170 B/s rd, 15 MiB/s wr, 984 op/s; 8068/298800 objects degraded (2.700%); 7877/298800 objects misplaced (2.636%); 10 MiB/s, 164 objects/s recovering 2023-10-09T06:24:05.178 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:05.231 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:05.402 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:24:05.609 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:05.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:05.716 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:05.757 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:05.898 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:05.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:05.926 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:06.005 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.059 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:06.139 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:06.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:06.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:06.245 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:06.304 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:06.477 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:06.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:06.584 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:06.627 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:06.729 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:06.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:06.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:06.837 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:06.864 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:06.905 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:06.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:07.008 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:07.032 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:06 smithi150 ceph-mon[194178]: pgmap v821: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 128 B/s rd, 11 MiB/s wr, 774 op/s; 8068/298074 objects degraded (2.707%); 7877/298074 objects misplaced (2.643%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:24:07.034 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:07.105 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:07.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:07.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:07.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:06 smithi107 ceph-mon[212750]: pgmap v821: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 128 B/s rd, 11 MiB/s wr, 774 op/s; 8068/298074 objects degraded (2.707%); 7877/298074 objects misplaced (2.643%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:24:07.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:06 smithi107 ceph-mon[220921]: pgmap v821: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.1 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 128 B/s rd, 11 MiB/s wr, 774 op/s; 8068/298074 objects degraded (2.707%); 7877/298074 objects misplaced (2.643%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:24:07.208 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:07.260 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:07.417 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:07.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:07.523 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:07.570 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:07.834 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:07.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8068/298074 objects degraded (2.707%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:07.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:07 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:07.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:07.843 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8068/298074 objects degraded (2.707%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:07.844 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:07 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:07.942 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:162322 8.d 8:b0b2eb65:::benchmark_data_smithi107_251897_object45408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:24:07.983 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 556.78942192, 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.051913984, 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 162322 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188435+0000", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188434+0000", 2023-10-09T06:24:07.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188444+0000", 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0066e-05 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188459+0000", 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5274e-05 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.188466+0000", 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.131e-06 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.191241+0000", 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002774136 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:07.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:51.191248+0000", 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8590000000000005e-06 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223345+0000", 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.032096401 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223500+0000", 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015488200000000001 2023-10-09T06:24:07.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.223624+0000", 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000124547 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.229333+0000", 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057091090000000004 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240173+0000", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010839867 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240236+0000", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2704000000000001e-05 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:52.240349+0000", 2023-10-09T06:24:07.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113404 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176903 8.4 8:27754661:::benchmark_data_smithi107_251897_object59989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.27504372800001, 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1790666839999999, 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:07.991 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176903 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:07.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702814+0000", 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702812+0000", 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702818+0000", 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0780000000000001e-06 2023-10-09T06:24:07.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702831+0000", 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605e-05 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702837+0000", 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7039999999999999e-06 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702889+0000", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2024000000000002e-05 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.702901+0000", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2029e-05 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872747+0000", 2023-10-09T06:24:07.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16984611299999999 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872752+0000", 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4589999999999997e-06 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859323+0000", 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9865709140000001 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859469+0000", 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000145601 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.859570+0000", 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010074800000000001 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:07.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872610+0000", 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013039996 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.876987+0000", 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043772840000000004 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:07.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881653+0000", 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046657829999999997 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881708+0000", 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5614e-05 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881880+0000", 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017200200000000001 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176909 8.4 8:22feb7e8:::benchmark_data_smithi107_251897_object59995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.27083315300001, 2023-10-09T06:24:08.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1750447849999999, 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176909 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:24:08.002 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707024+0000", 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707022+0000", 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707030+0000", 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8760000000000005e-06 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707048+0000", 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7941999999999999e-05 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707054+0000", 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9989999999999997e-06 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707113+0000", 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8666999999999999e-05 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.707120+0000", 2023-10-09T06:24:08.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3939999999999998e-06 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872779+0000", 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.165658478 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872784+0000", 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9880000000000004e-06 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864131+0000", 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9913474199999999 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864242+0000", 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110747 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:24:08.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.864344+0000", 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010181 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.876999+0000", 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012655680000000001 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877174+0000", 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017434199999999999 2023-10-09T06:24:08.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881928+0000", 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047547379999999997 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.881972+0000", 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3727000000000003e-05 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882069+0000", 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6674000000000003e-05 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176918 8.d 8:b53b7e9b:::benchmark_data_smithi107_251897_object60004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.25489896199997, 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1380759039999999, 2023-10-09T06:24:08.010 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176918 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722958+0000", 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722957+0000", 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722963+0000", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1469999999999998e-06 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722976+0000", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2649999999999999e-05 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.722982+0000", 2023-10-09T06:24:08.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1870000000000002e-06 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.723026+0000", 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3983000000000002e-05 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.723034+0000", 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0139999999999998e-06 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.871899+0000", 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14886518500000001 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.871912+0000", 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3059e-05 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850243+0000", 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.978331117 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850348+0000", 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104239 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.850442+0000", 2023-10-09T06:24:08.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4621000000000001e-05 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856219+0000", 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057764180000000002 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856284+0000", 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4817e-05 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.860850+0000", 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045668949999999996 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.860890+0000", 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9864999999999999e-05 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.018 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861034+0000", 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014390900000000001 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176926 8.4 8:2d6c801d:::benchmark_data_smithi107_251897_object60012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.24432847699995, 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1546555550000002, 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.019 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176926 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733529+0000", 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733527+0000", 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733534+0000", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5610000000000004e-06 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733550+0000", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6461999999999999e-05 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733557+0000", 2023-10-09T06:24:08.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.951e-06 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733609+0000", 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2036999999999997e-05 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.733618+0000", 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9420000000000001e-06 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872805+0000", 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13918652400000001 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872809+0000", 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0659999999999997e-06 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868505+0000", 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.995695926 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868615+0000", 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110449 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.868720+0000", 2023-10-09T06:24:08.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105222 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877006+0000", 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082859040000000002 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.877206+0000", 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019972699999999999 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888036+0000", 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010830358 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888083+0000", 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6508999999999998e-05 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888184+0000", 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010166300000000001 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176930 8.d 8:b7d0ab16:::benchmark_data_smithi107_251897_object60016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.23801850899997, 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.126278691, 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.028 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176930 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739839+0000", 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739838+0000", 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739843+0000", 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1479999999999997e-06 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739858+0000", 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4751e-05 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739863+0000", 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1619999999999999e-06 2023-10-09T06:24:08.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739908+0000", 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5120000000000002e-05 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.739916+0000", 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2509999999999994e-06 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872532+0000", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13261576799999999 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872537+0000", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1209999999999998e-06 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856314+0000", 2023-10-09T06:24:08.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9837771659999999 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856430+0000", 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115678 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.856532+0000", 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000102012 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861142+0000", 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046098279999999998 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.861431+0000", 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00028886999999999999 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.865951+0000", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045204829999999996 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.865993+0000", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1465999999999999e-05 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.866117+0000", 2023-10-09T06:24:08.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012464099999999999 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:176936 8.4 8:22e0b0cb:::benchmark_data_smithi107_251897_object60022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 536.20871313800001, 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1192077970000001, 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.037 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 176936 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769144+0000", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769142+0000", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769150+0000", 2023-10-09T06:24:08.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6930000000000007e-06 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769167+0000", 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7257000000000001e-05 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769187+0000", 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9686000000000001e-05 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769239+0000", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2289e-05 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.769249+0000", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7450000000000007e-06 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872828+0000", 2023-10-09T06:24:08.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.103578663 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:11.872835+0000", 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8569999999999997e-06 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872651+0000", 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.999816456 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872800+0000", 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014936699999999999 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.872917+0000", 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011622500000000001 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882080+0000", 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0091632380000000006 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.882556+0000", 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047562599999999999 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888224+0000", 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0056688099999999998 2023-10-09T06:24:08.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888260+0000", 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5235999999999999e-05 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:13.888352+0000", 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2367999999999994e-05 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.045 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:199436 8.d 8:ba48c3db:::benchmark_data_smithi107_251897_object82522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 503.80342522299998, 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0633727580000001, 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.046 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 199436 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174432+0000", 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174431+0000", 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174437+0000", 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174453+0000", 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6113e-05 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174459+0000", 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9200000000000001e-06 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174522+0000", 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2923999999999996e-05 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174530+0000", 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.181995+0000", 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0074647359999999996 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.182005+0000", 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8849999999999994e-06 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225165+0000", 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0431607389999999 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225304+0000", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000138639 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.225396+0000", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1552999999999996e-05 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230017+0000", 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046212960000000004 2023-10-09T06:24:08.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.234320+0000", 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043027839999999996 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237733+0000", 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034128320000000002 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237769+0000", 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237805+0000", 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5726000000000003e-05 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:199440 8.d 8:b1aca3b0:::benchmark_data_smithi107_251897_object82526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e135)", 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 503.80338594300002, 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0634993260000001, 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 199440 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.055 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174471+0000", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174474+0000", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.027e-06 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174482+0000", 2023-10-09T06:24:08.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2260000000000005e-06 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174486+0000", 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6219999999999998e-06 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174598+0000", 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000112007 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.174606+0000", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6989999999999992e-06 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.184897+0000", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010290875 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:44.184909+0000", 2023-10-09T06:24:08.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1392999999999999e-05 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230049+0000", 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.045139968 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230145+0000", 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6695999999999997e-05 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.230229+0000", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3279e-05 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237876+0000", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0076473210000000003 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237909+0000", 2023-10-09T06:24:08.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3229000000000003e-05 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237916+0000", 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0840000000000003e-06 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237948+0000", 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1696000000000003e-05 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:15:45.237971+0000", 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2626e-05 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231182 8.8 8:1499a486:::benchmark_data_smithi107_251897_object114268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.41177532400002, 2023-10-09T06:24:08.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.088702015, 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231182 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:24:08.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566082+0000", 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566081+0000", 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566086+0000", 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8790000000000002e-06 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566097+0000", 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1422e-05 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.566104+0000", 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2280000000000003e-06 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581612+0000", 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015507917 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581747+0000", 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135372 2023-10-09T06:24:08.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.581874+0000", 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127367 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.603525+0000", 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.021651113999999999 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608048+0000", 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045228769999999998 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608056+0000", 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9440000000000005e-06 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.654661+0000", 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0466043810000001 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.654784+0000", 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012337900000000001 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.070 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231183 8.8 8:18f72ceb:::benchmark_data_smithi107_251897_object114269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.41020510599998, 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.087511761, 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231183 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.071 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567652+0000", 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567651+0000", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567656+0000", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5390000000000001e-06 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567665+0000", 2023-10-09T06:24:08.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9449999999999999e-06 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.567670+0000", 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6010000000000002e-06 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587108+0000", 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019437578 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587234+0000", 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012631599999999999 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.587314+0000", 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9302999999999996e-05 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608084+0000", 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020770617000000002 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608090+0000", 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4040000000000003e-06 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655105+0000", 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047015048 2023-10-09T06:24:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655164+0000", 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9188999999999999e-05 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231188 8.8 8:1a06de55:::benchmark_data_smithi107_251897_object114274:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:24:08.077 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.402975728, 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0803605979999999, 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231188 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574882+0000", 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574880+0000", 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574885+0000", 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8049999999999997e-06 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574893+0000", 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9859999999999994e-06 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574898+0000", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8559999999999998e-06 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592228+0000", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017330007000000001 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592360+0000", 2023-10-09T06:24:08.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013205199999999999 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592479+0000", 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119114 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608169+0000", 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.01568955 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608175+0000", 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0469999999999999e-06 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655187+0000", 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047012791 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655242+0000", 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4656e-05 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231189 8.8 8:17d1e237:::benchmark_data_smithi107_251897_object114275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.40295465600002, 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.080469938, 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.084 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231189 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574903+0000", 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574902+0000", 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574904+0000", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1169999999999998e-06 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574909+0000", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7740000000000004e-06 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.574912+0000", 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2559999999999998e-06 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597542+0000", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022629911999999999 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597662+0000", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120011 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.597760+0000", 2023-10-09T06:24:08.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7397000000000003e-05 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608199+0000", 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010439321999999999 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608204+0000", 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3709999999999999e-06 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655287+0000", 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.047082477 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.655373+0000", 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5600999999999996e-05 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231200 8.8 8:19497162:::benchmark_data_smithi107_251897_object114286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.38511704299998, 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1339951880000001, 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231200 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.091 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592740+0000", 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592739+0000", 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592746+0000", 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2910000000000002e-06 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592762+0000", 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5903999999999999e-05 2023-10-09T06:24:08.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.592769+0000", 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2899999999999997e-06 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608231+0000", 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015461805 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608236+0000", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0869999999999997e-06 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707544+0000", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0993079569999999 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707644+0000", 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010024800000000001 2023-10-09T06:24:08.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.707766+0000", 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012206600000000001 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.721160+0000", 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013393693999999999 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726598+0000", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0054379540000000001 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726652+0000", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4061000000000003e-05 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726735+0000", 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3263999999999994e-05 2023-10-09T06:24:08.097 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231204 8.8 8:148b492c:::benchmark_data_smithi107_251897_object114290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.38193147700002, 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1386598640000001, 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.098 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231204 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595926+0000", 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595925+0000", 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595932+0000", 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4150000000000002e-06 2023-10-09T06:24:08.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595947+0000", 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5262000000000001e-05 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.595954+0000", 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6080000000000001e-06 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608567+0000", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012613447 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608574+0000", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7889999999999997e-06 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.712886+0000", 2023-10-09T06:24:08.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.104311611 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.713018+0000", 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013194699999999999 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 7", 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.713140+0000", 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012193600000000001 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726754+0000", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013614131999999999 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734474+0000", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.007720144 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734509+0000", 2023-10-09T06:24:08.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5472999999999997e-05 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.734586+0000", 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6318999999999999e-05 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231191 8.0 8:0eb72051:::benchmark_data_smithi107_251897_object114277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:24:08.105 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.38102530100002, 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.106525803, 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231191 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596832+0000", 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596831+0000", 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:08.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596836+0000", 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1610000000000002e-06 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596847+0000", 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1122e-05 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596853+0000", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9220000000000002e-06 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608255+0000", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011402123 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608260+0000", 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.994e-06 2023-10-09T06:24:08.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703247+0000", 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0949865670000001 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703358+0000", 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110809 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.110 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231194 8.0 8:0cbc2529:::benchmark_data_smithi107_251897_object114280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.38099865599997, 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.106776822, 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.111 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231194 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596859+0000", 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596858+0000", 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596861+0000", 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3479999999999998e-06 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596866+0000", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.49e-06 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596870+0000", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4620000000000001e-06 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608278+0000", 2023-10-09T06:24:08.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011408702999999999 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608283+0000", 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9649999999999999e-06 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703571+0000", 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0952875129999999 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703635+0000", 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4727000000000006e-05 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231198 8.0 8:080624c4:::benchmark_data_smithi107_251897_object114284:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.38098421299998, 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.12997952, 2023-10-09T06:24:08.116 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231198 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596873+0000", 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596875+0000", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0490000000000002e-06 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596880+0000", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8910000000000003e-06 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.596883+0000", 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.253e-06 2023-10-09T06:24:08.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608302+0000", 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011419123 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.608307+0000", 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1109999999999999e-06 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703683+0000", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0953758200000001 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for missing object", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703802+0000", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011947600000000001 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726773+0000", 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022970059000000001 2023-10-09T06:24:08.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.726853+0000", 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0047000000000001e-05 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:231206 8.0 8:0fa8cc49:::benchmark_data_smithi107_251897_object114292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:24:08.122 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.36550842499997, 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1043428710000001, 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 231206 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:08.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612349+0000", 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612347+0000", 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612353+0000", 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4909999999999996e-06 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612367+0000", 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.419e-05 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:08.125 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612373+0000", 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8980000000000001e-06 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612410+0000", 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6918999999999997e-05 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:29.612415+0000", 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7620000000000003e-06 2023-10-09T06:24:08.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703839+0000", 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0914244500000001 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.703923+0000", 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3504999999999994e-05 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.704006+0000", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3324000000000004e-05 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.712425+0000", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0084194160000000007 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716586+0000", 2023-10-09T06:24:08.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041602469999999997 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716623+0000", 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7771999999999997e-05 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:16:30.716692+0000", 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8279000000000001e-05 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.129 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.130 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:08.130 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:08.130 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:08.130 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:08.131 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8068/298074 objects degraded (2.707%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:08.131 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:07 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:08.131 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:08.143 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:08.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:08.257 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.302 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:08.389 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:08.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:08.495 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:08.541 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:08.616 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:08.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:08.722 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:08.754 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:08.841 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:08.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:08.877 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:08 smithi150 ceph-mon[194178]: pgmap v822: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 673 KiB/s wr, 668 op/s; 7809/292194 objects degraded (2.673%); 7354/292194 objects misplaced (2.517%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:08.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:24:08.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:08 smithi107 ceph-mon[212750]: pgmap v822: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 673 KiB/s wr, 668 op/s; 7809/292194 objects degraded (2.673%); 7354/292194 objects misplaced (2.517%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:09.011 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:08 smithi107 ceph-mon[220921]: pgmap v822: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 673 KiB/s wr, 668 op/s; 7809/292194 objects degraded (2.673%); 7354/292194 objects misplaced (2.517%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:09.012 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:09.147 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:24:09.179 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:09.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:09.203 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:24:09.253 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:24:09.292 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:09.306 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:24:09.354 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:24:09.384 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:09.396 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:24:09.437 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:09.455 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:24:09.491 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:24:09.493 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.506 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:24:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:09.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:09.608 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:09.674 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:09.751 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:09.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:09.819 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:24:09.856 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:09.926 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:10.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2364195561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:10.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:09 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:10.108 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2364195561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2364195561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:10.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:10.663 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:10.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:10.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:10.774 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:10.817 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: pgmap v823: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 601 KiB/s wr, 793 op/s; 7809/286275 objects degraded (2.728%); 7354/286275 objects misplaced (2.569%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: pgmap v823: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 601 KiB/s wr, 793 op/s; 7809/286275 objects degraded (2.728%); 7354/286275 objects misplaced (2.569%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: failed to encode map e272 with expected crc 2023-10-09T06:24:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:10 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:10.933 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:10.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:11.038 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:11.098 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: pgmap v823: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 601 KiB/s wr, 793 op/s; 7809/286275 objects degraded (2.728%); 7354/286275 objects misplaced (2.569%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:24:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:24:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1009800666' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: osdmap e272: 8 total, 8 up, 7 in 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: failed to encode map e272 with expected crc 2023-10-09T06:24:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:10 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:11.563 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:24:11.563 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:24:11.601 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:11.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:11.710 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:11.790 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:11.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:11.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:24:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:12.148 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:12.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:12] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:24:12.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:12.253 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:12.307 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:12.535 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:24:12.591 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:12.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:12.697 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:12.769 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:12.890 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:12.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:13.002 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:13.079 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[212750]: pgmap v825: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 7809/283560 objects degraded (2.754%); 7354/283560 objects misplaced (2.593%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:24:13.079 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.079 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.079 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7809/283560 objects degraded (2.754%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:13.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[220921]: pgmap v825: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 7809/283560 objects degraded (2.754%); 7354/283560 objects misplaced (2.593%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:24:13.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.081 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7809/283560 objects degraded (2.754%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:13.094 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:12 smithi150 ceph-mon[194178]: pgmap v825: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 7809/283560 objects degraded (2.754%); 7354/283560 objects misplaced (2.593%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T06:24:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7809/283560 objects degraded (2.754%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:13.234 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:13.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:13.350 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:13.373 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:13.433 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:13.529 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:13.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:13.633 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:13.719 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:13.863 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:13.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:13.922 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:24:13.942 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:24:13.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[212750]: pgmap v826: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 7571/274830 objects degraded (2.755%); 6853/274830 objects misplaced (2.494%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3410207143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[220921]: pgmap v826: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 7571/274830 objects degraded (2.755%); 6853/274830 objects misplaced (2.494%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3410207143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:13.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:13 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:13.973 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:14.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:13 smithi150 ceph-mon[194178]: pgmap v826: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 798 op/s; 7571/274830 objects degraded (2.755%); 6853/274830 objects misplaced (2.494%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:24:14.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3410207143' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:14.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:24:14.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:13 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:14.028 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:14.200 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:24:14.650 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:14.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:14.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:14.827 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:24:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:14 smithi150 ceph-mon[194178]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T06:24:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:14 smithi150 ceph-mon[194178]: failed to encode map e273 with expected crc 2023-10-09T06:24:14.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:14 smithi150 ceph-mon[194178]: failed to encode map e273 with expected crc 2023-10-09T06:24:14.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:14 smithi150 ceph-mon[194178]: 10.10 deep-scrub starts 2023-10-09T06:24:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:24:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[212750]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T06:24:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T06:24:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[212750]: failed to encode map e273 with expected crc 2023-10-09T06:24:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[212750]: 10.10 deep-scrub starts 2023-10-09T06:24:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:24:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[220921]: osdmap e273: 8 total, 8 up, 7 in 2023-10-09T06:24:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[220921]: failed to encode map e273 with expected crc 2023-10-09T06:24:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[220921]: failed to encode map e273 with expected crc 2023-10-09T06:24:15.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:14 smithi107 ceph-mon[220921]: 10.10 deep-scrub starts 2023-10-09T06:24:15.190 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:15.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:15.296 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:15.350 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:15.467 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:15.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:15.574 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:15.644 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:24:15.695 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:15.859 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:15.862 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:15.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:15 smithi107 ceph-mon[212750]: pgmap v828: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 7571/274080 objects degraded (2.762%); 6853/274080 objects misplaced (2.500%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:24:15.920 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:15.955 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:24:15.955 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:24:15.967 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:16.022 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:15 smithi150 ceph-mon[194178]: pgmap v828: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 7571/274080 objects degraded (2.762%); 6853/274080 objects misplaced (2.500%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:24:16.261 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:16.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:16.332 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:15 smithi107 ceph-mon[220921]: pgmap v828: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 7571/274080 objects degraded (2.762%); 6853/274080 objects misplaced (2.500%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:24:16.365 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:16.399 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:16.462 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:16.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:16.569 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:16.610 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:17.237 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:17.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:17.313 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:24:17.350 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:17.411 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7571/274080 objects degraded (2.762%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7571/274080 objects degraded (2.762%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7571/274080 objects degraded (2.762%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:18.170 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:18.593 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:18.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:18.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:18.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:18.734 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:18.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:18 smithi107 ceph-mon[212750]: pgmap v829: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 7336/268560 objects degraded (2.732%); 6273/268560 objects misplaced (2.336%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:24:18.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:18 smithi107 ceph-mon[220921]: pgmap v829: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 7336/268560 objects degraded (2.732%); 6273/268560 objects misplaced (2.336%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:24:18.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:18 smithi150 ceph-mon[194178]: pgmap v829: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 7336/268560 objects degraded (2.732%); 6273/268560 objects misplaced (2.336%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T06:24:19.171 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:19.186 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:19.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:19.291 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:19.323 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:19.892 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:19 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:19.996 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:19 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:19.996 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:19 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:20.188 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:20.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:20.445 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:20.694 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:24:21.070 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:24:21.070 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:24:21.078 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:24:21.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:20 smithi150 ceph-mon[194178]: pgmap v830: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 7336/261579 objects degraded (2.805%); 6273/261579 objects misplaced (2.398%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:21.079 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:20 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:21.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.080 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3543550430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:21.081 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:24:21.081 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:24:21.082 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:24:21.082 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:24:21.082 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:24:21.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[212750]: pgmap v830: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 7336/261579 objects degraded (2.805%); 6273/261579 objects misplaced (2.398%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:21.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:21.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.085 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3543550430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:21.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[220921]: pgmap v830: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 886 op/s; 7336/261579 objects degraded (2.805%); 6273/261579 objects misplaced (2.398%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:24:21.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:21.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:21.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3543550430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:21.087 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:21.087 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:24:21.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:24:21.087 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:24:21.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.34644.0:167072 8.9 8:97461ba3:::benchmark_data_smithi107_251897_object50158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:14:58.106514+0000", 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 562.85821539000005, 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24336654999999999, 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.34644", 2023-10-09T06:24:21.088 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2562237582", 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 167072 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106514+0000", 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106514+0000", 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106513+0000", 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106518+0000", 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7190000000000001e-06 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106530+0000", 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1516e-05 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106534+0000", 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4850000000000003e-06 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106571+0000", 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6591999999999998e-05 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106671+0000", 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100191 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:24:21.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.106750+0000", 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8992999999999994e-05 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.111459+0000", 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047085249999999999 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.111726+0000", 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000267607 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.349810+0000", 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23808400800000001 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.349851+0000", 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0874000000000002e-05 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:14:58.349881+0000", 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.989e-05 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.096 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:1516 10.a 10:5ba9a3f3:::benchmark_data_smithi107_363200_object1515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:26.233646+0000", 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 114.731083528, 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.240807459, 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.097 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 1516 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233646+0000", 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233646+0000", 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233646+0000", 2023-10-09T06:24:21.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233676+0000", 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0351000000000001e-05 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233684+0000", 2023-10-09T06:24:21.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233688+0000", 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4490000000000001e-06 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233733+0000", 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5451999999999998e-05 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233861+0000", 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127613 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233981+0000", 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012026 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234619+0000", 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00063802100000000001 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.236942+0000", 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023228490000000001 2023-10-09T06:24:21.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474349+0000", 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23740683600000001 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474408+0000", 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8822e-05 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474454+0000", 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5816999999999999e-05 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:1517 10.7 10:ef6582d4:::benchmark_data_smithi107_363200_object1516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:26.233840+0000", 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 114.730889792, 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24098941900000001, 2023-10-09T06:24:21.109 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 1517 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233840+0000", 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233840+0000", 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233839+0000", 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233897+0000", 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8202999999999998e-05 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233908+0000", 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0550999999999999e-05 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.233913+0000", 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.938e-06 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234394+0000", 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00048111999999999997 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234501+0000", 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106653 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234676+0000", 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017539300000000001 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.235370+0000", 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00069410399999999997 2023-10-09T06:24:21.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.236546+0000", 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011758509999999999 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474762+0000", 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.238215866 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474805+0000", 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2765000000000001e-05 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474829+0000", 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4647999999999998e-05 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:1519 10.f 10:f3c31667:::benchmark_data_smithi107_363200_object1518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:26.234068+0000", 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 114.730661579, 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.240769278, 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.117 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 1519 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234068+0000", 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234068+0000", 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234067+0000", 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234151+0000", 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3494999999999999e-05 2023-10-09T06:24:21.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234173+0000", 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1888000000000001e-05 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234179+0000", 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7239999999999997e-06 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234294+0000", 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115065 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234403+0000", 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010905599999999999 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:21.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234533+0000", 2023-10-09T06:24:21.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013039700000000001 2023-10-09T06:24:21.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.125 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.235366+0000", 2023-10-09T06:24:21.125 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00083277700000000004 2023-10-09T06:24:21.125 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.125 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.236516+0000", 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011505409999999999 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474745+0000", 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.238228315 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474800+0000", 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5778e-05 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.474837+0000", 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6986000000000001e-05 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:1522 10.f 10:fe41b956:::benchmark_data_smithi107_363200_object1521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:26.234184+0000", 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 114.7305454, 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.241376435, 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 1522 2023-10-09T06:24:21.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234184+0000", 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234184+0000", 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234184+0000", 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234759+0000", 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00057528500000000003 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234771+0000", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1763000000000001e-05 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234785+0000", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4572999999999999e-05 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234867+0000", 2023-10-09T06:24:21.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1457000000000003e-05 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.234971+0000", 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104639 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.235094+0000", 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000122609 2023-10-09T06:24:21.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.235655+0000", 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00056076900000000002 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.236829+0000", 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011745390000000001 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.475479+0000", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.238649583 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.475524+0000", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5138e-05 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.475561+0000", 2023-10-09T06:24:21.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.667e-05 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:1535 10.a 10:520335f2:::benchmark_data_smithi107_363200_object1534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:26.237516+0000", 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 114.727214144, 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.242359036, 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.135 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 1535 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237516+0000", 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237516+0000", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237515+0000", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237575+0000", 2023-10-09T06:24:21.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0232999999999997e-05 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237584+0000", 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3160000000000003e-06 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237590+0000", 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6230000000000002e-06 2023-10-09T06:24:21.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237633+0000", 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3337000000000002e-05 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237744+0000", 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110956 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.237895+0000", 2023-10-09T06:24:21.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015127900000000001 2023-10-09T06:24:21.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.238355+0000", 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045983100000000001 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.475472+0000", 2023-10-09T06:24:21.230 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23711684 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.479819+0000", 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043471330000000004 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.479854+0000", 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4366000000000003e-05 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:26.479875+0000", 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0947000000000001e-05 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:40157 10.a 10:58c779dc:::benchmark_data_smithi107_363200_object40156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:46.173612+0000", 2023-10-09T06:24:21.232 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 94.791117478000004, 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25077989099999998, 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 40157 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.233 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173612+0000", 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173612+0000", 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173611+0000", 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173672+0000", 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0816999999999997e-05 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173686+0000", 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3662000000000001e-05 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173692+0000", 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8220000000000003e-06 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173730+0000", 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8510000000000002e-05 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173826+0000", 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6185000000000002e-05 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.173919+0000", 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2670000000000006e-05 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.175388+0000", 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0014686009999999999 2023-10-09T06:24:21.237 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176479+0000", 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010912719999999999 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424295+0000", 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24781579500000001 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.238 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424351+0000", 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6168000000000002e-05 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424392+0000", 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1362000000000001e-05 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.239 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:40162 10.b 10:dbb2e4f6:::benchmark_data_smithi107_363200_object40161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:46.176217+0000", 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 94.788512585999996, 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.248181018, 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 40162 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.240 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176217+0000", 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176217+0000", 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176217+0000", 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176255+0000", 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8495999999999999e-05 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176263+0000", 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8949999999999998e-06 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176268+0000", 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4720000000000003e-06 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176310+0000", 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1229000000000001e-05 2023-10-09T06:24:21.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176424+0000", 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011468000000000001 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176560+0000", 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013595799999999999 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.244 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.178993+0000", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024331019999999999 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180882+0000", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001889059 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424321+0000", 2023-10-09T06:24:21.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24343853400000001 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424362+0000", 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1318e-05 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424398+0000", 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5849000000000002e-05 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.246 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:40166 10.a 10:5a64a297:::benchmark_data_smithi107_363200_object40165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:46.176280+0000", 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 94.788449295999996, 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24833248599999999, 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.247 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 40166 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176280+0000", 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176280+0000", 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176280+0000", 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176374+0000", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4110999999999994e-05 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176382+0000", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3610000000000001e-06 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176387+0000", 2023-10-09T06:24:21.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176499+0000", 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000112138 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176612+0000", 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113112 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.176751+0000", 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000138446 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.179013+0000", 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022623819999999998 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180188+0000", 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011748640000000001 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424457+0000", 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24426930099999999 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424531+0000", 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3750000000000004e-05 2023-10-09T06:24:21.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.424613+0000", 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2028999999999997e-05 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:40170 10.d 10:b77bbc28:::benchmark_data_smithi107_363200_object40169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:46.180374+0000", 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 94.784355517999998, 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25299053100000002, 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.255 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 40170 2023-10-09T06:24:21.257 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.258 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.259 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.259 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180374+0000", 2023-10-09T06:24:21.259 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180374+0000", 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.260 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180373+0000", 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.261 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180457+0000", 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3863000000000007e-05 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.262 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.263 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180467+0000", 2023-10-09T06:24:21.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6840000000000008e-06 2023-10-09T06:24:21.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.263 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.264 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180476+0000", 2023-10-09T06:24:21.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9789999999999999e-06 2023-10-09T06:24:21.280 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180528+0000", 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2317000000000001e-05 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180674+0000", 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014535700000000001 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180846+0000", 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017272699999999999 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.182818+0000", 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019721270000000002 2023-10-09T06:24:21.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.183026+0000", 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020778499999999999 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.433289+0000", 2023-10-09T06:24:21.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25026240900000002 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.433335+0000", 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6733999999999997e-05 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.433365+0000", 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9349e-05 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.284 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82055 10.d 10:b13ecc6e:::benchmark_data_smithi107_363200_object82054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.803836+0000", 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 55.160894194999997, 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27575927900000002, 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.285 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82055 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803836+0000", 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803836+0000", 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803832+0000", 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999967 2023-10-09T06:24:21.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803901+0000", 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8427000000000001e-05 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803932+0000", 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1843e-05 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803945+0000", 2023-10-09T06:24:21.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2704000000000001e-05 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.808054+0000", 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041086179999999996 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.808217+0000", 2023-10-09T06:24:21.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016365499999999999 2023-10-09T06:24:21.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.808388+0000", 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000170922 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.824640+0000", 2023-10-09T06:24:21.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016251748999999999 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.843927+0000", 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019286422000000001 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079481+0000", 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23555489700000001 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079544+0000", 2023-10-09T06:24:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2806999999999997e-05 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079595+0000", 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0577e-05 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82056 10.a 10:5a18732a:::benchmark_data_smithi107_363200_object82055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2023-10-09T06:24:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.803962+0000", 2023-10-09T06:24:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 55.160767479, 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32546068099999997, 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82056 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803962+0000", 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803962+0000", 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.803962+0000", 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804009+0000", 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7623999999999997e-05 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804017+0000", 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6229999999999997e-06 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804021+0000", 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.048e-06 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804407+0000", 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00038618300000000001 2023-10-09T06:24:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804563+0000", 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015584799999999999 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804730+0000", 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000166627 2023-10-09T06:24:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.827819+0000", 2023-10-09T06:24:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.023089337000000001 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.835978+0000", 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0081585400000000006 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129339+0000", 2023-10-09T06:24:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29336094800000001 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129387+0000", 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8266000000000002e-05 2023-10-09T06:24:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129423+0000", 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6146000000000003e-05 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82061 10.d 10:bbcb4359:::benchmark_data_smithi107_363200_object82060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2023-10-09T06:24:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.804097+0000", 2023-10-09T06:24:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 55.160632876000001, 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27565803700000002, 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82061 2023-10-09T06:24:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804097+0000", 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804097+0000", 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804096+0000", 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804140+0000", 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3535000000000002e-05 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804147+0000", 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5599999999999999e-06 2023-10-09T06:24:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.804151+0000", 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9600000000000002e-06 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.811646+0000", 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0074959069999999996 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.811808+0000", 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016148399999999999 2023-10-09T06:24:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.811972+0000", 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016445000000000001 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.824647+0000", 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012674545000000001 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.843956+0000", 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019309231999999999 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079692+0000", 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.235735527 2023-10-09T06:24:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079729+0000", 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7833000000000002e-05 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.079755+0000", 2023-10-09T06:24:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5398e-05 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82064 10.a 10:5df6a19f:::benchmark_data_smithi107_363200_object82063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.866880+0000", 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 55.097849418999999, 2023-10-09T06:24:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26270463500000002, 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82064 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866880+0000", 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866880+0000", 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866879+0000", 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866926+0000", 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7333000000000002e-05 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866943+0000", 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6595999999999998e-05 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.866953+0000", 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0214000000000001e-05 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.867047+0000", 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4351999999999994e-05 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.867225+0000", 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017815299999999999 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.867385+0000", 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000160116 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.879260+0000", 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011874735000000001 2023-10-09T06:24:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.931850+0000", 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.052589790999999997 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129485+0000", 2023-10-09T06:24:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19763497999999999 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129541+0000", 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6329000000000002e-05 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129585+0000", 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3680000000000002e-05 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:113095 10.d 10:b73d3061:::benchmark_data_smithi107_363200_object9003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.781508+0000", 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.183222148, 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26743418899999999, 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 113095 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781508+0000", 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781508+0000", 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781507+0000", 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781512+0000", 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2000000000000002e-06 2023-10-09T06:24:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781528+0000", 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5926e-05 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781538+0000", 2023-10-09T06:24:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0373e-05 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.785101+0000", 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003563311 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.785229+0000", 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127446 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.785347+0000", 2023-10-09T06:24:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117742 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.791268+0000", 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005921104 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796521+0000", 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0052534010000000004 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048859+0000", 2023-10-09T06:24:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25233733000000003 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048901+0000", 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2561999999999997e-05 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048942+0000", 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0686999999999999e-05 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:113098 10.d 10:bf36af97:::benchmark_data_smithi107_363200_object9006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.781556+0000", 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.183174084999999, 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27162677000000002, 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 113098 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781556+0000", 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781556+0000", 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781555+0000", 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781560+0000", 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6400000000000002e-06 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781577+0000", 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6892999999999999e-05 2023-10-09T06:24:21.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.781587+0000", 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5200000000000003e-06 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.787845+0000", 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0062582569999999997 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.787957+0000", 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011204 2023-10-09T06:24:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.788054+0000", 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7436999999999996e-05 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.791354+0000", 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032996810000000001 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048983+0000", 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25762855499999998 2023-10-09T06:24:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053098+0000", 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041154010000000003 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053143+0000", 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5028000000000003e-05 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053182+0000", 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9314999999999999e-05 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:113106 10.d 10:becde19c:::benchmark_data_smithi107_363200_object9014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.793031+0000", 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.171698920000001, 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26024633899999999, 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 113106 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793031+0000", 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793031+0000", 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793030+0000", 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793035+0000", 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8760000000000002e-06 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793047+0000", 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1094000000000001e-05 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793052+0000", 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8699999999999997e-06 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793090+0000", 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7846999999999999e-05 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793195+0000", 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104381 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.793298+0000", 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010331300000000001 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048953+0000", 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25565526599999999 2023-10-09T06:24:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.049011+0000", 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7337999999999999e-05 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053211+0000", 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004200802 2023-10-09T06:24:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053247+0000", 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6131999999999999e-05 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.053277+0000", 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9671000000000001e-05 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:113111 10.a 10:5768d381:::benchmark_data_smithi107_363200_object9019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.796463+0000", 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.168266745, 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26042044399999997, 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 113111 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796463+0000", 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796463+0000", 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796462+0000", 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796467+0000", 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2700000000000004e-06 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796478+0000", 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0827e-05 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796484+0000", 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8529999999999997e-06 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.796526+0000", 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2471e-05 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044835+0000", 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24830882700000001 2023-10-09T06:24:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044974+0000", 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013853199999999999 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.051196+0000", 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006222048 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.051211+0000", 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5279999999999999e-05 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.056797+0000", 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0055863969999999999 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.056844+0000", 2023-10-09T06:24:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6607000000000003e-05 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.056883+0000", 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9371e-05 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:113114 10.a 10:5d1b3691:::benchmark_data_smithi107_363200_object9022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.797471+0000", 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.167258658, 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25956426900000001, 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 113114 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797471+0000", 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797471+0000", 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797470+0000", 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797478+0000", 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7849999999999992e-06 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044814+0000", 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247336364 2023-10-09T06:24:21.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044828+0000", 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4137e-05 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048027+0000", 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031985350000000002 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048145+0000", 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000118527 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:24:21.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048258+0000", 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000112385 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.051243+0000", 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002985461 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.052873+0000", 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001629744 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.056963+0000", 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0040899559999999996 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.057004+0000", 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1430999999999997e-05 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.057035+0000", 2023-10-09T06:24:21.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0864000000000001e-05 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.45650.0:113095 10.d e271/238 ondisk, result = 0)", 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.797549+0000", 2023-10-09T06:24:21.481 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.167180446, 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25140523799999998, 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:21.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797549+0000", 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797549+0000", 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797549+0000", 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.797554+0000", 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9840000000000001e-06 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044798+0000", 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24724457799999999 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044815+0000", 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7249e-05 2023-10-09T06:24:21.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048846+0000", 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0040305080000000004 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048853+0000", 2023-10-09T06:24:21.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0620000000000003e-06 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.048955+0000", 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101568 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.487 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:21.488 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:21.488 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:21.489 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:21.676 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:24:21.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:24:21.934 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:24:22.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:24:22.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:24:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:22] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-10-09T06:24:22.412 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:22.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: pgmap v831: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 7336/258807 objects degraded (2.835%); 6273/258807 objects misplaced (2.424%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7336/258807 objects degraded (2.835%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: pgmap v831: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 7336/258807 objects degraded (2.835%); 6273/258807 objects misplaced (2.424%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7336/258807 objects degraded (2.835%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:22.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: pgmap v831: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 7336/258807 objects degraded (2.835%); 6273/258807 objects misplaced (2.424%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/931630018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: osdmap e274: 8 total, 8 up, 7 in 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.912 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: failed to encode map e274 with expected crc 2023-10-09T06:24:22.913 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7336/258807 objects degraded (2.835%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:23.665 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:23.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:23.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:23.904 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:24.074 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:24.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:24.179 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:24.234 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:24.430 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:24.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:24.538 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:24.539 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:24.540 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:24:24.540 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:24:24.540 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:24.580 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:24.767 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:24.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:24.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:24.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:24 smithi150 ceph-mon[194178]: pgmap v833: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7049/249732 objects degraded (2.823%); 5650/249732 objects misplaced (2.262%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:24.963 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:24 smithi107 ceph-mon[220921]: pgmap v833: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7049/249732 objects degraded (2.823%); 5650/249732 objects misplaced (2.262%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:24.967 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:24 smithi107 ceph-mon[212750]: pgmap v833: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 883 op/s; 7049/249732 objects degraded (2.823%); 5650/249732 objects misplaced (2.262%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:24.973 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:25.050 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:25.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:25.154 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:25.222 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:25.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:25.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:25.448 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:25.492 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:25.595 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:24:25.611 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:25.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:25.725 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:25.794 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:25.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:25.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:25.820 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:24:25.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:25.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:25.821 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:25.852 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:24:25.872 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:24:25.917 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:24:25.928 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:25.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:26.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:26.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:24:26.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:26.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:26.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:24:26.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:26.069 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:26.128 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:26.140 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:26.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:26.246 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:26.352 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:27.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: pgmap v834: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7049/249000 objects degraded (2.831%); 5650/249000 objects misplaced (2.269%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[220921]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: pgmap v834: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7049/249000 objects degraded (2.831%); 5650/249000 objects misplaced (2.269%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:24:27.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:26 smithi107 ceph-mon[212750]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: pgmap v834: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 7049/249000 objects degraded (2.831%); 5650/249000 objects misplaced (2.269%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/584235461' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: osdmap e275: 8 total, 8 up, 7 in 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:26 smithi150 ceph-mon[194178]: failed to encode map e275 with expected crc 2023-10-09T06:24:27.510 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:27.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:27.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:27.626 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:24:27.626 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:27.706 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:27.842 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:27.842 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:27.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:27.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7049/249000 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:27.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:27.953 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:28.000 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:28.134 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:28.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:28.139 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7049/249000 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:28.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7049/249000 objects degraded (2.831%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:28.233 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:28.234 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:24:28.234 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:24:28.234 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:24:28.234 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:28.239 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:28.290 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:28.516 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:28.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:28.623 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:28.690 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:28 smithi107 ceph-mon[220921]: pgmap v836: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 6749/243192 objects degraded (2.775%); 5034/243192 objects misplaced (2.070%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:29.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:28 smithi107 ceph-mon[212750]: pgmap v836: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 6749/243192 objects degraded (2.775%); 5034/243192 objects misplaced (2.070%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:29.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:28 smithi150 ceph-mon[194178]: pgmap v836: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 6749/243192 objects degraded (2.775%); 5034/243192 objects misplaced (2.070%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:29.210 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:24:29.655 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:29.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:29.768 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:29.794 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:24:29.820 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:29.916 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:29.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:30.020 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:30.047 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:30.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:29 smithi150 ceph-mon[194178]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 6749/236847 objects degraded (2.850%); 5034/236847 objects misplaced (2.125%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:30.084 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:29 smithi107 ceph-mon[220921]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 6749/236847 objects degraded (2.850%); 5034/236847 objects misplaced (2.125%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:30.085 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:29 smithi107 ceph-mon[212750]: pgmap v837: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 914 op/s; 6749/236847 objects degraded (2.850%); 5034/236847 objects misplaced (2.125%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:24:30.155 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:30.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:30.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:30.262 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:30.330 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:30.607 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:30.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:30.712 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:30.798 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:30.850 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:30.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:30.930 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:30.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:30.995 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:31.218 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:31.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:31.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:31.372 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:31.475 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:31.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:31.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:31.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:31.676 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:31.906 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:24:31.909 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:31.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.949 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90926 9.d 9:b3ca72e7:::benchmark_data_smithi107_300016_object90925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 306.80523671499998, 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98166545400000005, 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90926 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.950 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141079+0000", 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141077+0000", 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141118+0000", 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0886999999999997e-05 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141136+0000", 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7960000000000001e-05 2023-10-09T06:24:31.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141144+0000", 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3710000000000002e-06 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141226+0000", 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2038000000000003e-05 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.141235+0000", 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7119999999999995e-06 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091572+0000", 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95033658300000001 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091680+0000", 2023-10-09T06:24:31.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108069 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.091823+0000", 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014296400000000001 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.100838+0000", 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0090148290000000002 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122479+0000", 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.021641525000000002 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122680+0000", 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020095599999999999 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:31.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122712+0000", 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1483999999999999e-05 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.122745+0000", 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2886000000000003e-05 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90932 9.d 9:bfbd009d:::benchmark_data_smithi107_300016_object90931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:24:31.957 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 306.80369491200003, 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98438990199999998, 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90932 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.958 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142621+0000", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142620+0000", 2023-10-09T06:24:31.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142659+0000", 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8958999999999997e-05 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142675+0000", 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6461e-05 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142682+0000", 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4239999999999998e-06 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142740+0000", 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7855000000000001e-05 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.142751+0000", 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1357e-05 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094465+0000", 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95171417800000002 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094549+0000", 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4246999999999995e-05 2023-10-09T06:24:31.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.094636+0000", 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6484999999999996e-05 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104716+0000", 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010080428000000001 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126844+0000", 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022127320999999998 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126903+0000", 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9360000000000001e-05 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.126964+0000", 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0622999999999999e-05 2023-10-09T06:24:31.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127011+0000", 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7321999999999999e-05 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90935 9.d 9:b2389883:::benchmark_data_smithi107_300016_object90934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:24:31.965 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 306.79557082700001, 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97652245800000004, 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90935 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.966 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150745+0000", 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150744+0000", 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:31.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150812+0000", 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8767000000000001e-05 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150826+0000", 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3434e-05 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150831+0000", 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0039999999999999e-06 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150893+0000", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2515999999999996e-05 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.150906+0000", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2364999999999999e-05 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096362+0000", 2023-10-09T06:24:31.970 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.945456729 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096452+0000", 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9395999999999996e-05 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.096552+0000", 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100128 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.104724+0000", 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0081719050000000001 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127064+0000", 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.022340598 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127172+0000", 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000107358 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127223+0000", 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1517000000000002e-05 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127268+0000", 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4280000000000003e-05 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.973 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:90943 9.d 9:b3157ca5:::benchmark_data_smithi107_300016_object90942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 306.78793406800003, 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96914109000000004, 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.974 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 90943 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158382+0000", 2023-10-09T06:24:31.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158381+0000", 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158462+0000", 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1271000000000002e-05 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158475+0000", 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2584e-05 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158480+0000", 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7840000000000003e-06 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158523+0000", 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3368999999999998e-05 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:25.158532+0000", 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6829999999999997e-06 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.098947+0000", 2023-10-09T06:24:31.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94041511099999997 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.099038+0000", 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0902000000000005e-05 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.099136+0000", 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8568999999999998e-05 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.108977+0000", 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.009841163 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127329+0000", 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.018351781000000001 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127431+0000", 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101494 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127483+0000", 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2043999999999998e-05 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:19:26.127523+0000", 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0329e-05 2023-10-09T06:24:31.981 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128124 9.3 9:c89580c2:::benchmark_data_smithi107_300016_object30552:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.791138926, 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0212228619999999, 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:31.982 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128124 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155176+0000", 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155182+0000", 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.421e-06 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155193+0000", 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0852e-05 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155199+0000", 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1369999999999999e-06 2023-10-09T06:24:31.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155278+0000", 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9202000000000005e-05 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155289+0000", 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0137e-05 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176313+0000", 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.021024865 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176400+0000", 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6502000000000006e-05 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128130 9.3 9:c8d10970:::benchmark_data_smithi107_300016_object30558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.7911082, 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0213139, 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128130 2023-10-09T06:24:31.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155207+0000", 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155210+0000", 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3010000000000001e-06 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155216+0000", 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8850000000000004e-06 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155220+0000", 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1489999999999996e-06 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155312+0000", 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2647999999999998e-05 2023-10-09T06:24:31.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155321+0000", 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3299999999999999e-06 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176444+0000", 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0211233209999999 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176522+0000", 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7644000000000004e-05 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128141 9.3 9:ccf706f5:::benchmark_data_smithi107_300016_object30569:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.79109222899999, 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.051517185, 2023-10-09T06:24:31.993 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128141 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155223+0000", 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:31.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155226+0000", 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7269999999999999e-06 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155231+0000", 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2979999999999999e-06 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155235+0000", 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4400000000000001e-06 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155342+0000", 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000107181 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155350+0000", 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0809999999999994e-06 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176542+0000", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.021192391 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176616+0000", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3244000000000005e-05 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176694+0000", 2023-10-09T06:24:31.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8126000000000004e-05 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188370+0000", 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011675899999999999 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206655+0000", 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.018285286000000001 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:31.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206684+0000", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8500000000000002e-05 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206717+0000", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3290999999999997e-05 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206741+0000", 2023-10-09T06:24:32.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4049e-05 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128156 9.3 9:c32ea811:::benchmark_data_smithi107_300016_object30584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.77767731899999, 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.038242423, 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.001 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128156 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168638+0000", 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168643+0000", 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5550000000000002e-06 2023-10-09T06:24:32.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168655+0000", 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2065999999999999e-05 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168662+0000", 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3929999999999996e-06 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168734+0000", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2033000000000006e-05 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168747+0000", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.322e-05 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.179788+0000", 2023-10-09T06:24:32.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.011041641 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.179917+0000", 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012808599999999999 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.180032+0000", 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115358 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.191939+0000", 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011907449000000001 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206789+0000", 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014849571000000001 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206832+0000", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2661999999999999e-05 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206859+0000", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7631999999999999e-05 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206881+0000", 2023-10-09T06:24:32.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1809000000000001e-05 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128192 9.3 9:c85dd9b2:::benchmark_data_smithi107_300016_object30620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.729879853, 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99057770000000001, 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.009 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128192 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216438+0000", 2023-10-09T06:24:32.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0999999999999998e-06 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216445+0000", 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8929999999999999e-06 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216449+0000", 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1679999999999997e-06 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216513+0000", 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3713000000000005e-05 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216523+0000", 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0349e-05 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.182852+0000", 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96632860799999998 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.182975+0000", 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012307299999999999 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:24:32.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.183107+0000", 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013245499999999999 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.191948+0000", 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0088404629999999998 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206813+0000", 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014865295000000001 2023-10-09T06:24:32.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206965+0000", 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015169000000000001 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206990+0000", 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.58e-05 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207014+0000", 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3433000000000001e-05 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128198 9.3 9:c0c2ed7c:::benchmark_data_smithi107_300016_object30626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.71693569300001, 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97778661200000005, 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.017 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128198 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229379+0000", 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.019 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229384+0000", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.172e-06 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229393+0000", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.927e-06 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229400+0000", 2023-10-09T06:24:32.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8449999999999997e-06 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229463+0000", 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2298999999999994e-05 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229475+0000", 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1949e-05 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186026+0000", 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95655169900000003 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186102+0000", 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5545999999999998e-05 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:24:32.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186180+0000", 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7992999999999997e-05 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.197183+0000", 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011003527000000001 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207065+0000", 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0098818469999999992 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207101+0000", 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6117e-05 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207132+0000", 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0901000000000001e-05 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207167+0000", 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4691999999999999e-05 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128203 9.3 9:cb76ab71:::benchmark_data_smithi107_300016_object30631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 251.716874907, 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97972671, 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.025 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128203 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229443+0000", 2023-10-09T06:24:32.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.61e-06 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229451+0000", 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229455+0000", 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0099999999999997e-06 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229506+0000", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1239000000000002e-05 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229515+0000", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9570000000000008e-06 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188401+0000", 2023-10-09T06:24:32.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.958885498 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188474+0000", 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2731999999999994e-05 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188540+0000", 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6625999999999995e-05 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.197191+0000", 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0086507489999999992 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207200+0000", 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010008993000000001 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209114+0000", 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019145500000000001 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209144+0000", 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9615999999999999e-05 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209168+0000", 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3657000000000001e-05 2023-10-09T06:24:32.032 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189411 9.7 9:e8e21091:::benchmark_data_smithi107_300016_object91839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 161.42415176399999, 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99458555100000001, 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.033 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189411 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:24:32.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522162+0000", 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522169+0000", 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8929999999999999e-06 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522184+0000", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4867e-05 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522205+0000", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0511999999999999e-05 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522246+0000", 2023-10-09T06:24:32.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1007999999999998e-05 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522252+0000", 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5060000000000001e-06 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496087+0000", 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97383483500000001 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496175+0000", 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7767000000000002e-05 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496252+0000", 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7286999999999994e-05 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499058+0000", 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00280555 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511729+0000", 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012671003 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516675+0000", 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049465480000000003 2023-10-09T06:24:32.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516706+0000", 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1047999999999998e-05 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516750+0000", 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3436000000000002e-05 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189413 9.d 9:b8ff20a4:::benchmark_data_smithi107_300016_object91841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 161.41949926699999, 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97899737899999995, 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189413 2023-10-09T06:24:32.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526816+0000", 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526821+0000", 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6450000000000002e-06 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526834+0000", 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2975e-05 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526840+0000", 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1900000000000003e-06 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526893+0000", 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2958999999999999e-05 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526903+0000", 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494707+0000", 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96780485699999996 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494861+0000", 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015316799999999999 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494947+0000", 2023-10-09T06:24:32.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6058999999999997e-05 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505292+0000", 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010345277999999999 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505617+0000", 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000325345 2023-10-09T06:24:32.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505718+0000", 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100776 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505783+0000", 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4508e-05 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505814+0000", 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1517999999999999e-05 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189420 9.7 9:ee572d28:::benchmark_data_smithi107_300016_object91848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 161.405006805, 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98343943099999998, 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.049 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189420 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541307+0000", 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541317+0000", 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6630000000000005e-06 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541338+0000", 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.073e-05 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541346+0000", 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4640000000000006e-06 2023-10-09T06:24:32.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541405+0000", 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9061999999999998e-05 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541417+0000", 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1871e-05 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499255+0000", 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95783806999999999 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499327+0000", 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1607999999999996e-05 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499399+0000", 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1766000000000004e-05 2023-10-09T06:24:32.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.506518+0000", 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071195950000000003 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511815+0000", 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0052969769999999996 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524668+0000", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012852763 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524703+0000", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5642999999999997e-05 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524749+0000", 2023-10-09T06:24:32.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.507e-05 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189422 9.7 9:e5521dd9:::benchmark_data_smithi107_300016_object91850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 161.402697585, 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98124100700000005, 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.057 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189422 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543617+0000", 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543623+0000", 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3809999999999998e-06 2023-10-09T06:24:32.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543636+0000", 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2639e-05 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543642+0000", 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8619999999999997e-06 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543692+0000", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9459999999999997e-05 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543699+0000", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8839999999999997e-06 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501750+0000", 2023-10-09T06:24:32.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95805110599999999 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501835+0000", 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5227000000000002e-05 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501898+0000", 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3380000000000006e-05 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.506526+0000", 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046276709999999999 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511862+0000", 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053355900000000003 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524806+0000", 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012944638 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524837+0000", 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0956e-05 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524859+0000", 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2084999999999999e-05 2023-10-09T06:24:32.064 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189423 9.d 9:bcb66210:::benchmark_data_smithi107_300016_object91851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.543778+0000", 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 161.40253797099999, 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96830715000000001, 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:24:32.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189423 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543778+0000", 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543778+0000", 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543777+0000", 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543782+0000", 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7690000000000004e-06 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543793+0000", 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0614999999999999e-05 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543798+0000", 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8699999999999997e-06 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543842+0000", 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3049000000000001e-05 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543848+0000", 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1029999999999998e-06 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.497669+0000", 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95382101500000005 2023-10-09T06:24:32.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.497751+0000", 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2748000000000001e-05 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.497834+0000", 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2707000000000006e-05 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505402+0000", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0075681569999999998 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505667+0000", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000264608 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.512031+0000", 2023-10-09T06:24:32.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006364013 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.512062+0000", 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0670999999999998e-05 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.512085+0000", 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3575e-05 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.072 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:40174 10.c 10:399fa3d5:::benchmark_data_smithi107_363200_object40173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:22:46.180971+0000", 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 105.765344802, 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.248282481, 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.073 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 40174 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180971+0000", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180971+0000", 2023-10-09T06:24:32.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.180970+0000", 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181026+0000", 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5537e-05 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181043+0000", 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7631000000000001e-05 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181049+0000", 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5079999999999996e-06 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181082+0000", 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2808999999999997e-05 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181206+0000", 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012409699999999999 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.181413+0000", 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020724299999999999 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.183203+0000", 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017902020000000001 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.184551+0000", 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001347729 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.429199+0000", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.244647904 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.429232+0000", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3380000000000002e-05 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:22:46.429254+0000", 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1467999999999998e-05 2023-10-09T06:24:32.079 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:83872 10.c 10:3f8ef237:::benchmark_data_smithi107_363200_object83871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:28.368291+0000", 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 63.578024485, 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.236010625, 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:24:32.080 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 83872 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368291+0000", 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368291+0000", 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368290+0000", 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368457+0000", 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016747800000000001 2023-10-09T06:24:32.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368472+0000", 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4963000000000001e-05 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368479+0000", 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7499999999999997e-06 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368551+0000", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2248000000000004e-05 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368669+0000", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011758500000000001 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.368805+0000", 2023-10-09T06:24:32.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000136091 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.372276+0000", 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034705389999999999 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.373284+0000", 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010085599999999999 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.604205+0000", 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.230921346 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.604263+0000", 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7506000000000001e-05 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:28.604302+0000", 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9134999999999997e-05 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45650.0:113127 10.13 e271/264 10:cb0622e1:::benchmark_data_smithi107_363200_object9035:head v 271'6175, mlcod=242'768)", 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.812588+0000", 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 25.133727597, 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23301376600000001, 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.087 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812588+0000", 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812588+0000", 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812587+0000", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812602+0000", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5107e-05 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042038+0000", 2023-10-09T06:24:32.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.229435786 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042056+0000", 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8040999999999999e-05 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042098+0000", 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2054000000000001e-05 2023-10-09T06:24:32.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042130+0000", 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1952999999999997e-05 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.045563+0000", 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034331750000000001 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.045602+0000", 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9304000000000002e-05 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45650.0:113129 10.18 e271/254 10:1aa47e8c:::benchmark_data_smithi107_363200_object9037:head v 271'6062, mlcod=271'6060)", 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:24:06.812805+0000", 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 25.133510694000002, 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.231458938, 2023-10-09T06:24:32.092 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812805+0000", 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812805+0000", 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812804+0000", 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:06.812815+0000", 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1253000000000001e-05 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:24:32.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042041+0000", 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.229226078 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042053+0000", 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2007e-05 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042110+0000", 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6456999999999998e-05 2023-10-09T06:24:32.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.042131+0000", 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1129999999999999e-05 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044224+0000", 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002093494 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:24:32.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:24:07.044264+0000", 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9869000000000001e-05 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:24:32.097 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:24:32.097 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:32.099 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:24:32.099 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:24:32.099 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:24:32.099 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:24:32.127 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:24:32.145 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:32.169 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:24:32.215 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:24:32.219 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:32.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:32.237 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.237 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:24:32.241 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:32] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T06:24:32.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:32.472 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:32.526 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:32.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:32.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:32.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 6749/234102 objects degraded (2.883%); 5034/234102 objects misplaced (2.150%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:24:32.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:32.581 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:32.581 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4165428146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:32.581 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6749/234102 objects degraded (2.883%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:32.581 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1151652793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.581 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.629 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:32.721 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 6749/234102 objects degraded (2.883%); 5034/234102 objects misplaced (2.150%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4165428146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6749/234102 objects degraded (2.883%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1151652793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: pgmap v838: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 788 op/s; 6749/234102 objects degraded (2.883%); 5034/234102 objects misplaced (2.150%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T06:24:32.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:24:32.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:32.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4165428146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:32.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6749/234102 objects degraded (2.883%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:32.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1151652793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:24:32.940 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:24:33.159 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:24:33.516 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:33.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:33.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:33.670 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:33.739 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:33.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:33.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:33.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:33.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:33.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:33.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[220921]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T06:24:33.899 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[220921]: failed to encode map e276 with expected crc 2023-10-09T06:24:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[212750]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T06:24:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:33 smithi107 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T06:24:33.902 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:33.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:33 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:24:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:33 smithi150 ceph-mon[194178]: osdmap e276: 8 total, 8 up, 7 in 2023-10-09T06:24:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:33 smithi150 ceph-mon[194178]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.611 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[212750]: pgmap v839: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 6485/225303 objects degraded (2.878%); 4336/225303 objects misplaced (1.925%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:34.611 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.612 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[212750]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[220921]: pgmap v839: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 6485/225303 objects degraded (2.878%); 4336/225303 objects misplaced (1.925%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[220921]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:34 smithi107 ceph-mon[220921]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:34 smithi150 ceph-mon[194178]: pgmap v839: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 6485/225303 objects degraded (2.878%); 4336/225303 objects misplaced (1.925%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:24:34.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:34 smithi150 ceph-mon[194178]: failed to encode map e276 with expected crc 2023-10-09T06:24:34.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:34 smithi150 ceph-mon[194178]: failed to encode map e276 with expected crc 2023-10-09T06:24:35.049 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:35.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:35.139 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:35.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:35.204 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:35.271 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:35.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:35.378 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:35.426 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:35.508 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:35.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:35.613 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:35.699 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:35.846 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:35.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:35.952 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:36.041 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:36.150 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:36.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:36.257 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:36.333 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:36.419 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:36.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:36.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:36.522 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:36.549 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:36.590 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:36.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:36.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:36 smithi107 ceph-mon[212750]: pgmap v841: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 6485/224475 objects degraded (2.889%); 4336/224475 objects misplaced (1.932%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:24:36.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:24:36.621 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:36 smithi107 ceph-mon[220921]: pgmap v841: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 6485/224475 objects degraded (2.889%); 4336/224475 objects misplaced (1.932%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:24:36.621 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:24:36.638 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:24:36.697 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:36.764 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:36 smithi150 ceph-mon[194178]: pgmap v841: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 789 op/s; 6485/224475 objects degraded (2.889%); 4336/224475 objects misplaced (1.932%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:24:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:24:36.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:24:37.054 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:24:37.213 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:37.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:37.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:37.403 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:37.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[220921]: failed to encode map e277 with expected crc 2023-10-09T06:24:37.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:24:37.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[220921]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T06:24:37.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6485/224475 objects degraded (2.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[212750]: failed to encode map e277 with expected crc 2023-10-09T06:24:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:24:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[212750]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T06:24:37.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6485/224475 objects degraded (2.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:37.841 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:37.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:37 smithi150 ceph-mon[194178]: failed to encode map e277 with expected crc 2023-10-09T06:24:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2734268236' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:24:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:37 smithi150 ceph-mon[194178]: osdmap e277: 8 total, 8 up, 7 in 2023-10-09T06:24:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6485/224475 objects degraded (2.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:24:37.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:38.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:38.060 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:24:38.060 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:24:38.301 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:38.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:38.418 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:38.506 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:38.586 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:38.621 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:38.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:38.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:38.694 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:38.715 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:24:38.715 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:24:38.729 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:38.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:38 smithi107 ceph-mon[212750]: pgmap v843: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 6233/218727 objects degraded (2.850%); 3697/218727 objects misplaced (1.690%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:38.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:38 smithi107 ceph-mon[220921]: pgmap v843: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 6233/218727 objects degraded (2.850%); 3697/218727 objects misplaced (1.690%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:38.843 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:38 smithi150 ceph-mon[194178]: pgmap v843: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 6233/218727 objects degraded (2.850%); 3697/218727 objects misplaced (1.690%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:39.021 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:39.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:39.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:39.129 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:39.178 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:39.273 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:39.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:39.382 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:39.459 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:39.947 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:40.440 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:40.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:40.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:40.548 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:40.637 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:24:40.687 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:40.710 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:40 smithi107 ceph-mon[220921]: pgmap v844: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 6233/212043 objects degraded (2.939%); 3697/212043 objects misplaced (1.744%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:40.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:40 smithi107 ceph-mon[212750]: pgmap v844: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 6233/212043 objects degraded (2.939%); 3697/212043 objects misplaced (1.744%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:40 smithi150 ceph-mon[194178]: pgmap v844: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 6233/212043 objects degraded (2.939%); 3697/212043 objects misplaced (1.744%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:24:41.152 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:41.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:41.260 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.302 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:41.385 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:41.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:41.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:41.541 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:41.625 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:41.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:41.733 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:41.780 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:41.860 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:41.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:41.975 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:42.066 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:42.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:42] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2023-10-09T06:24:42.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:42 smithi107 ceph-mon[212750]: pgmap v845: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 6233/209130 objects degraded (2.980%); 3697/209130 objects misplaced (1.768%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:24:42.859 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6233/209130 objects degraded (2.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:42.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:42 smithi107 ceph-mon[220921]: pgmap v845: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 6233/209130 objects degraded (2.980%); 3697/209130 objects misplaced (1.768%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:24:42.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6233/209130 objects degraded (2.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:42.926 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:42.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:43.046 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:43.124 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:42 smithi150 ceph-mon[194178]: pgmap v845: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 6233/209130 objects degraded (2.980%); 3697/209130 objects misplaced (1.768%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T06:24:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6233/209130 objects degraded (2.980%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:43.742 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:43.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:43.848 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:43.898 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:43.968 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:43.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:44.077 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:44.113 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:44.176 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:44.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:44.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:44.221 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:44.284 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:44.321 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:44.394 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:44.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:44.500 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:44.583 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:44.605 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:24:44.673 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:24:44.723 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:24:44.772 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:24:44.818 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:24:44.869 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:24:44.894 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:24:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:44 smithi107 ceph-mon[212750]: pgmap v846: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 5973/199365 objects degraded (2.996%); 2964/199365 objects misplaced (1.487%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1737458314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:44 smithi107 ceph-mon[220921]: pgmap v846: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 5973/199365 objects degraded (2.996%); 2964/199365 objects misplaced (1.487%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:44.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1737458314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:44.917 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.920 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:24:44.920 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:24:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:44 smithi150 ceph-mon[194178]: pgmap v846: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 5973/199365 objects degraded (2.996%); 2964/199365 objects misplaced (1.487%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:45.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1737458314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:45.802 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:24:45.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2580541944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:45.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:45 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:45.992 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2580541944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:45.992 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:45 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2580541944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:46.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:45 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:24:46.346 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:46.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:46.458 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:46.534 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:46.847 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:24:46.878 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:24:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: pgmap v847: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5973/198585 objects degraded (3.008%); 2964/198585 objects misplaced (1.493%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:47.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: pgmap v847: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5973/198585 objects degraded (3.008%); 2964/198585 objects misplaced (1.493%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: pgmap v847: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 5973/198585 objects degraded (3.008%); 2964/198585 objects misplaced (1.493%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: from='client.45821 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: osdmap e278: 8 total, 8 up, 7 in 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: failed to encode map e278 with expected crc 2023-10-09T06:24:47.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:24:47.854 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:47.854 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5973/198585 objects degraded (3.008%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:47.855 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5973/198585 objects degraded (3.008%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:47.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:47.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:48.045 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5973/198585 objects degraded (3.008%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:48.387 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:48.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:48.400 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":847,"stamp":"2023-10-09T06:24:45.091338+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4336981944,"num_objects":66195,"num_object_clones":0,"num_object_copies":198585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5973,"num_objects_misplaced":2964,"num_objects_unfound":0,"num_objects_dirty":66195,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123436,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6073,"num_bytes_recovered":396503130,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180202,"ondisk_log_size":180202,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17943328,"kb_used_data":13445160,"kb_used_omap":20,"kb_used_meta":4497963,"kb_avail":638272736,"statfs":{"total":671965249536,"available":653591281664,"internally_reserved":0,"allocated":13767843840,"data_stored":13760547722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4605914766},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-565575680,"num_objects":-8630,"num_object_clones":0,"num_object_copies":-25890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-512,"num_objects_misplaced":-1372,"num_objects_unfound":0,"num_objects_dirty":-8630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8630,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":814,"num_bytes_recovered":53346304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003376"},"pg_stats":[{"pgid":"10.13","version":"277'7013","reported_seq":13436,"reported_epoch":277,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:24:41.135333+0000","last_change":"2023-10-09T06:23:56.684310+0000","last_active":"2023-10-09T06:24:41.135333+0000","last_peered":"2023-10-09T06:24:41.135333+0000","last_clean":"2023-10-09T06:23:38.908387+0000","last_became_active":"2023-10-09T06:23:42.387234+0000","last_became_peered":"2023-10-09T06:23:42.387234+0000","last_unstale":"2023-10-09T06:24:41.135333+0000","last_undegraded":"2023-10-09T06:24:41.135333+0000","last_fullsized":"2023-10-09T06:23:41.928935+0000","mapping_epoch":264,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822208,"num_objects":2103,"num_object_clones":0,"num_object_copies":6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1380,"num_objects_unfound":0,"num_objects_dirty":2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4433,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2120,"num_bytes_recovered":138936320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[2,1],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"1,2","objects":1413},{"shards":"2,3,4","objects":690}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":256,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156322+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:24:33.156322+0000","last_peered":"2023-10-09T06:24:33.156322+0000","last_clean":"2023-10-09T06:24:33.156322+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:24:33.156322+0000","last_undegraded":"2023-10-09T06:24:33.156322+0000","last_fullsized":"2023-10-09T06:24:33.156322+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210729+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:24:33.210729+0000","last_peered":"2023-10-09T06:24:33.210729+0000","last_clean":"2023-10-09T06:24:33.210729+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:24:33.210729+0000","last_undegraded":"2023-10-09T06:24:33.210729+0000","last_fullsized":"2023-10-09T06:24:33.210729+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"277'7005","reported_seq":12550,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.974607+0000","last_change":"2023-10-09T06:24:19.321164+0000","last_active":"2023-10-09T06:24:42.974607+0000","last_peered":"2023-10-09T06:24:42.974607+0000","last_clean":"2023-10-09T06:24:42.974607+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:24:42.974607+0000","last_undegraded":"2023-10-09T06:24:42.974607+0000","last_fullsized":"2023-10-09T06:24:42.974607+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133890048,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4447,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":248,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178163+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:24:33.178163+0000","last_peered":"2023-10-09T06:24:33.178163+0000","last_clean":"2023-10-09T06:24:33.178163+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:24:33.178163+0000","last_undegraded":"2023-10-09T06:24:33.178163+0000","last_fullsized":"2023-10-09T06:24:33.178163+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":248,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173494+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:24:33.173494+0000","last_peered":"2023-10-09T06:24:33.173494+0000","last_clean":"2023-10-09T06:24:33.173494+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:24:33.173494+0000","last_undegraded":"2023-10-09T06:24:33.173494+0000","last_fullsized":"2023-10-09T06:24:33.173494+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"277'6988","reported_seq":7049,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:24:40.182168+0000","last_change":"2023-10-09T06:23:38.088533+0000","last_active":"2023-10-09T06:24:40.182168+0000","last_peered":"2023-10-09T06:24:40.182168+0000","last_clean":"2023-10-09T06:23:34.462031+0000","last_became_active":"2023-10-09T06:23:38.083231+0000","last_became_peered":"2023-10-09T06:23:38.083231+0000","last_unstale":"2023-10-09T06:24:40.182168+0000","last_undegraded":"2023-10-09T06:23:38.014323+0000","last_fullsized":"2023-10-09T06:23:37.903678+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142475264,"num_objects":2174,"num_object_clones":0,"num_object_copies":6522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2805,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4370,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":1707},{"shards":"0,7","objects":-631}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":232,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210508+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:24:33.210508+0000","last_peered":"2023-10-09T06:24:33.210508+0000","last_clean":"2023-10-09T06:24:33.210508+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:24:33.210508+0000","last_undegraded":"2023-10-09T06:24:33.210508+0000","last_fullsized":"2023-10-09T06:24:33.210508+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":256,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179449+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:24:33.179449+0000","last_peered":"2023-10-09T06:24:33.179449+0000","last_clean":"2023-10-09T06:24:33.179449+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:24:33.179449+0000","last_undegraded":"2023-10-09T06:24:33.179449+0000","last_fullsized":"2023-10-09T06:24:33.179449+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173373+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:24:33.173373+0000","last_peered":"2023-10-09T06:24:33.173373+0000","last_clean":"2023-10-09T06:24:33.173373+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:24:33.173373+0000","last_undegraded":"2023-10-09T06:24:33.173373+0000","last_fullsized":"2023-10-09T06:24:33.173373+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":214,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210429+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:24:33.210429+0000","last_peered":"2023-10-09T06:24:33.210429+0000","last_clean":"2023-10-09T06:24:33.210429+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:24:33.210429+0000","last_undegraded":"2023-10-09T06:24:33.210429+0000","last_fullsized":"2023-10-09T06:24:33.210429+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185771+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:24:33.185771+0000","last_peered":"2023-10-09T06:24:33.185771+0000","last_clean":"2023-10-09T06:24:33.185771+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:24:33.185771+0000","last_undegraded":"2023-10-09T06:24:33.185771+0000","last_fullsized":"2023-10-09T06:24:33.185771+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":212,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210724+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:24:33.210724+0000","last_peered":"2023-10-09T06:24:33.210724+0000","last_clean":"2023-10-09T06:24:33.210724+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:24:33.210724+0000","last_undegraded":"2023-10-09T06:24:33.210724+0000","last_fullsized":"2023-10-09T06:24:33.210724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":126,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185565+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:24:33.185565+0000","last_peered":"2023-10-09T06:24:33.185565+0000","last_clean":"2023-10-09T06:24:33.185565+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:24:33.185565+0000","last_undegraded":"2023-10-09T06:24:33.185565+0000","last_fullsized":"2023-10-09T06:24:33.185565+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.625210+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:24:36.625210+0000","last_peered":"2023-10-09T06:24:36.625210+0000","last_clean":"2023-10-09T06:24:36.625210+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:24:36.625210+0000","last_undegraded":"2023-10-09T06:24:36.625210+0000","last_fullsized":"2023-10-09T06:24:36.625210+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":283,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156803+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:24:33.156803+0000","last_peered":"2023-10-09T06:24:33.156803+0000","last_clean":"2023-10-09T06:24:33.156803+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:24:33.156803+0000","last_undegraded":"2023-10-09T06:24:33.156803+0000","last_fullsized":"2023-10-09T06:24:33.156803+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":247,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156692+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:24:33.156692+0000","last_peered":"2023-10-09T06:24:33.156692+0000","last_clean":"2023-10-09T06:24:33.156692+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:24:33.156692+0000","last_undegraded":"2023-10-09T06:24:33.156692+0000","last_fullsized":"2023-10-09T06:24:33.156692+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":126,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185341+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:24:33.185341+0000","last_peered":"2023-10-09T06:24:33.185341+0000","last_clean":"2023-10-09T06:24:33.185341+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:24:33.185341+0000","last_undegraded":"2023-10-09T06:24:33.185341+0000","last_fullsized":"2023-10-09T06:24:33.185341+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":232,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.211434+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:24:33.211434+0000","last_peered":"2023-10-09T06:24:33.211434+0000","last_clean":"2023-10-09T06:24:33.211434+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:24:33.211434+0000","last_undegraded":"2023-10-09T06:24:33.211434+0000","last_fullsized":"2023-10-09T06:24:33.211434+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":247,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:35.246527+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:24:35.246527+0000","last_peered":"2023-10-09T06:24:35.246527+0000","last_clean":"2023-10-09T06:24:35.246527+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:24:35.246527+0000","last_undegraded":"2023-10-09T06:24:35.246527+0000","last_fullsized":"2023-10-09T06:24:35.246527+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185420+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:24:33.185420+0000","last_peered":"2023-10-09T06:24:33.185420+0000","last_clean":"2023-10-09T06:24:33.185420+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:24:33.185420+0000","last_undegraded":"2023-10-09T06:24:33.185420+0000","last_fullsized":"2023-10-09T06:24:33.185420+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"277'6893","reported_seq":6950,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.085470+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:24:40.085470+0000","last_peered":"2023-10-09T06:24:40.085470+0000","last_clean":"2023-10-09T06:24:40.085470+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:24:40.085470+0000","last_undegraded":"2023-10-09T06:24:40.085470+0000","last_fullsized":"2023-10-09T06:24:40.085470+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137920512,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4341,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":120,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.186137+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:24:33.186137+0000","last_peered":"2023-10-09T06:24:33.186137+0000","last_clean":"2023-10-09T06:24:33.186137+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:24:33.186137+0000","last_undegraded":"2023-10-09T06:24:33.186137+0000","last_fullsized":"2023-10-09T06:24:33.186137+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":246,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173589+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:24:33.173589+0000","last_peered":"2023-10-09T06:24:33.173589+0000","last_clean":"2023-10-09T06:24:33.173589+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:24:33.173589+0000","last_undegraded":"2023-10-09T06:24:33.173589+0000","last_fullsized":"2023-10-09T06:24:33.173589+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"277'6966","reported_seq":7024,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.044979+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:24:41.044979+0000","last_peered":"2023-10-09T06:24:41.044979+0000","last_clean":"2023-10-09T06:24:41.044979+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:24:41.044979+0000","last_undegraded":"2023-10-09T06:24:41.044979+0000","last_fullsized":"2023-10-09T06:24:41.044979+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6966,"log_dups_size":0,"ondisk_log_size":6966,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138051584,"num_objects":2107,"num_object_clones":0,"num_object_copies":6321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4401,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":265,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178962+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:24:33.178962+0000","last_peered":"2023-10-09T06:24:33.178962+0000","last_clean":"2023-10-09T06:24:33.178962+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:24:33.178962+0000","last_undegraded":"2023-10-09T06:24:33.178962+0000","last_fullsized":"2023-10-09T06:24:33.178962+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.213079+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:24:33.213079+0000","last_peered":"2023-10-09T06:24:33.213079+0000","last_clean":"2023-10-09T06:24:33.213079+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:24:33.213079+0000","last_undegraded":"2023-10-09T06:24:33.213079+0000","last_fullsized":"2023-10-09T06:24:33.213079+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"277'6962","reported_seq":7089,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.195213+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:24:40.195213+0000","last_peered":"2023-10-09T06:24:40.195213+0000","last_clean":"2023-10-09T06:24:40.195213+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:24:40.195213+0000","last_undegraded":"2023-10-09T06:24:40.195213+0000","last_fullsized":"2023-10-09T06:24:40.195213+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":6962,"log_dups_size":0,"ondisk_log_size":6962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140574720,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1700,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":138,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185806+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:24:33.185806+0000","last_peered":"2023-10-09T06:24:33.185806+0000","last_clean":"2023-10-09T06:24:33.185806+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:24:33.185806+0000","last_undegraded":"2023-10-09T06:24:33.185806+0000","last_fullsized":"2023-10-09T06:24:33.185806+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":187,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.186690+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:24:33.186690+0000","last_peered":"2023-10-09T06:24:33.186690+0000","last_clean":"2023-10-09T06:24:33.186690+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:24:33.186690+0000","last_undegraded":"2023-10-09T06:24:33.186690+0000","last_fullsized":"2023-10-09T06:24:33.186690+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"277'8821","reported_seq":8995,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.847979+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:24:42.847979+0000","last_peered":"2023-10-09T06:24:42.847979+0000","last_clean":"2023-10-09T06:24:42.847979+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:24:42.847979+0000","last_undegraded":"2023-10-09T06:24:42.847979+0000","last_fullsized":"2023-10-09T06:24:42.847979+0000","mapping_epoch":238,"log_start":"242'500","ondisk_log_start":"242'500","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":8321,"log_dups_size":0,"ondisk_log_size":8321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129564672,"num_objects":1977,"num_object_clones":0,"num_object_copies":5931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1719,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":124,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185696+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:24:33.185696+0000","last_peered":"2023-10-09T06:24:33.185696+0000","last_clean":"2023-10-09T06:24:33.185696+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:24:33.185696+0000","last_undegraded":"2023-10-09T06:24:33.185696+0000","last_fullsized":"2023-10-09T06:24:33.185696+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":254,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156511+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:24:33.156511+0000","last_peered":"2023-10-09T06:24:33.156511+0000","last_clean":"2023-10-09T06:24:33.156511+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:24:33.156511+0000","last_undegraded":"2023-10-09T06:24:33.156511+0000","last_fullsized":"2023-10-09T06:24:33.156511+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"277'7085","reported_seq":7144,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.171126+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:24:40.171126+0000","last_peered":"2023-10-09T06:24:40.171126+0000","last_clean":"2023-10-09T06:24:40.171126+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:24:40.171126+0000","last_undegraded":"2023-10-09T06:24:40.171126+0000","last_fullsized":"2023-10-09T06:24:40.171126+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7085,"log_dups_size":0,"ondisk_log_size":7085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135430144,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4482,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":138,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185341+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:24:33.185341+0000","last_peered":"2023-10-09T06:24:33.185341+0000","last_clean":"2023-10-09T06:24:33.185341+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:24:33.185341+0000","last_undegraded":"2023-10-09T06:24:33.185341+0000","last_fullsized":"2023-10-09T06:24:33.185341+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":256,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179168+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:24:33.179168+0000","last_peered":"2023-10-09T06:24:33.179168+0000","last_clean":"2023-10-09T06:24:33.179168+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:24:33.179168+0000","last_undegraded":"2023-10-09T06:24:33.179168+0000","last_fullsized":"2023-10-09T06:24:33.179168+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"277'6961","reported_seq":7020,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.125357+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:24:41.125357+0000","last_peered":"2023-10-09T06:24:41.125357+0000","last_clean":"2023-10-09T06:24:41.125357+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:24:41.125357+0000","last_undegraded":"2023-10-09T06:24:41.125357+0000","last_fullsized":"2023-10-09T06:24:41.125357+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6961,"log_dups_size":0,"ondisk_log_size":6961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4419,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.211301+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:24:33.211301+0000","last_peered":"2023-10-09T06:24:33.211301+0000","last_clean":"2023-10-09T06:24:33.211301+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:24:33.211301+0000","last_undegraded":"2023-10-09T06:24:33.211301+0000","last_fullsized":"2023-10-09T06:24:33.211301+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":229,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173005+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:24:33.173005+0000","last_peered":"2023-10-09T06:24:33.173005+0000","last_clean":"2023-10-09T06:24:33.173005+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:24:33.173005+0000","last_undegraded":"2023-10-09T06:24:33.173005+0000","last_fullsized":"2023-10-09T06:24:33.173005+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"277'6973","reported_seq":7102,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.795082+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:24:41.795082+0000","last_peered":"2023-10-09T06:24:41.795082+0000","last_clean":"2023-10-09T06:24:41.795082+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:24:41.795082+0000","last_undegraded":"2023-10-09T06:24:41.795082+0000","last_fullsized":"2023-10-09T06:24:41.795082+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":6973,"log_dups_size":0,"ondisk_log_size":6973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137822208,"num_objects":2103,"num_object_clones":0,"num_object_copies":6309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1679,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":569,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185944+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:24:33.185944+0000","last_peered":"2023-10-09T06:24:33.185944+0000","last_clean":"2023-10-09T06:24:33.185944+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:24:33.185944+0000","last_undegraded":"2023-10-09T06:24:33.185944+0000","last_fullsized":"2023-10-09T06:24:33.185944+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":240,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210752+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:24:33.210752+0000","last_peered":"2023-10-09T06:24:33.210752+0000","last_clean":"2023-10-09T06:24:33.210752+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:24:33.210752+0000","last_undegraded":"2023-10-09T06:24:33.210752+0000","last_fullsized":"2023-10-09T06:24:33.210752+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156828+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:24:33.156828+0000","last_peered":"2023-10-09T06:24:33.156828+0000","last_clean":"2023-10-09T06:24:33.156828+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:24:33.156828+0000","last_undegraded":"2023-10-09T06:24:33.156828+0000","last_fullsized":"2023-10-09T06:24:33.156828+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"277'7077","reported_seq":7136,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.121661+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:24:41.121661+0000","last_peered":"2023-10-09T06:24:41.121661+0000","last_clean":"2023-10-09T06:24:41.121661+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:24:41.121661+0000","last_undegraded":"2023-10-09T06:24:41.121661+0000","last_fullsized":"2023-10-09T06:24:41.121661+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7077,"log_dups_size":0,"ondisk_log_size":7077,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141885440,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4497,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":560,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.161407+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:24:33.161407+0000","last_peered":"2023-10-09T06:24:33.161407+0000","last_clean":"2023-10-09T06:24:33.161407+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:24:33.161407+0000","last_undegraded":"2023-10-09T06:24:33.161407+0000","last_fullsized":"2023-10-09T06:24:33.161407+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179017+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:24:33.179017+0000","last_peered":"2023-10-09T06:24:33.179017+0000","last_clean":"2023-10-09T06:24:33.179017+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:24:33.179017+0000","last_undegraded":"2023-10-09T06:24:33.179017+0000","last_fullsized":"2023-10-09T06:24:33.179017+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156673+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:24:33.156673+0000","last_peered":"2023-10-09T06:24:33.156673+0000","last_clean":"2023-10-09T06:24:33.156673+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:24:33.156673+0000","last_undegraded":"2023-10-09T06:24:33.156673+0000","last_fullsized":"2023-10-09T06:24:33.156673+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"277'6975","reported_seq":7147,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.875143+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:24:42.875143+0000","last_peered":"2023-10-09T06:24:42.875143+0000","last_clean":"2023-10-09T06:24:42.875143+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:24:42.875143+0000","last_undegraded":"2023-10-09T06:24:42.875143+0000","last_fullsized":"2023-10-09T06:24:42.875143+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":6975,"log_dups_size":0,"ondisk_log_size":6975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136708096,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1840,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":258,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156355+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:24:33.156355+0000","last_peered":"2023-10-09T06:24:33.156355+0000","last_clean":"2023-10-09T06:24:33.156355+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:24:33.156355+0000","last_undegraded":"2023-10-09T06:24:33.156355+0000","last_fullsized":"2023-10-09T06:24:33.156355+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156878+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:24:33.156878+0000","last_peered":"2023-10-09T06:24:33.156878+0000","last_clean":"2023-10-09T06:24:33.156878+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:24:33.156878+0000","last_undegraded":"2023-10-09T06:24:33.156878+0000","last_fullsized":"2023-10-09T06:24:33.156878+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"277'7170","reported_seq":7349,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:43.004707+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:24:43.004707+0000","last_peered":"2023-10-09T06:24:43.004707+0000","last_clean":"2023-10-09T06:24:43.004707+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:24:43.004707+0000","last_undegraded":"2023-10-09T06:24:43.004707+0000","last_fullsized":"2023-10-09T06:24:43.004707+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":7170,"log_dups_size":0,"ondisk_log_size":7170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136642560,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1745,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":556,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.161785+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:24:33.161785+0000","last_peered":"2023-10-09T06:24:33.161785+0000","last_clean":"2023-10-09T06:24:33.161785+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:24:33.161785+0000","last_undegraded":"2023-10-09T06:24:33.161785+0000","last_fullsized":"2023-10-09T06:24:33.161785+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":233,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210361+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:24:33.210361+0000","last_peered":"2023-10-09T06:24:33.210361+0000","last_clean":"2023-10-09T06:24:33.210361+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:24:33.210361+0000","last_undegraded":"2023-10-09T06:24:33.210361+0000","last_fullsized":"2023-10-09T06:24:33.210361+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178287+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:24:33.178287+0000","last_peered":"2023-10-09T06:24:33.178287+0000","last_clean":"2023-10-09T06:24:33.178287+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:24:33.178287+0000","last_undegraded":"2023-10-09T06:24:33.178287+0000","last_fullsized":"2023-10-09T06:24:33.178287+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"277'8739","reported_seq":8786,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:43.064860+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:24:43.064860+0000","last_peered":"2023-10-09T06:24:43.064860+0000","last_clean":"2023-10-09T06:24:43.064860+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:24:43.064860+0000","last_undegraded":"2023-10-09T06:24:43.064860+0000","last_fullsized":"2023-10-09T06:24:43.064860+0000","mapping_epoch":238,"log_start":"244'2000","ondisk_log_start":"244'2000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259588096,"num_objects":3961,"num_object_clones":0,"num_object_copies":11883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8739,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":499,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179529+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:24:33.179529+0000","last_peered":"2023-10-09T06:24:33.179529+0000","last_clean":"2023-10-09T06:24:33.179529+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:24:33.179529+0000","last_undegraded":"2023-10-09T06:24:33.179529+0000","last_fullsized":"2023-10-09T06:24:33.179529+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":259,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178539+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:24:33.178539+0000","last_peered":"2023-10-09T06:24:33.178539+0000","last_clean":"2023-10-09T06:24:33.178539+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:24:33.178539+0000","last_undegraded":"2023-10-09T06:24:33.178539+0000","last_fullsized":"2023-10-09T06:24:33.178539+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210587+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:24:33.210587+0000","last_peered":"2023-10-09T06:24:33.210587+0000","last_clean":"2023-10-09T06:24:33.210587+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:24:33.210587+0000","last_undegraded":"2023-10-09T06:24:33.210587+0000","last_fullsized":"2023-10-09T06:24:33.210587+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":202,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.188488+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:24:33.188488+0000","last_peered":"2023-10-09T06:24:33.188488+0000","last_clean":"2023-10-09T06:24:33.188488+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:24:33.188488+0000","last_undegraded":"2023-10-09T06:24:33.188488+0000","last_fullsized":"2023-10-09T06:24:33.188488+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"277'9041","reported_seq":9086,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.840934+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:24:42.840934+0000","last_peered":"2023-10-09T06:24:42.840934+0000","last_clean":"2023-10-09T06:24:42.840934+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:24:42.840934+0000","last_undegraded":"2023-10-09T06:24:42.840934+0000","last_fullsized":"2023-10-09T06:24:42.840934+0000","mapping_epoch":238,"log_start":"247'3000","ondisk_log_start":"247'3000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268369920,"num_objects":4095,"num_object_clones":0,"num_object_copies":12285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9041,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":561,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.175982+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:24:33.175982+0000","last_peered":"2023-10-09T06:24:33.175982+0000","last_clean":"2023-10-09T06:24:33.175982+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:24:33.175982+0000","last_undegraded":"2023-10-09T06:24:33.175982+0000","last_fullsized":"2023-10-09T06:24:33.175982+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":133,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.188624+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:24:33.188624+0000","last_peered":"2023-10-09T06:24:33.188624+0000","last_clean":"2023-10-09T06:24:33.188624+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:24:33.188624+0000","last_undegraded":"2023-10-09T06:24:33.188624+0000","last_fullsized":"2023-10-09T06:24:33.188624+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":238,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156547+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:24:33.156547+0000","last_peered":"2023-10-09T06:24:33.156547+0000","last_clean":"2023-10-09T06:24:33.156547+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:24:33.156547+0000","last_undegraded":"2023-10-09T06:24:33.156547+0000","last_fullsized":"2023-10-09T06:24:33.156547+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":223,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178473+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:24:33.178473+0000","last_peered":"2023-10-09T06:24:33.178473+0000","last_clean":"2023-10-09T06:24:33.178473+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:24:33.178473+0000","last_undegraded":"2023-10-09T06:24:33.178473+0000","last_fullsized":"2023-10-09T06:24:33.178473+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"277'7064","reported_seq":7187,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.070451+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:24:41.070451+0000","last_peered":"2023-10-09T06:24:41.070451+0000","last_clean":"2023-10-09T06:24:41.070451+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:24:41.070451+0000","last_undegraded":"2023-10-09T06:24:41.070451+0000","last_fullsized":"2023-10-09T06:24:41.070451+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140312576,"num_objects":2141,"num_object_clones":0,"num_object_copies":6423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1892,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":569,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185314+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:24:33.185314+0000","last_peered":"2023-10-09T06:24:33.185314+0000","last_clean":"2023-10-09T06:24:33.185314+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:24:33.185314+0000","last_undegraded":"2023-10-09T06:24:33.185314+0000","last_fullsized":"2023-10-09T06:24:33.185314+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":242,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.161421+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:24:33.161421+0000","last_peered":"2023-10-09T06:24:33.161421+0000","last_clean":"2023-10-09T06:24:33.161421+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:24:33.161421+0000","last_undegraded":"2023-10-09T06:24:33.161421+0000","last_fullsized":"2023-10-09T06:24:33.161421+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":249,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178448+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:24:33.178448+0000","last_peered":"2023-10-09T06:24:33.178448+0000","last_clean":"2023-10-09T06:24:33.178448+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:24:33.178448+0000","last_undegraded":"2023-10-09T06:24:33.178448+0000","last_fullsized":"2023-10-09T06:24:33.178448+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":245,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178417+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:24:33.178417+0000","last_peered":"2023-10-09T06:24:33.178417+0000","last_clean":"2023-10-09T06:24:33.178417+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:24:33.178417+0000","last_undegraded":"2023-10-09T06:24:33.178417+0000","last_fullsized":"2023-10-09T06:24:33.178417+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"277'8947","reported_seq":8993,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.871942+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:24:42.871942+0000","last_peered":"2023-10-09T06:24:42.871942+0000","last_clean":"2023-10-09T06:24:42.871942+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:24:42.871942+0000","last_undegraded":"2023-10-09T06:24:42.871942+0000","last_fullsized":"2023-10-09T06:24:42.871942+0000","mapping_epoch":238,"log_start":"247'2900","ondisk_log_start":"247'2900","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6047,"log_dups_size":0,"ondisk_log_size":6047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266797056,"num_objects":4071,"num_object_clones":0,"num_object_copies":12213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8947,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.162971+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:24:33.162971+0000","last_peered":"2023-10-09T06:24:33.162971+0000","last_clean":"2023-10-09T06:24:33.162971+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:24:33.162971+0000","last_undegraded":"2023-10-09T06:24:33.162971+0000","last_fullsized":"2023-10-09T06:24:33.162971+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":550,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.186804+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:24:33.186804+0000","last_peered":"2023-10-09T06:24:33.186804+0000","last_clean":"2023-10-09T06:24:33.186804+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:24:33.186804+0000","last_undegraded":"2023-10-09T06:24:33.186804+0000","last_fullsized":"2023-10-09T06:24:33.186804+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":203,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210274+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:24:33.210274+0000","last_peered":"2023-10-09T06:24:33.210274+0000","last_clean":"2023-10-09T06:24:33.210274+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:24:33.210274+0000","last_undegraded":"2023-10-09T06:24:33.210274+0000","last_fullsized":"2023-10-09T06:24:33.210274+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":239,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179453+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:24:33.179453+0000","last_peered":"2023-10-09T06:24:33.179453+0000","last_clean":"2023-10-09T06:24:33.179453+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:24:33.179453+0000","last_undegraded":"2023-10-09T06:24:33.179453+0000","last_fullsized":"2023-10-09T06:24:33.179453+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"277'6901","reported_seq":7023,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.234121+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:24:40.234121+0000","last_peered":"2023-10-09T06:24:40.234121+0000","last_clean":"2023-10-09T06:24:40.234121+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:24:40.234121+0000","last_undegraded":"2023-10-09T06:24:40.234121+0000","last_fullsized":"2023-10-09T06:24:40.234121+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135462912,"num_objects":2067,"num_object_clones":0,"num_object_copies":6201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1723,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":559,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.180745+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:24:33.180745+0000","last_peered":"2023-10-09T06:24:33.180745+0000","last_clean":"2023-10-09T06:24:33.180745+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:24:33.180745+0000","last_undegraded":"2023-10-09T06:24:33.180745+0000","last_fullsized":"2023-10-09T06:24:33.180745+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.186604+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:24:33.186604+0000","last_peered":"2023-10-09T06:24:33.186604+0000","last_clean":"2023-10-09T06:24:33.186604+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:24:33.186604+0000","last_undegraded":"2023-10-09T06:24:33.186604+0000","last_fullsized":"2023-10-09T06:24:33.186604+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":229,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.175917+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:24:33.175917+0000","last_peered":"2023-10-09T06:24:33.175917+0000","last_clean":"2023-10-09T06:24:33.175917+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:24:33.175917+0000","last_undegraded":"2023-10-09T06:24:33.175917+0000","last_fullsized":"2023-10-09T06:24:33.175917+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173580+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:24:33.173580+0000","last_peered":"2023-10-09T06:24:33.173580+0000","last_clean":"2023-10-09T06:24:33.173580+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:24:33.173580+0000","last_undegraded":"2023-10-09T06:24:33.173580+0000","last_fullsized":"2023-10-09T06:24:33.173580+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"277'9103","reported_seq":9151,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.881535+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:24:42.881535+0000","last_peered":"2023-10-09T06:24:42.881535+0000","last_clean":"2023-10-09T06:24:42.881535+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:24:42.881535+0000","last_undegraded":"2023-10-09T06:24:42.881535+0000","last_fullsized":"2023-10-09T06:24:42.881535+0000","mapping_epoch":238,"log_start":"247'3100","ondisk_log_start":"247'3100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269352984,"num_objects":4111,"num_object_clones":0,"num_object_copies":12333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4111,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9103,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":499,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179413+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:24:33.179413+0000","last_peered":"2023-10-09T06:24:33.179413+0000","last_clean":"2023-10-09T06:24:33.179413+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:24:33.179413+0000","last_undegraded":"2023-10-09T06:24:33.179413+0000","last_fullsized":"2023-10-09T06:24:33.179413+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":120,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.185492+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:24:33.185492+0000","last_peered":"2023-10-09T06:24:33.185492+0000","last_clean":"2023-10-09T06:24:33.185492+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:24:33.185492+0000","last_undegraded":"2023-10-09T06:24:33.185492+0000","last_fullsized":"2023-10-09T06:24:33.185492+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.178403+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:24:33.178403+0000","last_peered":"2023-10-09T06:24:33.178403+0000","last_clean":"2023-10-09T06:24:33.178403+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:24:33.178403+0000","last_undegraded":"2023-10-09T06:24:33.178403+0000","last_fullsized":"2023-10-09T06:24:33.178403+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":241,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173217+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:24:33.173217+0000","last_peered":"2023-10-09T06:24:33.173217+0000","last_clean":"2023-10-09T06:24:33.173217+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:24:33.173217+0000","last_undegraded":"2023-10-09T06:24:33.173217+0000","last_fullsized":"2023-10-09T06:24:33.173217+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"277'8851","reported_seq":8898,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.992877+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:24:42.992877+0000","last_peered":"2023-10-09T06:24:42.992877+0000","last_clean":"2023-10-09T06:24:42.992877+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:24:42.992877+0000","last_undegraded":"2023-10-09T06:24:42.992877+0000","last_fullsized":"2023-10-09T06:24:42.992877+0000","mapping_epoch":238,"log_start":"248'3400","ondisk_log_start":"248'3400","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262471680,"num_objects":4005,"num_object_clones":0,"num_object_copies":12015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8851,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":560,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156578+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:24:33.156578+0000","last_peered":"2023-10-09T06:24:33.156578+0000","last_clean":"2023-10-09T06:24:33.156578+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:24:33.156578+0000","last_undegraded":"2023-10-09T06:24:33.156578+0000","last_fullsized":"2023-10-09T06:24:33.156578+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":266,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.213138+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:24:33.213138+0000","last_peered":"2023-10-09T06:24:33.213138+0000","last_clean":"2023-10-09T06:24:33.213138+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:24:33.213138+0000","last_undegraded":"2023-10-09T06:24:33.213138+0000","last_fullsized":"2023-10-09T06:24:33.213138+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":235,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.182189+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:24:33.182189+0000","last_peered":"2023-10-09T06:24:33.182189+0000","last_clean":"2023-10-09T06:24:33.182189+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:24:33.182189+0000","last_undegraded":"2023-10-09T06:24:33.182189+0000","last_fullsized":"2023-10-09T06:24:33.182189+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.636933+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:24:36.636933+0000","last_peered":"2023-10-09T06:24:36.636933+0000","last_clean":"2023-10-09T06:24:36.636933+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:24:36.636933+0000","last_undegraded":"2023-10-09T06:24:36.636933+0000","last_fullsized":"2023-10-09T06:24:36.636933+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"277'9009","reported_seq":9056,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.868844+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:24:42.868844+0000","last_peered":"2023-10-09T06:24:42.868844+0000","last_clean":"2023-10-09T06:24:42.868844+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:24:42.868844+0000","last_undegraded":"2023-10-09T06:24:42.868844+0000","last_fullsized":"2023-10-09T06:24:42.868844+0000","mapping_epoch":238,"log_start":"247'3000","ondisk_log_start":"247'3000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9009,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":580,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.211568+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:24:33.211568+0000","last_peered":"2023-10-09T06:24:33.211568+0000","last_clean":"2023-10-09T06:24:33.211568+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:24:33.211568+0000","last_undegraded":"2023-10-09T06:24:33.211568+0000","last_fullsized":"2023-10-09T06:24:33.211568+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":261,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.179682+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:24:33.179682+0000","last_peered":"2023-10-09T06:24:33.179682+0000","last_clean":"2023-10-09T06:24:33.179682+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:24:33.179682+0000","last_undegraded":"2023-10-09T06:24:33.179682+0000","last_fullsized":"2023-10-09T06:24:33.179682+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":233,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173489+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:24:33.173489+0000","last_peered":"2023-10-09T06:24:33.173489+0000","last_clean":"2023-10-09T06:24:33.173489+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:24:33.173489+0000","last_undegraded":"2023-10-09T06:24:33.173489+0000","last_fullsized":"2023-10-09T06:24:33.173489+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":230,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156458+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:24:33.156458+0000","last_peered":"2023-10-09T06:24:33.156458+0000","last_clean":"2023-10-09T06:24:33.156458+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:24:33.156458+0000","last_undegraded":"2023-10-09T06:24:33.156458+0000","last_fullsized":"2023-10-09T06:24:33.156458+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"277'7933","reported_seq":12900,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:24:41.847459+0000","last_change":"2023-10-09T06:23:40.105141+0000","last_active":"2023-10-09T06:24:41.847459+0000","last_peered":"2023-10-09T06:24:41.847459+0000","last_clean":"2023-10-09T06:23:37.701201+0000","last_became_active":"2023-10-09T06:23:40.099906+0000","last_became_peered":"2023-10-09T06:23:40.099906+0000","last_unstale":"2023-10-09T06:24:41.847459+0000","last_undegraded":"2023-10-09T06:23:39.992159+0000","last_fullsized":"2023-10-09T06:23:39.910881+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":7933,"log_dups_size":0,"ondisk_log_size":7933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132808704,"num_objects":2027,"num_object_clones":0,"num_object_copies":6081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3168,"num_objects_misplaced":1584,"num_objects_unfound":0,"num_objects_dirty":2027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4429,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1202,"num_bytes_recovered":78774272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[1,2],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1583},{"shards":"1","objects":1},{"shards":"1,2","objects":443}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":235,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173145+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:24:33.173145+0000","last_peered":"2023-10-09T06:24:33.173145+0000","last_clean":"2023-10-09T06:24:33.173145+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:24:33.173145+0000","last_undegraded":"2023-10-09T06:24:33.173145+0000","last_fullsized":"2023-10-09T06:24:33.173145+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":258,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.161487+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:24:33.161487+0000","last_peered":"2023-10-09T06:24:33.161487+0000","last_clean":"2023-10-09T06:24:33.161487+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:24:33.161487+0000","last_undegraded":"2023-10-09T06:24:33.161487+0000","last_fullsized":"2023-10-09T06:24:33.161487+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"277'6946","reported_seq":7003,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:41.129167+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:24:41.129167+0000","last_peered":"2023-10-09T06:24:41.129167+0000","last_clean":"2023-10-09T06:24:41.129167+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:24:41.129167+0000","last_undegraded":"2023-10-09T06:24:41.129167+0000","last_fullsized":"2023-10-09T06:24:41.129167+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6946,"log_dups_size":0,"ondisk_log_size":6946,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138805248,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4404,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":82,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156721+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:24:33.156721+0000","last_peered":"2023-10-09T06:24:33.156721+0000","last_clean":"2023-10-09T06:24:33.156721+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:24:33.156721+0000","last_undegraded":"2023-10-09T06:24:33.156721+0000","last_fullsized":"2023-10-09T06:24:33.156721+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156536+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:24:33.156536+0000","last_peered":"2023-10-09T06:24:33.156536+0000","last_clean":"2023-10-09T06:24:33.156536+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:24:33.156536+0000","last_undegraded":"2023-10-09T06:24:33.156536+0000","last_fullsized":"2023-10-09T06:24:33.156536+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"277'7047","reported_seq":7105,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.239380+0000","last_change":"2023-10-09T06:23:26.834184+0000","last_active":"2023-10-09T06:24:40.239380+0000","last_peered":"2023-10-09T06:24:40.239380+0000","last_clean":"2023-10-09T06:24:40.239380+0000","last_became_active":"2023-10-09T06:23:26.832172+0000","last_became_peered":"2023-10-09T06:23:26.832172+0000","last_unstale":"2023-10-09T06:24:40.239380+0000","last_undegraded":"2023-10-09T06:24:40.239380+0000","last_fullsized":"2023-10-09T06:24:40.239380+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7047,"log_dups_size":0,"ondisk_log_size":7047,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141066240,"num_objects":2153,"num_object_clones":0,"num_object_copies":6459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4445,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":82,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156684+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:24:33.156684+0000","last_peered":"2023-10-09T06:24:33.156684+0000","last_clean":"2023-10-09T06:24:33.156684+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:24:33.156684+0000","last_undegraded":"2023-10-09T06:24:33.156684+0000","last_fullsized":"2023-10-09T06:24:33.156684+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156410+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:24:33.156410+0000","last_peered":"2023-10-09T06:24:33.156410+0000","last_clean":"2023-10-09T06:24:33.156410+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:24:33.156410+0000","last_undegraded":"2023-10-09T06:24:33.156410+0000","last_fullsized":"2023-10-09T06:24:33.156410+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"277'6904","reported_seq":6961,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:42.796192+0000","last_change":"2023-10-09T06:23:26.832837+0000","last_active":"2023-10-09T06:24:42.796192+0000","last_peered":"2023-10-09T06:24:42.796192+0000","last_clean":"2023-10-09T06:24:42.796192+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:24:42.796192+0000","last_undegraded":"2023-10-09T06:24:42.796192+0000","last_fullsized":"2023-10-09T06:24:42.796192+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6904,"log_dups_size":0,"ondisk_log_size":6904,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131268608,"num_objects":2003,"num_object_clones":0,"num_object_copies":6009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4375,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":235,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.638173+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:24:36.638173+0000","last_peered":"2023-10-09T06:24:36.638173+0000","last_clean":"2023-10-09T06:24:36.638173+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:24:36.638173+0000","last_undegraded":"2023-10-09T06:24:36.638173+0000","last_fullsized":"2023-10-09T06:24:36.638173+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":249,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.210923+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:24:33.210923+0000","last_peered":"2023-10-09T06:24:33.210923+0000","last_clean":"2023-10-09T06:24:33.210923+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:24:33.210923+0000","last_undegraded":"2023-10-09T06:24:33.210923+0000","last_fullsized":"2023-10-09T06:24:33.210923+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"277'6902","reported_seq":6960,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:40.166505+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:24:40.166505+0000","last_peered":"2023-10-09T06:24:40.166505+0000","last_clean":"2023-10-09T06:24:40.166505+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:24:40.166505+0000","last_undegraded":"2023-10-09T06:24:40.166505+0000","last_fullsized":"2023-10-09T06:24:40.166505+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139952128,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4344,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":248,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.173222+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:24:33.173222+0000","last_peered":"2023-10-09T06:24:33.173222+0000","last_clean":"2023-10-09T06:24:33.173222+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:24:33.173222+0000","last_undegraded":"2023-10-09T06:24:33.173222+0000","last_fullsized":"2023-10-09T06:24:33.173222+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":237,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-10-09T06:24:33.156147+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:24:33.156147+0000","last_peered":"2023-10-09T06:24:33.156147+0000","last_clean":"2023-10-09T06:24:33.156147+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:24:33.156147+0000","last_undegraded":"2023-10-09T06:24:33.156147+0000","last_fullsized":"2023-10-09T06:24:33.156147+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":4336386072,"num_objects":66171,"num_object_clones":0,"num_object_copies":198513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5973,"num_objects_misplaced":2964,"num_objects_unfound":0,"num_objects_dirty":66171,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":123375,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6045,"num_bytes_recovered":396165120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13681635328,"data_stored":13681623112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":178376,"ondisk_log_size":178376,"up":78,"acting":75,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234329,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208160,"kb_used_data":1299736,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":91536992,"statfs":{"total":95995035648,"available":93733879808,"internally_reserved":0,"allocated":1330929664,"data_stored":1330155681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:44 2023","interfaces":[{"interface":"back","average":{"1min":13.856,"5min":7.093,"15min":3.2669999999999999},"min":{"1min":0.23499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":152.28999999999999,"5min":219.41900000000001,"15min":219.41900000000001},"last":5.6710000000000003},{"interface":"front","average":{"1min":14.646000000000001,"5min":7.3979999999999997,"15min":3.351},"min":{"1min":0.27200000000000002,"5min":0.218,"15min":0.218},"max":{"1min":152.30000000000001,"5min":219.399,"15min":219.399},"last":4.79}]},{"osd":1,"last update":"Mon Oct 9 06:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":3.7959999999999998,"15min":2.6680000000000001},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.7720000000000002,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.881},{"interface":"front","average":{"1min":2.04,"5min":3.6400000000000001,"15min":2.4049999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":12.805999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":5.7629999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":17.384,"5min":10.077999999999999,"15min":4.5700000000000003},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":131.773,"5min":215.34800000000001,"15min":215.34800000000001},"last":2.996},{"interface":"front","average":{"1min":18.056999999999999,"5min":10.329000000000001,"15min":4.742},"min":{"1min":0.35599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":132.28100000000001,"5min":215.869,"15min":215.869},"last":8.6989999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.0209999999999999,"15min":1.4350000000000001},"min":{"1min":0.35699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.7080000000000002,"5min":50.966999999999999,"15min":50.966999999999999},"last":4.7030000000000003},{"interface":"front","average":{"1min":1.4359999999999999,"5min":3.1749999999999998,"15min":1.5069999999999999},"min":{"1min":0.38800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6959999999999997,"5min":51.133000000000003,"15min":51.133000000000003},"last":5.266}]},{"osd":4,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.6080000000000001,"15min":1.3260000000000001},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6790000000000003,"5min":25.678999999999998,"15min":25.678999999999998},"last":5.7380000000000004},{"interface":"front","average":{"1min":1.103,"5min":1.425,"15min":1.117},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.681,"5min":13.305,"15min":13.305},"last":2.3180000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.391,"15min":1.6890000000000001},"min":{"1min":0.34300000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.863,"5min":50.304000000000002,"15min":50.304000000000002},"last":1.9159999999999999},{"interface":"front","average":{"1min":1.395,"5min":2.024,"15min":1.369},"min":{"1min":0.38100000000000001,"5min":0.34399999999999997,"15min":0.28299999999999997},"max":{"1min":5.242,"5min":48.774999999999999,"15min":48.774999999999999},"last":5.7130000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.5489999999999999,"15min":1.302},"min":{"1min":0.39300000000000002,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":7.8479999999999999,"5min":8.4890000000000008,"15min":11.878},"last":5.7889999999999997},{"interface":"front","average":{"1min":1.238,"5min":1.518,"15min":1.218},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":3.722,"5min":12.141999999999999,"15min":12.141999999999999},"last":2.347}]}]},{"osd":6,"up_from":46,"seq":197568495966,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2796440,"kb_used_data":1901520,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":90948712,"statfs":{"total":95995035648,"available":93131481088,"internally_reserved":0,"allocated":1947156480,"data_stored":1946391077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:53 2023","interfaces":[{"interface":"back","average":{"1min":4.8220000000000001,"5min":6.0439999999999996,"15min":3.032},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":58.270000000000003,"5min":110.926,"15min":110.926},"last":0.32600000000000001},{"interface":"front","average":{"1min":4.8010000000000002,"5min":6.5149999999999997,"15min":3.3180000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":58.253999999999998,"5min":110.911,"15min":110.911},"last":0.308}]},{"osd":1,"last update":"Mon Oct 9 06:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":4.9089999999999998,"15min":3.242},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.881,"5min":114.202,"15min":114.202},"last":0.251},{"interface":"front","average":{"1min":1.4259999999999999,"5min":4.3840000000000003,"15min":3.0710000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.1790000000000003,"5min":114.179,"15min":114.179},"last":0.35899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":3.484,"15min":2.6040000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.8419999999999996,"5min":75.191999999999993,"15min":75.191999999999993},"last":1.593},{"interface":"front","average":{"1min":2.4769999999999999,"5min":3.0939999999999999,"15min":2.5059999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.215999999999999,"5min":75.120000000000005,"15min":75.120000000000005},"last":1.6020000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:23:53 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":4.0709999999999997,"15min":1.752},"min":{"1min":0.31900000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.1969999999999992,"5min":110.358,"15min":110.358},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.7430000000000001,"5min":4.2190000000000003,"15min":1.8129999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.7999999999999998,"5min":110.43899999999999,"15min":110.43899999999999},"last":1.1559999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4870000000000001,"15min":1.2549999999999999},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":3.6480000000000001,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.141,"5min":1.4990000000000001,"15min":1.256},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":6.8780000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.39200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.806,"15min":1.3460000000000001},"min":{"1min":0.39200000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":4.3620000000000001,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.597,"5min":1.6499999999999999,"15min":1.3740000000000001},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.2149999999999999,"5min":24.401,"15min":24.401},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":2.4630000000000001,"15min":1.482},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":10.755000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.476,"15min":1.6910000000000001},"min":{"1min":0.40300000000000002,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":100.995,"15min":100.995},"last":0.42299999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402702,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648972,"kb_used_data":1201732,"kb_used_omap":4,"kb_used_meta":447163,"kb_avail":92096180,"statfs":{"total":95995035648,"available":94306488320,"internally_reserved":0,"allocated":1230573568,"data_stored":1229347996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":457895822},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":20.606999999999999,"5min":7.0389999999999997,"15min":3.7789999999999999},"min":{"1min":0.255,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":145.96600000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":22.132000000000001,"5min":7.8310000000000004,"15min":4.2510000000000003},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":145.982,"5min":149.02699999999999,"15min":149.02699999999999},"last":6.2169999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":15.206,"5min":5.3970000000000002,"15min":3.048},"min":{"1min":0.21299999999999999,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":139.75399999999999,"5min":139.75399999999999,"15min":139.75399999999999},"last":5.1929999999999996},{"interface":"front","average":{"1min":16.838000000000001,"5min":6.3470000000000004,"15min":3.8319999999999999},"min":{"1min":0.24199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":140.29900000000001,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.29699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:24:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":5.5369999999999999,"15min":2.198},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.215},"max":{"1min":77.331999999999994,"5min":153.18199999999999,"15min":153.18199999999999},"last":1.6659999999999999},{"interface":"front","average":{"1min":7.5999999999999996,"5min":4.4409999999999998,"15min":1.8240000000000001},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":77.197000000000003,"5min":152.52000000000001,"15min":152.52000000000001},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":16.911999999999999,"5min":6.9329999999999998,"15min":3.8330000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":99.391000000000005,"5min":199.43799999999999,"15min":199.43799999999999},"last":1.706},{"interface":"front","average":{"1min":16.670999999999999,"5min":6.5439999999999996,"15min":3.7650000000000001},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":99.462000000000003,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.376}]},{"osd":5,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":7.665,"5min":3.1459999999999999,"15min":2.7210000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":77.436999999999998,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.496},{"interface":"front","average":{"1min":8.4740000000000002,"5min":3.7610000000000001,"15min":3.0059999999999998},"min":{"1min":0.371,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":77.388000000000005,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":9.5429999999999993,"5min":3.9060000000000001,"15min":3.0030000000000001},"min":{"1min":0.38700000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":77.292000000000002,"5min":77.292000000000002,"15min":77.292000000000002},"last":1.8999999999999999},{"interface":"front","average":{"1min":8.7949999999999999,"5min":3.8450000000000002,"15min":2.7669999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.374},"max":{"1min":77.378,"5min":77.378,"15min":77.378},"last":1.9099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":8.6129999999999995,"5min":3.6909999999999998,"15min":2.7799999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":77.456999999999994,"5min":77.456999999999994,"15min":77.456999999999994},"last":1.7430000000000001},{"interface":"front","average":{"1min":8.1199999999999992,"5min":3.117,"15min":2.2759999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":77.359999999999999,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.53300000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024135,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3401780,"kb_used_data":2745708,"kb_used_omap":5,"kb_used_meta":656058,"kb_avail":90343372,"statfs":{"total":95995035648,"available":92511612928,"internally_reserved":0,"allocated":2811604992,"data_stored":2810169983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":671804026},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":4.673,"15min":2.8130000000000002},"min":{"1min":0.19500000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":13.916,"5min":185.17400000000001,"15min":185.17400000000001},"last":7.5369999999999999},{"interface":"front","average":{"1min":2.9609999999999999,"5min":4.7080000000000002,"15min":3.3500000000000001},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":13.945,"5min":186.691,"15min":186.691},"last":4.4020000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.2349999999999999,"15min":2.2599999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":13.211,"5min":13.211,"15min":17.588999999999999},"last":6.1420000000000003},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.472,"15min":2.391},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":11.956,"5min":13.081,"15min":21.198},"last":3.855}]},{"osd":3,"last update":"Mon Oct 9 06:23:56 2023","interfaces":[{"interface":"back","average":{"1min":3.9329999999999998,"5min":6.8959999999999999,"15min":2.6309999999999998},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.269,"5min":185.89400000000001,"15min":185.89400000000001},"last":2.8180000000000001},{"interface":"front","average":{"1min":2.073,"5min":6.9640000000000004,"15min":2.665},"min":{"1min":0.27500000000000002,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":8.2929999999999993,"5min":186.72900000000001,"15min":186.72900000000001},"last":2.5390000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.6559999999999999,"15min":1.621},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.7969999999999997,"5min":11.162000000000001,"15min":15.835000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.639,"5min":1.879,"15min":2.0550000000000002},"min":{"1min":0.30299999999999999,"5min":0.29499999999999998,"15min":0.25700000000000001},"max":{"1min":5.9889999999999999,"5min":12.752000000000001,"15min":15.023},"last":0.36099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.149,"5min":2.3820000000000001,"15min":2.0569999999999999},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.6239999999999997,"5min":62.984999999999999,"15min":62.984999999999999},"last":0.435},{"interface":"front","average":{"1min":3.5920000000000001,"5min":2.9750000000000001,"15min":2.5920000000000001},"min":{"1min":0.37,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":11.898,"5min":64.983999999999995,"15min":64.983999999999995},"last":2.8759999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7320000000000002,"5min":2.4020000000000001,"15min":2.3740000000000001},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.29299999999999998},"max":{"1min":11.928000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":2.903},{"interface":"front","average":{"1min":2.2370000000000001,"5min":1.8100000000000001,"15min":1.9510000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6539999999999999,"5min":11.236000000000001,"15min":15.893000000000001},"last":0.40400000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":2.4929999999999999,"15min":2.306},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.28899999999999998},"max":{"1min":5.8689999999999998,"5min":62.262,"15min":62.262},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.5019999999999998,"5min":2.8700000000000001,"15min":2.605},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.314},"max":{"1min":11.859,"5min":62.229999999999997,"15min":62.229999999999997},"last":2.9900000000000002}]}]},{"osd":2,"up_from":128,"seq":549755814004,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484152,"kb_used_data":2169968,"kb_used_omap":7,"kb_used_meta":314104,"kb_avail":91261000,"statfs":{"total":95995035648,"available":93451264000,"internally_reserved":0,"allocated":2222047232,"data_stored":2220837925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":321642577},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":3.802,"15min":3.4159999999999999},"min":{"1min":0.20799999999999999,"5min":0.157,"15min":0.157},"max":{"1min":10.356,"5min":110.611,"15min":163.40100000000001},"last":1.752},{"interface":"front","average":{"1min":2.5099999999999998,"5min":3.968,"15min":3.8540000000000001},"min":{"1min":0.24099999999999999,"5min":0.182,"15min":0.182},"max":{"1min":9.0679999999999996,"5min":110.66500000000001,"15min":163.363},"last":2.0569999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:24:27 2023","interfaces":[{"interface":"back","average":{"1min":13.189,"5min":4.4189999999999996,"15min":3.573},"min":{"1min":0.26100000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":205.148,"5min":205.148,"15min":222.50299999999999},"last":0.87},{"interface":"front","average":{"1min":12.093999999999999,"5min":4.5579999999999998,"15min":4.0209999999999999},"min":{"1min":0.316,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":205.267,"5min":205.267,"15min":222.19399999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":3.3100000000000001,"15min":1.482},"min":{"1min":0.28999999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.9300000000000002,"5min":81.183999999999997,"15min":81.183999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.829,"5min":3.5139999999999998,"15min":1.5249999999999999},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.6790000000000003,"5min":81.061999999999998,"15min":81.061999999999998},"last":4.2699999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":7.7530000000000001,"5min":2.7330000000000001,"15min":2.1339999999999999},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":90.938999999999993,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.47799999999999998},{"interface":"front","average":{"1min":7.2480000000000002,"5min":3.1110000000000002,"15min":2.4169999999999998},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":90.950000000000003,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.76700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":1.5409999999999999,"15min":1.579},"min":{"1min":0.41499999999999998,"5min":0.317,"15min":0.309},"max":{"1min":8.9589999999999996,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.675,"5min":2.0259999999999998,"15min":2.1899999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.4320000000000004,"5min":10.548,"15min":10.548},"last":0.81599999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":9.2420000000000009,"5min":3.2189999999999999,"15min":2.4630000000000001},"min":{"1min":0.36199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":138.291,"5min":138.291,"15min":138.291},"last":0.84199999999999997},{"interface":"front","average":{"1min":8.9420000000000002,"5min":2.9399999999999999,"15min":1.9990000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":138.01599999999999,"5min":138.01599999999999,"15min":138.01599999999999},"last":0.94899999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.7949999999999999,"15min":1.651},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":0.89200000000000002},{"interface":"front","average":{"1min":2.73,"5min":1.7809999999999999,"15min":2.3399999999999999},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.9960000000000004,"5min":8.9960000000000004,"15min":9.3249999999999993},"last":0.501}]}]},{"osd":3,"up_from":149,"seq":639950127202,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1742040,"kb_used_data":1351504,"kb_used_omap":2,"kb_used_meta":390525,"kb_avail":92003112,"statfs":{"total":95995035648,"available":94211186688,"internally_reserved":0,"allocated":1383940096,"data_stored":1382819376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":399897653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":7.609,"5min":6.7220000000000004,"15min":2.677},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":63.384999999999998,"5min":187.852,"15min":187.852},"last":0.38800000000000001},{"interface":"front","average":{"1min":6.742,"5min":7.6870000000000003,"15min":3.2240000000000002},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":63.319000000000003,"5min":187.923,"15min":187.923},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":6.048,"15min":2.7069999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.223},"max":{"1min":16.193999999999999,"5min":187.89099999999999,"15min":187.89099999999999},"last":2.6869999999999998},{"interface":"front","average":{"1min":3.145,"5min":3.5939999999999999,"15min":1.6359999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":23.146000000000001,"5min":67.028000000000006,"15min":67.028000000000006},"last":3.6219999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0350000000000001,"5min":5.6349999999999998,"15min":2.5190000000000001},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":39.951999999999998,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.377},{"interface":"front","average":{"1min":3.8540000000000001,"5min":4.5599999999999996,"15min":1.8200000000000001},"min":{"1min":0.26200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":25.530999999999999,"5min":187.79499999999999,"15min":187.79499999999999},"last":3.298}]},{"osd":4,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.7,"15min":0.999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":22.585999999999999,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.0779999999999998,"5min":3.464,"15min":1.946},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":15.942,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.40999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":3.867,"15min":2.109},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.29899999999999999},"max":{"1min":16.001999999999999,"5min":187.833,"15min":187.833},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.3740000000000001,"5min":3.7919999999999998,"15min":2.0779999999999998},"min":{"1min":0.38200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":16.167999999999999,"5min":187.952,"15min":187.952},"last":0.42599999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":3.7530000000000001,"15min":2.0569999999999999},"min":{"1min":0.36199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.976000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":1.929},{"interface":"front","average":{"1min":1.6719999999999999,"5min":3.5899999999999999,"15min":1.9910000000000001},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":16.134,"5min":187.81800000000001,"15min":187.81800000000001},"last":1.603}]},{"osd":7,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":3.8050000000000002,"15min":1.7130000000000001},"min":{"1min":0.42999999999999999,"5min":0.35599999999999998,"15min":0.34100000000000003},"max":{"1min":22.533000000000001,"5min":192.20400000000001,"15min":192.20400000000001},"last":3.3359999999999999},{"interface":"front","average":{"1min":2.476,"5min":3.6499999999999999,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":15.901999999999999,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.52200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019238,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3661784,"kb_used_data":2774992,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":90083368,"statfs":{"total":95995035648,"available":92245368832,"internally_reserved":0,"allocated":2841591808,"data_stored":2840825684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":9.375,"5min":2.968,"15min":2.0329999999999999},"min":{"1min":0.26100000000000001,"5min":0.23899999999999999,"15min":0.222},"max":{"1min":76.944999999999993,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.53500000000000003},{"interface":"front","average":{"1min":9.798,"5min":3.5449999999999999,"15min":2.4329999999999998},"min":{"1min":0.39100000000000001,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":76.988,"5min":76.988,"15min":76.988},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":2.3130000000000002,"15min":2.093},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.2370000000000001,"5min":37.383000000000003,"15min":91.849999999999994},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.599,"5min":2.6379999999999999,"15min":2.1899999999999999},"min":{"1min":0.26000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.4199999999999999,"5min":37.436999999999998,"15min":91.885000000000005},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":2.2839999999999998,"15min":1.8919999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":4.4829999999999997,"5min":69.640000000000001,"15min":69.640000000000001},"last":2.9740000000000002},{"interface":"front","average":{"1min":1.657,"5min":2.5329999999999999,"15min":2.0630000000000002},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":5.3079999999999998,"5min":69.676000000000002,"15min":69.676000000000002},"last":5.1849999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:23:56 2023","interfaces":[{"interface":"back","average":{"1min":3.9119999999999999,"5min":4.1639999999999997,"15min":2.1000000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":45.591000000000001,"5min":57.826999999999998,"15min":91.697000000000003},"last":1.1299999999999999},{"interface":"front","average":{"1min":3.3370000000000002,"5min":3.718,"15min":2.0049999999999999},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":45.548999999999999,"5min":57.853000000000002,"15min":94.356999999999999},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.268,"15min":1.0620000000000001},"min":{"1min":0.29799999999999999,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":4.4400000000000004,"5min":10.673,"15min":10.673},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.202,"15min":1.2050000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":2.9990000000000001,"5min":12.552,"15min":12.552},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 06:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.745,"15min":0.83399999999999996},"min":{"1min":0.315,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":2.1080000000000001,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.2210000000000001,"15min":1.1850000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.4060000000000001,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:24:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":2.444,"15min":1.46},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":4.516,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.79500000000000004,"5min":2.2850000000000001,"15min":1.464},"min":{"1min":0.372,"5min":0.311,"15min":0.311},"max":{"1min":3.407,"5min":62.573,"15min":62.573},"last":0.38700000000000001}]}]},{"osd":5,"up_from":0,"seq":163208757603,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2807238656,"data_stored":2807234584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1220345856,"data_stored":1220345856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2213347328,"data_stored":2213347328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1371537408,"data_stored":1371537408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2822377472,"data_stored":2822373400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1928007680,"data_stored":1928003608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1318780928,"data_stored":1318780928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:24:48.408 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:24:48.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:48.533 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:48.608 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:48.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:48.718 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:48.803 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:48.856 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:24:48.856 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:24:48.856 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:24:48.856 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:24:48.857 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:24:48.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:48 smithi150 ceph-mon[194178]: pgmap v849: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 869 op/s; 5735/192621 objects degraded (2.977%); 2177/192621 objects misplaced (1.130%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:48.912 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:48.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:49.026 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:49.055 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:48 smithi107 ceph-mon[220921]: pgmap v849: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 869 op/s; 5735/192621 objects degraded (2.977%); 2177/192621 objects misplaced (1.130%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:49.055 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:48 smithi107 ceph-mon[212750]: pgmap v849: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 869 op/s; 5735/192621 objects degraded (2.977%); 2177/192621 objects misplaced (1.130%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:49.082 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:49.198 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:49.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:49.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:49.306 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:49.332 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:49.374 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:49.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:49.477 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:49.484 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:24:49.541 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:49.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:49 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:49.894 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:49 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:49.911 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:49.920 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:24:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:49.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:50.028 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:50.045 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:50.061 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:49 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:50.064 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:50.121 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:50.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:50.133 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:24:50.232 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:50.264 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:50.333 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:50.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:50.445 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:50.547 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:50.676 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:50.734 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:50.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:50.753 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:50 smithi107 ceph-mon[212750]: pgmap v850: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 5735/185682 objects degraded (3.089%); 2177/185682 objects misplaced (1.172%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:50.753 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:50 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:50.754 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:50 smithi107 ceph-mon[220921]: pgmap v850: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 5735/185682 objects degraded (3.089%); 2177/185682 objects misplaced (1.172%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:50.754 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:50 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:50.844 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:50.892 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:50.974 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:50.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:50 smithi150 ceph-mon[194178]: pgmap v850: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 878 op/s; 5735/185682 objects degraded (3.089%); 2177/185682 objects misplaced (1.172%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:50.978 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:50 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:50.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:51.079 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:51.120 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:51.161 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:51.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:51.265 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:51.290 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:51.332 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:51.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:51.436 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:51.486 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:51.568 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:51.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:51.679 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:24:51.756 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:51.871 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:24:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:51.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:24:51.981 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:24:52.007 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:51 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:52.010 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:52.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:51 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:52.022 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:24:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:24:52] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:24:52.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:51 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:52.068 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:24:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:52.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:24:52.175 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:52.184 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:52.208 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:52.259 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:24:52.266 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:52.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:52.373 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:52.401 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:52.452 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:52.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:52.558 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:52.591 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[212750]: pgmap v851: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 5735/182901 objects degraded (3.136%); 2177/182901 objects misplaced (1.190%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T06:24:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[212750]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[212750]: failed to encode map e279 with expected crc 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5735/182901 objects degraded (3.136%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[220921]: pgmap v851: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 5735/182901 objects degraded (3.136%); 2177/182901 objects misplaced (1.190%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[220921]: failed to encode map e279 with expected crc 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[220921]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[220921]: failed to encode map e279 with expected crc 2023-10-09T06:24:52.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5735/182901 objects degraded (3.136%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:53.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:52 smithi150 ceph-mon[194178]: pgmap v851: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 5735/182901 objects degraded (3.136%); 2177/182901 objects misplaced (1.190%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T06:24:53.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:52 smithi150 ceph-mon[194178]: failed to encode map e279 with expected crc 2023-10-09T06:24:53.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:52 smithi150 ceph-mon[194178]: osdmap e279: 8 total, 8 up, 7 in 2023-10-09T06:24:53.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:52 smithi150 ceph-mon[194178]: failed to encode map e279 with expected crc 2023-10-09T06:24:53.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5735/182901 objects degraded (3.136%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:53.567 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:53.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:53.675 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:53.717 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:53.785 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:53.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:53.895 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:24:53.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:53.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[220921]: failed to encode map e280 with expected crc 2023-10-09T06:24:53.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[220921]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T06:24:53.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[220921]: failed to encode map e280 with expected crc 2023-10-09T06:24:53.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:53.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T06:24:53.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[212750]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T06:24:53.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:53 smithi107 ceph-mon[212750]: failed to encode map e280 with expected crc 2023-10-09T06:24:53.936 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:54.001 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:24:54.005 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:53 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:54.005 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:53 smithi150 ceph-mon[194178]: failed to encode map e280 with expected crc 2023-10-09T06:24:54.005 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:53 smithi150 ceph-mon[194178]: osdmap e280: 8 total, 8 up, 7 in 2023-10-09T06:24:54.005 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:53 smithi150 ceph-mon[194178]: failed to encode map e280 with expected crc 2023-10-09T06:24:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:54.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:24:54.107 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.183 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:54.305 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:54.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:54.423 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:24:54.521 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:54.574 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:24:54.579 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:24:54.773 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:54 smithi107 ceph-mon[220921]: pgmap v854: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5487/174777 objects degraded (3.139%); 1341/174777 objects misplaced (0.767%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:24:55.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:54 smithi107 ceph-mon[212750]: pgmap v854: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5487/174777 objects degraded (3.139%); 1341/174777 objects misplaced (0.767%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:24:55.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:54 smithi150 ceph-mon[194178]: pgmap v854: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 5487/174777 objects degraded (3.139%); 1341/174777 objects misplaced (0.767%); 7.9 MiB/s, 126 objects/s recovering 2023-10-09T06:24:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:55 smithi107 ceph-mon[212750]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:56.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:55 smithi107 ceph-mon[220921]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:56.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:55 smithi150 ceph-mon[194178]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:24:56.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:56 smithi107 ceph-mon[212750]: pgmap v855: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 5487/174129 objects degraded (3.151%); 1341/174129 objects misplaced (0.770%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:24:56.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:56 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:24:56.833 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:24:56.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:56.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:24:56.936 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:57.007 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:57.132 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:56 smithi107 ceph-mon[220921]: pgmap v855: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 5487/174129 objects degraded (3.151%); 1341/174129 objects misplaced (0.770%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:24:57.132 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:56 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:24:57.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:56 smithi150 ceph-mon[194178]: pgmap v855: 109 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 5487/174129 objects degraded (3.151%); 1341/174129 objects misplaced (0.770%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:24:57.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:56 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:24:57.502 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:57.504 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":854,"stamp":"2023-10-09T06:24:53.100600+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3816888248,"num_objects":58259,"num_object_clones":0,"num_object_copies":174777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5487,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":58259,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131372,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7006,"num_bytes_recovered":457648218,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":184782,"ondisk_log_size":184782,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16674000,"kb_used_data":12148248,"kb_used_omap":20,"kb_used_meta":4525611,"kb_avail":639542064,"statfs":{"total":671965249536,"available":654891073536,"internally_reserved":0,"allocated":12439805952,"data_stored":12432470130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4634226318},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,6,6,7,0,1,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-459603968,"num_objects":-7013,"num_object_clones":0,"num_object_copies":-21039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-486,"num_objects_misplaced":-1623,"num_objects_unfound":0,"num_objects_dirty":-7013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7013,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":933,"num_bytes_recovered":61145088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.400156"},"pg_stats":[{"pgid":"10.13","version":"278'7274","reported_seq":15768,"reported_epoch":279,"state":"peering","last_fresh":"2023-10-09T06:24:51.787918+0000","last_change":"2023-10-09T06:24:51.787918+0000","last_active":"2023-10-09T06:24:51.787918+0000","last_peered":"2023-10-09T06:24:51.659128+0000","last_clean":"2023-10-09T06:23:38.908387+0000","last_became_active":"2023-10-09T06:23:42.387234+0000","last_became_peered":"2023-10-09T06:23:42.387234+0000","last_unstale":"2023-10-09T06:24:51.787918+0000","last_undegraded":"2023-10-09T06:24:51.787918+0000","last_fullsized":"2023-10-09T06:24:51.787918+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7274,"log_dups_size":0,"ondisk_log_size":7274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120717312,"num_objects":1842,"num_object_clones":0,"num_object_copies":5526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4694,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2810,"num_bytes_recovered":184156160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":259,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.756326+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:24:51.756326+0000","last_peered":"2023-10-09T06:24:51.756326+0000","last_clean":"2023-10-09T06:24:51.756326+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:24:51.756326+0000","last_undegraded":"2023-10-09T06:24:51.756326+0000","last_fullsized":"2023-10-09T06:24:51.756326+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":215,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637608+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:24:36.637608+0000","last_peered":"2023-10-09T06:24:36.637608+0000","last_clean":"2023-10-09T06:24:36.637608+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:24:36.637608+0000","last_undegraded":"2023-10-09T06:24:36.637608+0000","last_fullsized":"2023-10-09T06:24:36.637608+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"280'7245","reported_seq":12805,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.902272+0000","last_change":"2023-10-09T06:24:52.602120+0000","last_active":"2023-10-09T06:24:52.902272+0000","last_peered":"2023-10-09T06:24:52.902272+0000","last_clean":"2023-10-09T06:24:52.902272+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:24:52.902272+0000","last_undegraded":"2023-10-09T06:24:52.902272+0000","last_fullsized":"2023-10-09T06:24:52.902272+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7245,"log_dups_size":0,"ondisk_log_size":7245,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118161408,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4687,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":250,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.710582+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:24:45.710582+0000","last_peered":"2023-10-09T06:24:45.710582+0000","last_clean":"2023-10-09T06:24:45.710582+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:24:45.710582+0000","last_undegraded":"2023-10-09T06:24:45.710582+0000","last_fullsized":"2023-10-09T06:24:45.710582+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631950+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:24:36.631950+0000","last_peered":"2023-10-09T06:24:36.631950+0000","last_clean":"2023-10-09T06:24:36.631950+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:24:36.631950+0000","last_undegraded":"2023-10-09T06:24:36.631950+0000","last_fullsized":"2023-10-09T06:24:36.631950+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"278'7254","reported_seq":7316,"reported_epoch":278,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:24:50.213386+0000","last_change":"2023-10-09T06:23:38.088533+0000","last_active":"2023-10-09T06:24:50.213386+0000","last_peered":"2023-10-09T06:24:50.213386+0000","last_clean":"2023-10-09T06:23:34.462031+0000","last_became_active":"2023-10-09T06:23:38.083231+0000","last_became_peered":"2023-10-09T06:23:38.083231+0000","last_unstale":"2023-10-09T06:24:50.213386+0000","last_undegraded":"2023-10-09T06:23:38.014323+0000","last_fullsized":"2023-10-09T06:23:37.903678+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7254,"log_dups_size":0,"ondisk_log_size":7254,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2805,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4636,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":1441},{"shards":"0,7","objects":-897}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":233,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637063+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:24:36.637063+0000","last_peered":"2023-10-09T06:24:36.637063+0000","last_clean":"2023-10-09T06:24:36.637063+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:24:36.637063+0000","last_undegraded":"2023-10-09T06:24:36.637063+0000","last_fullsized":"2023-10-09T06:24:36.637063+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":259,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760472+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:24:51.760472+0000","last_peered":"2023-10-09T06:24:51.760472+0000","last_clean":"2023-10-09T06:24:51.760472+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:24:51.760472+0000","last_undegraded":"2023-10-09T06:24:51.760472+0000","last_fullsized":"2023-10-09T06:24:51.760472+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631726+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:24:36.631726+0000","last_peered":"2023-10-09T06:24:36.631726+0000","last_clean":"2023-10-09T06:24:36.631726+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:24:36.631726+0000","last_undegraded":"2023-10-09T06:24:36.631726+0000","last_fullsized":"2023-10-09T06:24:36.631726+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":215,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637013+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:24:36.637013+0000","last_peered":"2023-10-09T06:24:36.637013+0000","last_clean":"2023-10-09T06:24:36.637013+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:24:36.637013+0000","last_undegraded":"2023-10-09T06:24:36.637013+0000","last_fullsized":"2023-10-09T06:24:36.637013+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634106+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:24:36.634106+0000","last_peered":"2023-10-09T06:24:36.634106+0000","last_clean":"2023-10-09T06:24:36.634106+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:24:36.634106+0000","last_undegraded":"2023-10-09T06:24:36.634106+0000","last_fullsized":"2023-10-09T06:24:36.634106+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":213,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637266+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:24:36.637266+0000","last_peered":"2023-10-09T06:24:36.637266+0000","last_clean":"2023-10-09T06:24:36.637266+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:24:36.637266+0000","last_undegraded":"2023-10-09T06:24:36.637266+0000","last_fullsized":"2023-10-09T06:24:36.637266+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":127,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634065+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:24:36.634065+0000","last_peered":"2023-10-09T06:24:36.634065+0000","last_clean":"2023-10-09T06:24:36.634065+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:24:36.634065+0000","last_undegraded":"2023-10-09T06:24:36.634065+0000","last_fullsized":"2023-10-09T06:24:36.634065+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":210,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.713923+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:24:45.713923+0000","last_peered":"2023-10-09T06:24:45.713923+0000","last_clean":"2023-10-09T06:24:45.713923+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:24:45.713923+0000","last_undegraded":"2023-10-09T06:24:45.713923+0000","last_fullsized":"2023-10-09T06:24:45.713923+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":286,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758672+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:24:51.758672+0000","last_peered":"2023-10-09T06:24:51.758672+0000","last_clean":"2023-10-09T06:24:51.758672+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:24:51.758672+0000","last_undegraded":"2023-10-09T06:24:51.758672+0000","last_fullsized":"2023-10-09T06:24:51.758672+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":250,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.757122+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:24:51.757122+0000","last_peered":"2023-10-09T06:24:51.757122+0000","last_clean":"2023-10-09T06:24:51.757122+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:24:51.757122+0000","last_undegraded":"2023-10-09T06:24:51.757122+0000","last_fullsized":"2023-10-09T06:24:51.757122+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":127,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.633988+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:24:36.633988+0000","last_peered":"2023-10-09T06:24:36.633988+0000","last_clean":"2023-10-09T06:24:36.633988+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:24:36.633988+0000","last_undegraded":"2023-10-09T06:24:36.633988+0000","last_fullsized":"2023-10-09T06:24:36.633988+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":233,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637845+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:24:36.637845+0000","last_peered":"2023-10-09T06:24:36.637845+0000","last_clean":"2023-10-09T06:24:36.637845+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:24:36.637845+0000","last_undegraded":"2023-10-09T06:24:36.637845+0000","last_fullsized":"2023-10-09T06:24:36.637845+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":248,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.643347+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:24:36.643347+0000","last_peered":"2023-10-09T06:24:36.643347+0000","last_clean":"2023-10-09T06:24:36.643347+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:24:36.643347+0000","last_undegraded":"2023-10-09T06:24:36.643347+0000","last_fullsized":"2023-10-09T06:24:36.643347+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.633954+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:24:36.633954+0000","last_peered":"2023-10-09T06:24:36.633954+0000","last_clean":"2023-10-09T06:24:36.633954+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:24:36.633954+0000","last_undegraded":"2023-10-09T06:24:36.633954+0000","last_fullsized":"2023-10-09T06:24:36.633954+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"278'7149","reported_seq":7207,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.245442+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:24:50.245442+0000","last_peered":"2023-10-09T06:24:50.245442+0000","last_clean":"2023-10-09T06:24:50.245442+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:24:50.245442+0000","last_undegraded":"2023-10-09T06:24:50.245442+0000","last_fullsized":"2023-10-09T06:24:50.245442+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121143296,"num_objects":1849,"num_object_clones":0,"num_object_copies":5547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4597,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634399+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:24:36.634399+0000","last_peered":"2023-10-09T06:24:36.634399+0000","last_clean":"2023-10-09T06:24:36.634399+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:24:36.634399+0000","last_undegraded":"2023-10-09T06:24:36.634399+0000","last_fullsized":"2023-10-09T06:24:36.634399+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":247,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.632073+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:24:36.632073+0000","last_peered":"2023-10-09T06:24:36.632073+0000","last_clean":"2023-10-09T06:24:36.632073+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:24:36.632073+0000","last_undegraded":"2023-10-09T06:24:36.632073+0000","last_fullsized":"2023-10-09T06:24:36.632073+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"278'7237","reported_seq":7296,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.058872+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:24:51.058872+0000","last_peered":"2023-10-09T06:24:51.058872+0000","last_clean":"2023-10-09T06:24:51.058872+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:24:51.058872+0000","last_undegraded":"2023-10-09T06:24:51.058872+0000","last_fullsized":"2023-10-09T06:24:51.058872+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7237,"log_dups_size":0,"ondisk_log_size":7237,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120291328,"num_objects":1836,"num_object_clones":0,"num_object_copies":5508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4672,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":268,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760000+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:24:51.760000+0000","last_peered":"2023-10-09T06:24:51.760000+0000","last_clean":"2023-10-09T06:24:51.760000+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:24:51.760000+0000","last_undegraded":"2023-10-09T06:24:51.760000+0000","last_fullsized":"2023-10-09T06:24:51.760000+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":233,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637611+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:24:36.637611+0000","last_peered":"2023-10-09T06:24:36.637611+0000","last_clean":"2023-10-09T06:24:36.637611+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:24:36.637611+0000","last_undegraded":"2023-10-09T06:24:36.637611+0000","last_fullsized":"2023-10-09T06:24:36.637611+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"278'7240","reported_seq":7368,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.233878+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:24:50.233878+0000","last_peered":"2023-10-09T06:24:50.233878+0000","last_clean":"2023-10-09T06:24:50.233878+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:24:50.233878+0000","last_undegraded":"2023-10-09T06:24:50.233878+0000","last_fullsized":"2023-10-09T06:24:50.233878+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":7240,"log_dups_size":0,"ondisk_log_size":7240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1978,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":139,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634006+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:24:36.634006+0000","last_peered":"2023-10-09T06:24:36.634006+0000","last_clean":"2023-10-09T06:24:36.634006+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:24:36.634006+0000","last_undegraded":"2023-10-09T06:24:36.634006+0000","last_fullsized":"2023-10-09T06:24:36.634006+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":188,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.633811+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:24:36.633811+0000","last_peered":"2023-10-09T06:24:36.633811+0000","last_clean":"2023-10-09T06:24:36.633811+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:24:36.633811+0000","last_undegraded":"2023-10-09T06:24:36.633811+0000","last_fullsized":"2023-10-09T06:24:36.633811+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"280'9066","reported_seq":9243,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.895008+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:24:52.895008+0000","last_peered":"2023-10-09T06:24:52.895008+0000","last_clean":"2023-10-09T06:24:52.895008+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:24:52.895008+0000","last_undegraded":"2023-10-09T06:24:52.895008+0000","last_fullsized":"2023-10-09T06:24:52.895008+0000","mapping_epoch":238,"log_start":"243'956","ondisk_log_start":"243'956","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":8110,"log_dups_size":0,"ondisk_log_size":8110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1964,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":125,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634256+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:24:36.634256+0000","last_peered":"2023-10-09T06:24:36.634256+0000","last_clean":"2023-10-09T06:24:36.634256+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:24:36.634256+0000","last_undegraded":"2023-10-09T06:24:36.634256+0000","last_fullsized":"2023-10-09T06:24:36.634256+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":257,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758442+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:24:51.758442+0000","last_peered":"2023-10-09T06:24:51.758442+0000","last_clean":"2023-10-09T06:24:51.758442+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:24:51.758442+0000","last_undegraded":"2023-10-09T06:24:51.758442+0000","last_fullsized":"2023-10-09T06:24:51.758442+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"278'7348","reported_seq":7408,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.237733+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:24:50.237733+0000","last_peered":"2023-10-09T06:24:50.237733+0000","last_clean":"2023-10-09T06:24:50.237733+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:24:50.237733+0000","last_undegraded":"2023-10-09T06:24:50.237733+0000","last_fullsized":"2023-10-09T06:24:50.237733+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118194176,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4745,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":139,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.633930+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:24:36.633930+0000","last_peered":"2023-10-09T06:24:36.633930+0000","last_clean":"2023-10-09T06:24:36.633930+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:24:36.633930+0000","last_undegraded":"2023-10-09T06:24:36.633930+0000","last_fullsized":"2023-10-09T06:24:36.633930+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":259,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.762484+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:24:51.762484+0000","last_peered":"2023-10-09T06:24:51.762484+0000","last_clean":"2023-10-09T06:24:51.762484+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:24:51.762484+0000","last_undegraded":"2023-10-09T06:24:51.762484+0000","last_fullsized":"2023-10-09T06:24:51.762484+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"278'7219","reported_seq":7279,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.014753+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:24:51.014753+0000","last_peered":"2023-10-09T06:24:51.014753+0000","last_clean":"2023-10-09T06:24:51.014753+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:24:51.014753+0000","last_undegraded":"2023-10-09T06:24:51.014753+0000","last_fullsized":"2023-10-09T06:24:51.014753+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7219,"log_dups_size":0,"ondisk_log_size":7219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4677,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":169,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637700+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:24:36.637700+0000","last_peered":"2023-10-09T06:24:36.637700+0000","last_clean":"2023-10-09T06:24:36.637700+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:24:36.637700+0000","last_undegraded":"2023-10-09T06:24:36.637700+0000","last_fullsized":"2023-10-09T06:24:36.637700+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":230,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.630555+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:24:36.630555+0000","last_peered":"2023-10-09T06:24:36.630555+0000","last_clean":"2023-10-09T06:24:36.630555+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:24:36.630555+0000","last_undegraded":"2023-10-09T06:24:36.630555+0000","last_fullsized":"2023-10-09T06:24:36.630555+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"279'7209","reported_seq":7340,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.784312+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:24:51.784312+0000","last_peered":"2023-10-09T06:24:51.784312+0000","last_clean":"2023-10-09T06:24:51.784312+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:24:51.784312+0000","last_undegraded":"2023-10-09T06:24:51.784312+0000","last_fullsized":"2023-10-09T06:24:51.784312+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1915,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":570,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634174+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:24:36.634174+0000","last_peered":"2023-10-09T06:24:36.634174+0000","last_clean":"2023-10-09T06:24:36.634174+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:24:36.634174+0000","last_undegraded":"2023-10-09T06:24:36.634174+0000","last_fullsized":"2023-10-09T06:24:36.634174+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":241,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637881+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:24:36.637881+0000","last_peered":"2023-10-09T06:24:36.637881+0000","last_clean":"2023-10-09T06:24:36.637881+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:24:36.637881+0000","last_undegraded":"2023-10-09T06:24:36.637881+0000","last_fullsized":"2023-10-09T06:24:36.637881+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758694+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:24:51.758694+0000","last_peered":"2023-10-09T06:24:51.758694+0000","last_clean":"2023-10-09T06:24:51.758694+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:24:51.758694+0000","last_undegraded":"2023-10-09T06:24:51.758694+0000","last_fullsized":"2023-10-09T06:24:51.758694+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"278'7338","reported_seq":7398,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.052790+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:24:51.052790+0000","last_peered":"2023-10-09T06:24:51.052790+0000","last_clean":"2023-10-09T06:24:51.052790+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:24:51.052790+0000","last_undegraded":"2023-10-09T06:24:51.052790+0000","last_fullsized":"2023-10-09T06:24:51.052790+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124780544,"num_objects":1904,"num_object_clones":0,"num_object_copies":5712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4758,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":561,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.615792+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:24:36.615792+0000","last_peered":"2023-10-09T06:24:36.615792+0000","last_clean":"2023-10-09T06:24:36.615792+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:24:36.615792+0000","last_undegraded":"2023-10-09T06:24:36.615792+0000","last_fullsized":"2023-10-09T06:24:36.615792+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.762291+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:24:51.762291+0000","last_peered":"2023-10-09T06:24:51.762291+0000","last_clean":"2023-10-09T06:24:51.762291+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:24:51.762291+0000","last_undegraded":"2023-10-09T06:24:51.762291+0000","last_fullsized":"2023-10-09T06:24:51.762291+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":257,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758708+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:24:51.758708+0000","last_peered":"2023-10-09T06:24:51.758708+0000","last_clean":"2023-10-09T06:24:51.758708+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:24:51.758708+0000","last_undegraded":"2023-10-09T06:24:51.758708+0000","last_fullsized":"2023-10-09T06:24:51.758708+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"280'7237","reported_seq":7412,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.842493+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:24:52.842493+0000","last_peered":"2023-10-09T06:24:52.842493+0000","last_clean":"2023-10-09T06:24:52.842493+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:24:52.842493+0000","last_undegraded":"2023-10-09T06:24:52.842493+0000","last_fullsized":"2023-10-09T06:24:52.842493+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":7237,"log_dups_size":0,"ondisk_log_size":7237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119537664,"num_objects":1824,"num_object_clones":0,"num_object_copies":5472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2102,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":261,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758364+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:24:51.758364+0000","last_peered":"2023-10-09T06:24:51.758364+0000","last_clean":"2023-10-09T06:24:51.758364+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:24:51.758364+0000","last_undegraded":"2023-10-09T06:24:51.758364+0000","last_fullsized":"2023-10-09T06:24:51.758364+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.762369+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:24:51.762369+0000","last_peered":"2023-10-09T06:24:51.762369+0000","last_clean":"2023-10-09T06:24:51.762369+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:24:51.762369+0000","last_undegraded":"2023-10-09T06:24:51.762369+0000","last_fullsized":"2023-10-09T06:24:51.762369+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"280'7446","reported_seq":7628,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.965069+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:24:52.965069+0000","last_peered":"2023-10-09T06:24:52.965069+0000","last_clean":"2023-10-09T06:24:52.965069+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:24:52.965069+0000","last_undegraded":"2023-10-09T06:24:52.965069+0000","last_fullsized":"2023-10-09T06:24:52.965069+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":7446,"log_dups_size":0,"ondisk_log_size":7446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118554624,"num_objects":1809,"num_object_clones":0,"num_object_copies":5427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2021,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":557,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.615919+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:24:36.615919+0000","last_peered":"2023-10-09T06:24:36.615919+0000","last_clean":"2023-10-09T06:24:36.615919+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:24:36.615919+0000","last_undegraded":"2023-10-09T06:24:36.615919+0000","last_fullsized":"2023-10-09T06:24:36.615919+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":234,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.636989+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:24:36.636989+0000","last_peered":"2023-10-09T06:24:36.636989+0000","last_clean":"2023-10-09T06:24:36.636989+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:24:36.636989+0000","last_undegraded":"2023-10-09T06:24:36.636989+0000","last_fullsized":"2023-10-09T06:24:36.636989+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.711331+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:24:45.711331+0000","last_peered":"2023-10-09T06:24:45.711331+0000","last_clean":"2023-10-09T06:24:45.711331+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:24:45.711331+0000","last_undegraded":"2023-10-09T06:24:45.711331+0000","last_fullsized":"2023-10-09T06:24:45.711331+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"278'8997","reported_seq":9045,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:48.066566+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:24:48.066566+0000","last_peered":"2023-10-09T06:24:48.066566+0000","last_clean":"2023-10-09T06:24:48.066566+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:24:48.066566+0000","last_undegraded":"2023-10-09T06:24:48.066566+0000","last_fullsized":"2023-10-09T06:24:48.066566+0000","mapping_epoch":238,"log_start":"245'2300","ondisk_log_start":"245'2300","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242679808,"num_objects":3703,"num_object_clones":0,"num_object_copies":11109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8997,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":502,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760523+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:24:51.760523+0000","last_peered":"2023-10-09T06:24:51.760523+0000","last_clean":"2023-10-09T06:24:51.760523+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:24:51.760523+0000","last_undegraded":"2023-10-09T06:24:51.760523+0000","last_fullsized":"2023-10-09T06:24:51.760523+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":261,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.711014+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:24:45.711014+0000","last_peered":"2023-10-09T06:24:45.711014+0000","last_clean":"2023-10-09T06:24:45.711014+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:24:45.711014+0000","last_undegraded":"2023-10-09T06:24:45.711014+0000","last_fullsized":"2023-10-09T06:24:45.711014+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":186,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637460+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:24:36.637460+0000","last_peered":"2023-10-09T06:24:36.637460+0000","last_clean":"2023-10-09T06:24:36.637460+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:24:36.637460+0000","last_undegraded":"2023-10-09T06:24:36.637460+0000","last_fullsized":"2023-10-09T06:24:36.637460+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":204,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.713706+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:24:45.713706+0000","last_peered":"2023-10-09T06:24:45.713706+0000","last_clean":"2023-10-09T06:24:45.713706+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:24:45.713706+0000","last_undegraded":"2023-10-09T06:24:45.713706+0000","last_fullsized":"2023-10-09T06:24:45.713706+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"280'9516","reported_seq":9566,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.811371+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:24:52.811371+0000","last_peered":"2023-10-09T06:24:52.811371+0000","last_clean":"2023-10-09T06:24:52.811371+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:24:52.811371+0000","last_undegraded":"2023-10-09T06:24:52.811371+0000","last_fullsized":"2023-10-09T06:24:52.811371+0000","mapping_epoch":238,"log_start":"248'3500","ondisk_log_start":"248'3500","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237240320,"num_objects":3620,"num_object_clones":0,"num_object_copies":10860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9516,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":562,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631133+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:24:36.631133+0000","last_peered":"2023-10-09T06:24:36.631133+0000","last_clean":"2023-10-09T06:24:36.631133+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:24:36.631133+0000","last_undegraded":"2023-10-09T06:24:36.631133+0000","last_fullsized":"2023-10-09T06:24:36.631133+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":135,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.713778+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:24:45.713778+0000","last_peered":"2023-10-09T06:24:45.713778+0000","last_clean":"2023-10-09T06:24:45.713778+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:24:45.713778+0000","last_undegraded":"2023-10-09T06:24:45.713778+0000","last_fullsized":"2023-10-09T06:24:45.713778+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":241,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.756631+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:24:51.756631+0000","last_peered":"2023-10-09T06:24:51.756631+0000","last_clean":"2023-10-09T06:24:51.756631+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:24:51.756631+0000","last_undegraded":"2023-10-09T06:24:51.756631+0000","last_fullsized":"2023-10-09T06:24:51.756631+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":225,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.710944+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:24:45.710944+0000","last_peered":"2023-10-09T06:24:45.710944+0000","last_clean":"2023-10-09T06:24:45.710944+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:24:45.710944+0000","last_undegraded":"2023-10-09T06:24:45.710944+0000","last_fullsized":"2023-10-09T06:24:45.710944+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"278'7328","reported_seq":7452,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.068716+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:24:51.068716+0000","last_peered":"2023-10-09T06:24:51.068716+0000","last_clean":"2023-10-09T06:24:51.068716+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:24:51.068716+0000","last_undegraded":"2023-10-09T06:24:51.068716+0000","last_fullsized":"2023-10-09T06:24:51.068716+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2156,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":570,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634032+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:24:36.634032+0000","last_peered":"2023-10-09T06:24:36.634032+0000","last_clean":"2023-10-09T06:24:36.634032+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:24:36.634032+0000","last_undegraded":"2023-10-09T06:24:36.634032+0000","last_fullsized":"2023-10-09T06:24:36.634032+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":243,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.615845+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:24:36.615845+0000","last_peered":"2023-10-09T06:24:36.615845+0000","last_clean":"2023-10-09T06:24:36.615845+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:24:36.615845+0000","last_undegraded":"2023-10-09T06:24:36.615845+0000","last_fullsized":"2023-10-09T06:24:36.615845+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":251,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.710439+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:24:45.710439+0000","last_peered":"2023-10-09T06:24:45.710439+0000","last_clean":"2023-10-09T06:24:45.710439+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:24:45.710439+0000","last_undegraded":"2023-10-09T06:24:45.710439+0000","last_fullsized":"2023-10-09T06:24:45.710439+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":247,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.711401+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:24:45.711401+0000","last_peered":"2023-10-09T06:24:45.711401+0000","last_clean":"2023-10-09T06:24:45.711401+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:24:45.711401+0000","last_undegraded":"2023-10-09T06:24:45.711401+0000","last_fullsized":"2023-10-09T06:24:45.711401+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"280'9421","reported_seq":9471,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.832617+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:24:52.832617+0000","last_peered":"2023-10-09T06:24:52.832617+0000","last_clean":"2023-10-09T06:24:52.832617+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:24:52.832617+0000","last_undegraded":"2023-10-09T06:24:52.832617+0000","last_fullsized":"2023-10-09T06:24:52.832617+0000","mapping_epoch":238,"log_start":"248'3400","ondisk_log_start":"248'3400","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9421,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":210,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.615895+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:24:36.615895+0000","last_peered":"2023-10-09T06:24:36.615895+0000","last_clean":"2023-10-09T06:24:36.615895+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:24:36.615895+0000","last_undegraded":"2023-10-09T06:24:36.615895+0000","last_fullsized":"2023-10-09T06:24:36.615895+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":551,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634146+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:24:36.634146+0000","last_peered":"2023-10-09T06:24:36.634146+0000","last_clean":"2023-10-09T06:24:36.634146+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:24:36.634146+0000","last_undegraded":"2023-10-09T06:24:36.634146+0000","last_fullsized":"2023-10-09T06:24:36.634146+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":204,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637205+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:24:36.637205+0000","last_peered":"2023-10-09T06:24:36.637205+0000","last_clean":"2023-10-09T06:24:36.637205+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:24:36.637205+0000","last_undegraded":"2023-10-09T06:24:36.637205+0000","last_fullsized":"2023-10-09T06:24:36.637205+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760309+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:24:51.760309+0000","last_peered":"2023-10-09T06:24:51.760309+0000","last_clean":"2023-10-09T06:24:51.760309+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:24:51.760309+0000","last_undegraded":"2023-10-09T06:24:51.760309+0000","last_fullsized":"2023-10-09T06:24:51.760309+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"278'7152","reported_seq":7275,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.155539+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:24:50.155539+0000","last_peered":"2023-10-09T06:24:50.155539+0000","last_clean":"2023-10-09T06:24:50.155539+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:24:50.155539+0000","last_undegraded":"2023-10-09T06:24:50.155539+0000","last_fullsized":"2023-10-09T06:24:50.155539+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":7152,"log_dups_size":0,"ondisk_log_size":7152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1974,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":561,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.712584+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:24:45.712584+0000","last_peered":"2023-10-09T06:24:45.712584+0000","last_clean":"2023-10-09T06:24:45.712584+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:24:45.712584+0000","last_undegraded":"2023-10-09T06:24:45.712584+0000","last_fullsized":"2023-10-09T06:24:45.712584+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":121,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.633715+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:24:36.633715+0000","last_peered":"2023-10-09T06:24:36.633715+0000","last_clean":"2023-10-09T06:24:36.633715+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:24:36.633715+0000","last_undegraded":"2023-10-09T06:24:36.633715+0000","last_fullsized":"2023-10-09T06:24:36.633715+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":230,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631057+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:24:36.631057+0000","last_peered":"2023-10-09T06:24:36.631057+0000","last_clean":"2023-10-09T06:24:36.631057+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:24:36.631057+0000","last_undegraded":"2023-10-09T06:24:36.631057+0000","last_fullsized":"2023-10-09T06:24:36.631057+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":253,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.632011+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:24:36.632011+0000","last_peered":"2023-10-09T06:24:36.632011+0000","last_clean":"2023-10-09T06:24:36.632011+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:24:36.632011+0000","last_undegraded":"2023-10-09T06:24:36.632011+0000","last_fullsized":"2023-10-09T06:24:36.632011+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"280'9615","reported_seq":9666,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.871251+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:24:52.871251+0000","last_peered":"2023-10-09T06:24:52.871251+0000","last_clean":"2023-10-09T06:24:52.871251+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:24:52.871251+0000","last_undegraded":"2023-10-09T06:24:52.871251+0000","last_fullsized":"2023-10-09T06:24:52.871251+0000","mapping_epoch":238,"log_start":"248'3600","ondisk_log_start":"248'3600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235798552,"num_objects":3599,"num_object_clones":0,"num_object_copies":10797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3599,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9615,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":502,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760312+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:24:51.760312+0000","last_peered":"2023-10-09T06:24:51.760312+0000","last_clean":"2023-10-09T06:24:51.760312+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:24:51.760312+0000","last_undegraded":"2023-10-09T06:24:51.760312+0000","last_fullsized":"2023-10-09T06:24:51.760312+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.634061+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:24:36.634061+0000","last_peered":"2023-10-09T06:24:36.634061+0000","last_clean":"2023-10-09T06:24:36.634061+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:24:36.634061+0000","last_undegraded":"2023-10-09T06:24:36.634061+0000","last_fullsized":"2023-10-09T06:24:36.634061+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.710860+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:24:45.710860+0000","last_peered":"2023-10-09T06:24:45.710860+0000","last_clean":"2023-10-09T06:24:45.710860+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:24:45.710860+0000","last_undegraded":"2023-10-09T06:24:45.710860+0000","last_fullsized":"2023-10-09T06:24:45.710860+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":242,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.630809+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:24:36.630809+0000","last_peered":"2023-10-09T06:24:36.630809+0000","last_clean":"2023-10-09T06:24:36.630809+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:24:36.630809+0000","last_undegraded":"2023-10-09T06:24:36.630809+0000","last_fullsized":"2023-10-09T06:24:36.630809+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"280'9374","reported_seq":9424,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.927031+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:24:52.927031+0000","last_peered":"2023-10-09T06:24:52.927031+0000","last_clean":"2023-10-09T06:24:52.927031+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:24:52.927031+0000","last_undegraded":"2023-10-09T06:24:52.927031+0000","last_fullsized":"2023-10-09T06:24:52.927031+0000","mapping_epoch":238,"log_start":"248'4000","ondisk_log_start":"248'4000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228196352,"num_objects":3482,"num_object_clones":0,"num_object_copies":10446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9374,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":563,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758614+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:24:51.758614+0000","last_peered":"2023-10-09T06:24:51.758614+0000","last_clean":"2023-10-09T06:24:51.758614+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:24:51.758614+0000","last_undegraded":"2023-10-09T06:24:51.758614+0000","last_fullsized":"2023-10-09T06:24:51.758614+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":267,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637707+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:24:36.637707+0000","last_peered":"2023-10-09T06:24:36.637707+0000","last_clean":"2023-10-09T06:24:36.637707+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:24:36.637707+0000","last_undegraded":"2023-10-09T06:24:36.637707+0000","last_fullsized":"2023-10-09T06:24:36.637707+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":237,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:45.712069+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:24:45.712069+0000","last_peered":"2023-10-09T06:24:45.712069+0000","last_clean":"2023-10-09T06:24:45.712069+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:24:45.712069+0000","last_undegraded":"2023-10-09T06:24:45.712069+0000","last_fullsized":"2023-10-09T06:24:45.712069+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":257,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:43.800843+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:24:43.800843+0000","last_peered":"2023-10-09T06:24:43.800843+0000","last_clean":"2023-10-09T06:24:43.800843+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:24:43.800843+0000","last_undegraded":"2023-10-09T06:24:43.800843+0000","last_fullsized":"2023-10-09T06:24:43.800843+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"280'9538","reported_seq":9588,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.890529+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:24:52.890529+0000","last_peered":"2023-10-09T06:24:52.890529+0000","last_clean":"2023-10-09T06:24:52.890529+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:24:52.890529+0000","last_undegraded":"2023-10-09T06:24:52.890529+0000","last_fullsized":"2023-10-09T06:24:52.890529+0000","mapping_epoch":238,"log_start":"248'3500","ondisk_log_start":"248'3500","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6038,"log_dups_size":0,"ondisk_log_size":6038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235929600,"num_objects":3600,"num_object_clones":0,"num_object_copies":10800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":581,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637982+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:24:36.637982+0000","last_peered":"2023-10-09T06:24:36.637982+0000","last_clean":"2023-10-09T06:24:36.637982+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:24:36.637982+0000","last_undegraded":"2023-10-09T06:24:36.637982+0000","last_fullsized":"2023-10-09T06:24:36.637982+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":264,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.760687+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:24:51.760687+0000","last_peered":"2023-10-09T06:24:51.760687+0000","last_clean":"2023-10-09T06:24:51.760687+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:24:51.760687+0000","last_undegraded":"2023-10-09T06:24:51.760687+0000","last_fullsized":"2023-10-09T06:24:51.760687+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":234,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631885+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:24:36.631885+0000","last_peered":"2023-10-09T06:24:36.631885+0000","last_clean":"2023-10-09T06:24:36.631885+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:24:36.631885+0000","last_undegraded":"2023-10-09T06:24:36.631885+0000","last_fullsized":"2023-10-09T06:24:36.631885+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":233,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758400+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:24:51.758400+0000","last_peered":"2023-10-09T06:24:51.758400+0000","last_clean":"2023-10-09T06:24:51.758400+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:24:51.758400+0000","last_undegraded":"2023-10-09T06:24:51.758400+0000","last_fullsized":"2023-10-09T06:24:51.758400+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"278'8177","reported_seq":14118,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:24:51.785343+0000","last_change":"2023-10-09T06:23:40.105141+0000","last_active":"2023-10-09T06:24:51.785343+0000","last_peered":"2023-10-09T06:24:51.785343+0000","last_clean":"2023-10-09T06:23:37.701201+0000","last_became_active":"2023-10-09T06:23:40.099906+0000","last_became_peered":"2023-10-09T06:23:40.099906+0000","last_unstale":"2023-10-09T06:24:51.785343+0000","last_undegraded":"2023-10-09T06:23:39.992159+0000","last_fullsized":"2023-10-09T06:23:39.910881+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":8177,"log_dups_size":0,"ondisk_log_size":8177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116817920,"num_objects":1783,"num_object_clones":0,"num_object_copies":5349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2682,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":1783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4673,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[1,2],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1341},{"shards":"1,2","objects":442}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":236,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.630723+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:24:36.630723+0000","last_peered":"2023-10-09T06:24:36.630723+0000","last_clean":"2023-10-09T06:24:36.630723+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:24:36.630723+0000","last_undegraded":"2023-10-09T06:24:36.630723+0000","last_fullsized":"2023-10-09T06:24:36.630723+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":259,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.615922+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:24:36.615922+0000","last_peered":"2023-10-09T06:24:36.615922+0000","last_clean":"2023-10-09T06:24:36.615922+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:24:36.615922+0000","last_undegraded":"2023-10-09T06:24:36.615922+0000","last_fullsized":"2023-10-09T06:24:36.615922+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"278'7201","reported_seq":7259,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.366684+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:24:51.366684+0000","last_peered":"2023-10-09T06:24:51.366684+0000","last_clean":"2023-10-09T06:24:51.366684+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:24:51.366684+0000","last_undegraded":"2023-10-09T06:24:51.366684+0000","last_fullsized":"2023-10-09T06:24:51.366684+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7201,"log_dups_size":0,"ondisk_log_size":7201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122093568,"num_objects":1863,"num_object_clones":0,"num_object_copies":5589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4659,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":85,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758580+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:24:51.758580+0000","last_peered":"2023-10-09T06:24:51.758580+0000","last_clean":"2023-10-09T06:24:51.758580+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:24:51.758580+0000","last_undegraded":"2023-10-09T06:24:51.758580+0000","last_fullsized":"2023-10-09T06:24:51.758580+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":267,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.756579+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:24:51.756579+0000","last_peered":"2023-10-09T06:24:51.756579+0000","last_clean":"2023-10-09T06:24:51.756579+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:24:51.756579+0000","last_undegraded":"2023-10-09T06:24:51.756579+0000","last_fullsized":"2023-10-09T06:24:51.756579+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"278'7322","reported_seq":7380,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.237658+0000","last_change":"2023-10-09T06:23:26.834184+0000","last_active":"2023-10-09T06:24:50.237658+0000","last_peered":"2023-10-09T06:24:50.237658+0000","last_clean":"2023-10-09T06:24:50.237658+0000","last_became_active":"2023-10-09T06:23:26.832172+0000","last_became_peered":"2023-10-09T06:23:26.832172+0000","last_unstale":"2023-10-09T06:24:50.237658+0000","last_undegraded":"2023-10-09T06:24:50.237658+0000","last_fullsized":"2023-10-09T06:24:50.237658+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7322,"log_dups_size":0,"ondisk_log_size":7322,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123043840,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4720,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.758745+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:24:51.758745+0000","last_peered":"2023-10-09T06:24:51.758745+0000","last_clean":"2023-10-09T06:24:51.758745+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:24:51.758745+0000","last_undegraded":"2023-10-09T06:24:51.758745+0000","last_fullsized":"2023-10-09T06:24:51.758745+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":257,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.756396+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:24:51.756396+0000","last_peered":"2023-10-09T06:24:51.756396+0000","last_clean":"2023-10-09T06:24:51.756396+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:24:51.756396+0000","last_undegraded":"2023-10-09T06:24:51.756396+0000","last_fullsized":"2023-10-09T06:24:51.756396+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"280'7146","reported_seq":7206,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-10-09T06:24:52.768450+0000","last_change":"2023-10-09T06:23:26.832837+0000","last_active":"2023-10-09T06:24:52.768450+0000","last_peered":"2023-10-09T06:24:52.768450+0000","last_clean":"2023-10-09T06:24:52.768450+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:24:52.768450+0000","last_undegraded":"2023-10-09T06:24:52.768450+0000","last_fullsized":"2023-10-09T06:24:52.768450+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7146,"log_dups_size":0,"ondisk_log_size":7146,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115408896,"num_objects":1761,"num_object_clones":0,"num_object_copies":5283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4617,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":236,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:43.800982+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:24:43.800982+0000","last_peered":"2023-10-09T06:24:43.800982+0000","last_clean":"2023-10-09T06:24:43.800982+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:24:43.800982+0000","last_undegraded":"2023-10-09T06:24:43.800982+0000","last_fullsized":"2023-10-09T06:24:43.800982+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":250,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.637514+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:24:36.637514+0000","last_peered":"2023-10-09T06:24:36.637514+0000","last_clean":"2023-10-09T06:24:36.637514+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:24:36.637514+0000","last_undegraded":"2023-10-09T06:24:36.637514+0000","last_fullsized":"2023-10-09T06:24:36.637514+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"278'7163","reported_seq":7222,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T06:24:50.188863+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:24:50.188863+0000","last_peered":"2023-10-09T06:24:50.188863+0000","last_clean":"2023-10-09T06:24:50.188863+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:24:50.188863+0000","last_undegraded":"2023-10-09T06:24:50.188863+0000","last_fullsized":"2023-10-09T06:24:50.188863+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122847232,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4605,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T06:24:36.631601+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:24:36.631601+0000","last_peered":"2023-10-09T06:24:36.631601+0000","last_clean":"2023-10-09T06:24:36.631601+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:24:36.631601+0000","last_undegraded":"2023-10-09T06:24:36.631601+0000","last_fullsized":"2023-10-09T06:24:36.631601+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":240,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T06:24:51.756242+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:24:51.756242+0000","last_peered":"2023-10-09T06:24:51.756242+0000","last_clean":"2023-10-09T06:24:51.756242+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:24:51.756242+0000","last_undegraded":"2023-10-09T06:24:51.756242+0000","last_fullsized":"2023-10-09T06:24:51.756242+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":3816292376,"num_objects":58235,"num_object_clones":0,"num_object_copies":174705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5487,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":58235,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131311,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6978,"num_bytes_recovered":457310208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12348305408,"data_stored":12348293192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182956,"ondisk_log_size":182956,"up":78,"acting":76,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234330,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2138156,"kb_used_data":1237924,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":91606996,"statfs":{"total":95995035648,"available":93805563904,"internally_reserved":0,"allocated":1267634176,"data_stored":1266856123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:44 2023","interfaces":[{"interface":"back","average":{"1min":13.856,"5min":7.093,"15min":3.2669999999999999},"min":{"1min":0.23499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":152.28999999999999,"5min":219.41900000000001,"15min":219.41900000000001},"last":5.6710000000000003},{"interface":"front","average":{"1min":14.646000000000001,"5min":7.3979999999999997,"15min":3.351},"min":{"1min":0.27200000000000002,"5min":0.218,"15min":0.218},"max":{"1min":152.30000000000001,"5min":219.399,"15min":219.399},"last":4.79}]},{"osd":1,"last update":"Mon Oct 9 06:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":3.7959999999999998,"15min":2.6680000000000001},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.7720000000000002,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.881},{"interface":"front","average":{"1min":2.04,"5min":3.6400000000000001,"15min":2.4049999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":12.805999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":5.7629999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":17.384,"5min":10.077999999999999,"15min":4.5700000000000003},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":131.773,"5min":215.34800000000001,"15min":215.34800000000001},"last":2.996},{"interface":"front","average":{"1min":18.056999999999999,"5min":10.329000000000001,"15min":4.742},"min":{"1min":0.35599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":132.28100000000001,"5min":215.869,"15min":215.869},"last":8.6989999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.0209999999999999,"15min":1.4350000000000001},"min":{"1min":0.35699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.7080000000000002,"5min":50.966999999999999,"15min":50.966999999999999},"last":4.7030000000000003},{"interface":"front","average":{"1min":1.4359999999999999,"5min":3.1749999999999998,"15min":1.5069999999999999},"min":{"1min":0.38800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6959999999999997,"5min":51.133000000000003,"15min":51.133000000000003},"last":5.266}]},{"osd":4,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.6080000000000001,"15min":1.3260000000000001},"min":{"1min":0.27600000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6790000000000003,"5min":25.678999999999998,"15min":25.678999999999998},"last":5.7380000000000004},{"interface":"front","average":{"1min":1.103,"5min":1.425,"15min":1.117},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.681,"5min":13.305,"15min":13.305},"last":2.3180000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.391,"15min":1.6890000000000001},"min":{"1min":0.34300000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.863,"5min":50.304000000000002,"15min":50.304000000000002},"last":1.9159999999999999},{"interface":"front","average":{"1min":1.395,"5min":2.024,"15min":1.369},"min":{"1min":0.38100000000000001,"5min":0.34399999999999997,"15min":0.28299999999999997},"max":{"1min":5.242,"5min":48.774999999999999,"15min":48.774999999999999},"last":5.7130000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.5489999999999999,"15min":1.302},"min":{"1min":0.39300000000000002,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":7.8479999999999999,"5min":8.4890000000000008,"15min":11.878},"last":5.7889999999999997},{"interface":"front","average":{"1min":1.238,"5min":1.518,"15min":1.218},"min":{"1min":0.376,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":3.722,"5min":12.141999999999999,"15min":12.141999999999999},"last":2.347}]}]},{"osd":6,"up_from":46,"seq":197568495968,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2565612,"kb_used_data":1670692,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91179540,"statfs":{"total":95995035648,"available":93367848960,"internally_reserved":0,"allocated":1710788608,"data_stored":1710019132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:53 2023","interfaces":[{"interface":"back","average":{"1min":4.8220000000000001,"5min":6.0439999999999996,"15min":3.032},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.214},"max":{"1min":58.270000000000003,"5min":110.926,"15min":110.926},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.8010000000000002,"5min":6.5149999999999997,"15min":3.3180000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":58.253999999999998,"5min":110.911,"15min":110.911},"last":0.504}]},{"osd":1,"last update":"Mon Oct 9 06:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":4.9089999999999998,"15min":3.242},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.881,"5min":114.202,"15min":114.202},"last":1.5209999999999999},{"interface":"front","average":{"1min":1.4259999999999999,"5min":4.3840000000000003,"15min":3.0710000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.1790000000000003,"5min":114.179,"15min":114.179},"last":1.51}]},{"osd":2,"last update":"Mon Oct 9 06:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":3.484,"15min":2.6040000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.8419999999999996,"5min":75.191999999999993,"15min":75.191999999999993},"last":1.821},{"interface":"front","average":{"1min":2.4769999999999999,"5min":3.0939999999999999,"15min":2.5059999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.215999999999999,"5min":75.120000000000005,"15min":75.120000000000005},"last":1.8500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:23:53 2023","interfaces":[{"interface":"back","average":{"1min":2.121,"5min":4.0709999999999997,"15min":1.752},"min":{"1min":0.31900000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.1969999999999992,"5min":110.358,"15min":110.358},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.7430000000000001,"5min":4.2190000000000003,"15min":1.8129999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.7999999999999998,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.85399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4870000000000001,"15min":1.2549999999999999},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":3.6480000000000001,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.141,"5min":1.4990000000000001,"15min":1.256},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":6.8780000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.806,"15min":1.3460000000000001},"min":{"1min":0.39200000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":4.3620000000000001,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.597,"5min":1.6499999999999999,"15min":1.3740000000000001},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.2149999999999999,"5min":24.401,"15min":24.401},"last":0.435}]},{"osd":7,"last update":"Mon Oct 9 06:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":2.4630000000000001,"15min":1.482},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":10.755000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.476,"15min":1.6910000000000001},"min":{"1min":0.40300000000000002,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":100.995,"15min":100.995},"last":0.46500000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402704,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596124,"kb_used_data":1114772,"kb_used_omap":4,"kb_used_meta":481339,"kb_avail":92149028,"statfs":{"total":95995035648,"available":94360604672,"internally_reserved":0,"allocated":1141526528,"data_stored":1140293150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":492892046},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":20.606999999999999,"5min":7.0389999999999997,"15min":3.7789999999999999},"min":{"1min":0.255,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":145.96600000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":1.282},{"interface":"front","average":{"1min":22.132000000000001,"5min":7.8310000000000004,"15min":4.2510000000000003},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":145.982,"5min":149.02699999999999,"15min":149.02699999999999},"last":4.4980000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":15.206,"5min":5.3970000000000002,"15min":3.048},"min":{"1min":0.21299999999999999,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":139.75399999999999,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":16.838000000000001,"5min":6.3470000000000004,"15min":3.8319999999999999},"min":{"1min":0.24199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":140.29900000000001,"5min":140.29900000000001,"15min":140.29900000000001},"last":3.1459999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:24:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":5.5369999999999999,"15min":2.198},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.215},"max":{"1min":77.331999999999994,"5min":153.18199999999999,"15min":153.18199999999999},"last":3.4980000000000002},{"interface":"front","average":{"1min":7.5999999999999996,"5min":4.4409999999999998,"15min":1.8240000000000001},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":77.197000000000003,"5min":152.52000000000001,"15min":152.52000000000001},"last":1.014}]},{"osd":4,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":16.911999999999999,"5min":6.9329999999999998,"15min":3.8330000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":99.391000000000005,"5min":199.43799999999999,"15min":199.43799999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":16.670999999999999,"5min":6.5439999999999996,"15min":3.7650000000000001},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":99.462000000000003,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.58899999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":7.665,"5min":3.1459999999999999,"15min":2.7210000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":77.436999999999998,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":8.4740000000000002,"5min":3.7610000000000001,"15min":3.0059999999999998},"min":{"1min":0.371,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":77.388000000000005,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":9.5429999999999993,"5min":3.9060000000000001,"15min":3.0030000000000001},"min":{"1min":0.38700000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":77.292000000000002,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":8.7949999999999999,"5min":3.8450000000000002,"15min":2.7669999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.374},"max":{"1min":77.378,"5min":77.378,"15min":77.378},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":8.6129999999999995,"5min":3.6909999999999998,"15min":2.7799999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":77.456999999999994,"5min":77.456999999999994,"15min":77.456999999999994},"last":0.85199999999999998},{"interface":"front","average":{"1min":8.1199999999999992,"5min":3.117,"15min":2.2759999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":77.359999999999999,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.48899999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024137,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3123076,"kb_used_data":2453052,"kb_used_omap":5,"kb_used_meta":670010,"kb_avail":90622076,"statfs":{"total":95995035648,"available":92797005824,"internally_reserved":0,"allocated":2511925248,"data_stored":2510482433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":686090874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":4.673,"15min":2.8130000000000002},"min":{"1min":0.19500000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":13.916,"5min":185.17400000000001,"15min":185.17400000000001},"last":1.4059999999999999},{"interface":"front","average":{"1min":2.9609999999999999,"5min":4.7080000000000002,"15min":3.3500000000000001},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":13.945,"5min":186.691,"15min":186.691},"last":1.4890000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.2349999999999999,"15min":2.2599999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":13.211,"5min":13.211,"15min":17.588999999999999},"last":1.51},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.472,"15min":2.391},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":11.956,"5min":13.081,"15min":21.198},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:23:56 2023","interfaces":[{"interface":"back","average":{"1min":3.9329999999999998,"5min":6.8959999999999999,"15min":2.6309999999999998},"min":{"1min":0.24199999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.269,"5min":185.89400000000001,"15min":185.89400000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.073,"5min":6.9640000000000004,"15min":2.665},"min":{"1min":0.27500000000000002,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":8.2929999999999993,"5min":186.72900000000001,"15min":186.72900000000001},"last":0.93100000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":1.599,"15min":1.627},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9850000000000003,"5min":7.9660000000000002,"15min":15.835000000000001},"last":3.484},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.7829999999999999,"15min":2.0590000000000002},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.25700000000000001},"max":{"1min":5.1310000000000002,"5min":8.4890000000000008,"15min":14.792999999999999},"last":0.97399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.1699999999999999,"15min":2.036},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.0780000000000003,"5min":62.984999999999999,"15min":62.984999999999999},"last":3.444},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.8479999999999999,"15min":2.5920000000000001},"min":{"1min":0.35199999999999998,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":3.8610000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.42299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":2.3420000000000001,"15min":2.3759999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.4710000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":0.374},{"interface":"front","average":{"1min":1.669,"5min":1.7949999999999999,"15min":1.9430000000000001},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7439999999999998,"5min":10.606,"15min":15.893000000000001},"last":0.82299999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.516,"15min":2.3279999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28899999999999998},"max":{"1min":7.5010000000000003,"5min":62.262,"15min":62.262},"last":1.46},{"interface":"front","average":{"1min":2.3340000000000001,"5min":2.9740000000000002,"15min":2.645},"min":{"1min":0.377,"5min":0.377,"15min":0.314},"max":{"1min":14.167,"5min":62.229999999999997,"15min":62.229999999999997},"last":0.40000000000000002}]}]},{"osd":2,"up_from":128,"seq":549755814006,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2209860,"kb_used_data":1895740,"kb_used_omap":7,"kb_used_meta":314104,"kb_avail":91535292,"statfs":{"total":95995035648,"available":93732139008,"internally_reserved":0,"allocated":1941237760,"data_stored":1940024383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":321642577},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":3.802,"15min":3.4159999999999999},"min":{"1min":0.20799999999999999,"5min":0.157,"15min":0.157},"max":{"1min":10.356,"5min":110.611,"15min":163.40100000000001},"last":5.8520000000000003},{"interface":"front","average":{"1min":2.5099999999999998,"5min":3.968,"15min":3.8540000000000001},"min":{"1min":0.24099999999999999,"5min":0.182,"15min":0.182},"max":{"1min":9.0679999999999996,"5min":110.66500000000001,"15min":163.363},"last":6.0250000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:24:27 2023","interfaces":[{"interface":"back","average":{"1min":13.189,"5min":4.4189999999999996,"15min":3.573},"min":{"1min":0.26100000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":205.148,"5min":205.148,"15min":222.50299999999999},"last":2.6120000000000001},{"interface":"front","average":{"1min":12.093999999999999,"5min":4.5579999999999998,"15min":4.0209999999999999},"min":{"1min":0.316,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":205.267,"5min":205.267,"15min":222.19399999999999},"last":2.98}]},{"osd":3,"last update":"Mon Oct 9 06:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":3.3100000000000001,"15min":1.482},"min":{"1min":0.28999999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.9300000000000002,"5min":81.183999999999997,"15min":81.183999999999997},"last":3.085},{"interface":"front","average":{"1min":1.829,"5min":3.5139999999999998,"15min":1.5249999999999999},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.6790000000000003,"5min":81.061999999999998,"15min":81.061999999999998},"last":2.9039999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":7.7530000000000001,"5min":2.7330000000000001,"15min":2.1339999999999999},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":90.938999999999993,"5min":90.938999999999993,"15min":90.938999999999993},"last":2.9289999999999998},{"interface":"front","average":{"1min":7.2480000000000002,"5min":3.1110000000000002,"15min":2.4169999999999998},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":90.950000000000003,"5min":90.950000000000003,"15min":90.950000000000003},"last":2.8069999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":1.5409999999999999,"15min":1.579},"min":{"1min":0.41499999999999998,"5min":0.317,"15min":0.309},"max":{"1min":8.9589999999999996,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":3.052},{"interface":"front","average":{"1min":1.675,"5min":2.0259999999999998,"15min":2.1899999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.4320000000000004,"5min":10.548,"15min":10.548},"last":2.8900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":9.2420000000000009,"5min":3.2189999999999999,"15min":2.4630000000000001},"min":{"1min":0.36199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":138.291,"5min":138.291,"15min":138.291},"last":2.863},{"interface":"front","average":{"1min":8.9420000000000002,"5min":2.9399999999999999,"15min":1.9990000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":138.01599999999999,"5min":138.01599999999999,"15min":138.01599999999999},"last":5.7789999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.7949999999999999,"15min":1.651},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":5.8220000000000001},{"interface":"front","average":{"1min":2.73,"5min":1.7809999999999999,"15min":2.3399999999999999},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.9960000000000004,"5min":8.9960000000000004,"15min":9.3249999999999993},"last":2.9580000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127203,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656424,"kb_used_data":1278112,"kb_used_omap":2,"kb_used_meta":378237,"kb_avail":92088728,"statfs":{"total":95995035648,"available":94298857472,"internally_reserved":0,"allocated":1308786688,"data_stored":1307658162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":387314741},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":7.609,"5min":6.7220000000000004,"15min":2.677},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":63.384999999999998,"5min":187.852,"15min":187.852},"last":6.9859999999999998},{"interface":"front","average":{"1min":6.742,"5min":7.6870000000000003,"15min":3.2240000000000002},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":63.319000000000003,"5min":187.923,"15min":187.923},"last":2.7469999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":6.048,"15min":2.7069999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.223},"max":{"1min":16.193999999999999,"5min":187.89099999999999,"15min":187.89099999999999},"last":2.7839999999999998},{"interface":"front","average":{"1min":3.145,"5min":3.5939999999999999,"15min":1.6359999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":23.146000000000001,"5min":67.028000000000006,"15min":67.028000000000006},"last":4.0510000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0350000000000001,"5min":5.6349999999999998,"15min":2.5190000000000001},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":39.951999999999998,"5min":187.76400000000001,"15min":187.76400000000001},"last":2.6150000000000002},{"interface":"front","average":{"1min":3.8540000000000001,"5min":4.5599999999999996,"15min":1.8200000000000001},"min":{"1min":0.26200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":25.530999999999999,"5min":187.79499999999999,"15min":187.79499999999999},"last":4.2140000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.7,"15min":0.999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27600000000000002},"max":{"1min":22.585999999999999,"5min":22.585999999999999,"15min":22.585999999999999},"last":4.117},{"interface":"front","average":{"1min":2.0779999999999998,"5min":3.464,"15min":1.946},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":15.942,"5min":187.67400000000001,"15min":187.67400000000001},"last":1.044}]},{"osd":5,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":3.867,"15min":2.109},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.29899999999999999},"max":{"1min":16.001999999999999,"5min":187.833,"15min":187.833},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.3740000000000001,"5min":3.7919999999999998,"15min":2.0779999999999998},"min":{"1min":0.38200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":16.167999999999999,"5min":187.952,"15min":187.952},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":3.7530000000000001,"15min":2.0569999999999999},"min":{"1min":0.36199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.976000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":1.014},{"interface":"front","average":{"1min":1.6719999999999999,"5min":3.5899999999999999,"15min":1.9910000000000001},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":16.134,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.40799999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:23:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":3.8050000000000002,"15min":1.7130000000000001},"min":{"1min":0.42999999999999999,"5min":0.35599999999999998,"15min":0.34100000000000003},"max":{"1min":22.533000000000001,"5min":192.20400000000001,"15min":192.20400000000001},"last":4.0899999999999999},{"interface":"front","average":{"1min":2.476,"5min":3.6499999999999999,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":15.901999999999999,"5min":187.86500000000001,"15min":187.86500000000001},"last":1.071}]}]},{"osd":4,"up_from":30,"seq":128849019240,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3384748,"kb_used_data":2497956,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":90360404,"statfs":{"total":95995035648,"available":92529053696,"internally_reserved":0,"allocated":2557906944,"data_stored":2557136747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":3.8069999999999999,"15min":2.3140000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.18,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.69499999999999995},{"interface":"front","average":{"1min":5.444,"5min":4.2999999999999998,"15min":2.706},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":48.287999999999997,"5min":76.988,"15min":76.988},"last":0.70399999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":2.3130000000000002,"15min":2.093},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.2370000000000001,"5min":37.383000000000003,"15min":91.849999999999994},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.599,"5min":2.6379999999999999,"15min":2.1899999999999999},"min":{"1min":0.26000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.4199999999999999,"5min":37.436999999999998,"15min":91.885000000000005},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":2.2839999999999998,"15min":1.8919999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":4.4829999999999997,"5min":69.640000000000001,"15min":69.640000000000001},"last":4.2720000000000002},{"interface":"front","average":{"1min":1.657,"5min":2.5329999999999999,"15min":2.0630000000000002},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":5.3079999999999998,"5min":69.676000000000002,"15min":69.676000000000002},"last":4.6639999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:23:56 2023","interfaces":[{"interface":"back","average":{"1min":3.9119999999999999,"5min":4.1639999999999997,"15min":2.1000000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":45.591000000000001,"5min":57.826999999999998,"15min":91.697000000000003},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.3370000000000002,"5min":3.718,"15min":2.0049999999999999},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":45.548999999999999,"5min":57.853000000000002,"15min":94.356999999999999},"last":0.35899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.268,"15min":1.0620000000000001},"min":{"1min":0.29799999999999999,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":4.4400000000000004,"5min":10.673,"15min":10.673},"last":4.2060000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.202,"15min":1.2050000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":2.9990000000000001,"5min":12.552,"15min":12.552},"last":4.6189999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.77400000000000002,"15min":0.83499999999999996},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.8740000000000001,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.212,"15min":1.244},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5419999999999998,"5min":15.321999999999999,"15min":15.321999999999999},"last":8.859}]},{"osd":7,"last update":"Mon Oct 9 06:24:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":2.444,"15min":1.46},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":4.516,"5min":62.658000000000001,"15min":62.658000000000001},"last":5.4480000000000004},{"interface":"front","average":{"1min":0.79500000000000004,"5min":2.2850000000000001,"15min":1.464},"min":{"1min":0.372,"5min":0.311,"15min":0.311},"max":{"1min":3.407,"5min":62.573,"15min":62.573},"last":0.34399999999999997}]}]},{"osd":5,"up_from":0,"seq":163208757604,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2481131520,"data_stored":2481127448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1131282432,"data_stored":1131282432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1937244160,"data_stored":1937244160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1295974400,"data_stored":1295974400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2544570368,"data_stored":2544566296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1703415808,"data_stored":1703411736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1254686720,"data_stored":1254686720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:24:57.512 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:24:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:57.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:57.614 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:57.671 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:57.727 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:57.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:57.832 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:24:57.881 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:24:57.924 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:57.951 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:24:57.986 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:24:58.028 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:24:58.045 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:24:58.050 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5487/174129 objects degraded (3.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:58.051 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5487/174129 objects degraded (3.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:58.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:24:58.064 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:24:58.065 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:24:58.084 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:24:58.152 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:24:58.153 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:24:58.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5487/174129 objects degraded (3.151%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:24:58.190 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:24:58.191 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:58.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:24:58.248 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.249 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:24:58.265 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:24:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:58.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:24:58.374 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:24:58.419 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:58.494 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:24:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:58.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:24:58.606 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:58.704 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:24:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:58 smithi107 ceph-mon[212750]: pgmap v856: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 3830/168459 objects degraded (2.274%); 2370/168459 objects misplaced (1.407%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:24:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2015145560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:59.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:58 smithi150 ceph-mon[194178]: pgmap v856: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 3830/168459 objects degraded (2.274%); 2370/168459 objects misplaced (1.407%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:24:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2015145560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:59.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:58 smithi107 ceph-mon[220921]: pgmap v856: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 3830/168459 objects degraded (2.274%); 2370/168459 objects misplaced (1.407%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:24:59.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2015145560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:24:59.736 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:24:59.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:24:59.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:24:59.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:24:59.853 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:24:59.912 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:24:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:00.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:24:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:00.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:24:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:00.353 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:25:00.831 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:00.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: pgmap v857: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 3830/161403 objects degraded (2.373%); 2370/161403 objects misplaced (1.468%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:25:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[212750]: failed to encode map e281 with expected crc 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: pgmap v857: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 3830/161403 objects degraded (2.373%); 2370/161403 objects misplaced (1.468%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: failed to encode map e281 with expected crc 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T06:25:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:00 smithi107 ceph-mon[220921]: failed to encode map e281 with expected crc 2023-10-09T06:25:00.943 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:00.969 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:01.011 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:01.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:01.117 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: pgmap v857: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 894 op/s; 3830/161403 objects degraded (2.373%); 2370/161403 objects misplaced (1.468%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: failed to encode map e281 with expected crc 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1072597242' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: osdmap e281: 8 total, 8 up, 7 in 2023-10-09T06:25:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:00 smithi150 ceph-mon[194178]: failed to encode map e281 with expected crc 2023-10-09T06:25:01.189 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:01.503 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:01.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:01.560 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:01.615 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:01.629 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:01.716 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:01.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:25:01.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:01.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:01.915 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:01.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:01.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:02.020 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:02.061 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:02.103 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:02] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T06:25:02.104 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:25:02.105 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:02.105 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:02.131 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:02.139 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:01 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:25:02.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:02.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:02.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:02.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:02.242 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:02.324 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:02 smithi107 ceph-mon[212750]: pgmap v859: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 3830/158535 objects degraded (2.416%); 2046/158535 objects misplaced (1.291%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:25:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3830/158535 objects degraded (2.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:02 smithi107 ceph-mon[220921]: pgmap v859: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 3830/158535 objects degraded (2.416%); 2046/158535 objects misplaced (1.291%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:25:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3830/158535 objects degraded (2.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:03.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:02 smithi150 ceph-mon[194178]: pgmap v859: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 3830/158535 objects degraded (2.416%); 2046/158535 objects misplaced (1.291%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:25:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3830/158535 objects degraded (2.416%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:03.216 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:25:03.468 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:25:03.679 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:03.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:03.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:03.789 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:03.856 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:04.572 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:04.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:04.711 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:04.758 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:04.821 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:05.011 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:05.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:05.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:04 smithi107 ceph-mon[212750]: pgmap v860: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 847 op/s; 3562/149352 objects degraded (2.385%); 1912/149352 objects misplaced (1.280%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:05.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:04 smithi107 ceph-mon[220921]: pgmap v860: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 847 op/s; 3562/149352 objects degraded (2.385%); 1912/149352 objects misplaced (1.280%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:05.130 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:05.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:04 smithi150 ceph-mon[194178]: pgmap v860: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 847 op/s; 3562/149352 objects degraded (2.385%); 1912/149352 objects misplaced (1.280%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:05.212 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:05.341 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:05.370 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:05.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:05.479 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:05.567 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:05.922 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:05.922 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:06.039 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:06.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:06.157 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:06.195 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:06.263 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:06.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:06.303 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:25:06.303 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 3562/148599 objects degraded (2.397%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1119,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":281,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":7,"osd_in_since":1696832504,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":107},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":49533,"data_bytes":3245021112,"bytes_used":15383465984,"bytes_avail":656581783552,"bytes_total":671965249536,"degraded_objects":3562,"degraded_total":148599,"degraded_ratio":0.02397055161878613,"misplaced_objects":1912,"misplaced_total":148599,"misplaced_ratio":0.012866842980100808,"recovering_objects_per_sec":85,"recovering_bytes_per_sec":5595171,"recovering_keys_per_sec":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":850},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T06:22:27.038508+0000","services":{}},"progress_events":{"be02b243-5ad1-4e62-ae2b-ff1425dec846":{"message":"Global Recovery Event (80s)\n [===========================.] (remaining: 1s)","progress":0.98165136575698853,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:25:06.371 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:06.414 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:06.472 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:06.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:06.586 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:06.639 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:06 smithi107 ceph-mon[212750]: pgmap v861: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 3562/148599 objects degraded (2.397%); 1912/148599 objects misplaced (1.287%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:06.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2445903967' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:06.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:06 smithi107 ceph-mon[220921]: pgmap v861: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 3562/148599 objects degraded (2.397%); 1912/148599 objects misplaced (1.287%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:06.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2445903967' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:06 smithi150 ceph-mon[194178]: pgmap v861: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 3562/148599 objects degraded (2.397%); 1912/148599 objects misplaced (1.287%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2445903967' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:07.882 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3562/148599 objects degraded (2.397%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:08.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3562/148599 objects degraded (2.397%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:08.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3562/148599 objects degraded (2.397%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:08.978 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:25:09.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:08 smithi107 ceph-mon[212750]: pgmap v862: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 3280/142494 objects degraded (2.302%); 1451/142494 objects misplaced (1.018%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:08 smithi107 ceph-mon[212750]: 10.10 deep-scrub starts 2023-10-09T06:25:09.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:08 smithi107 ceph-mon[220921]: pgmap v862: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 3280/142494 objects degraded (2.302%); 1451/142494 objects misplaced (1.018%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:09.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:08 smithi107 ceph-mon[220921]: 10.10 deep-scrub starts 2023-10-09T06:25:09.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:08 smithi150 ceph-mon[194178]: pgmap v862: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 865 op/s; 3280/142494 objects degraded (2.302%); 1451/142494 objects misplaced (1.018%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:08 smithi150 ceph-mon[194178]: 10.10 deep-scrub starts 2023-10-09T06:25:09.725 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:25:09.726 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:25:09.899 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:09.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:09.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:09 smithi107 ceph-mon[212750]: 10.10 scrub starts 2023-10-09T06:25:09.972 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:09 smithi107 ceph-mon[220921]: 10.10 scrub starts 2023-10-09T06:25:10.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:10.106 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:09 smithi150 ceph-mon[194178]: 10.10 scrub starts 2023-10-09T06:25:10.279 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:10.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:10.390 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:10.477 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:10.594 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:10.642 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:25:10.695 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:25:10.758 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:10.769 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:25:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:10.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:10.825 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:25:10.864 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:25:10.872 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:10.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:10 smithi107 ceph-mon[212750]: pgmap v863: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 3280/136182 objects degraded (2.409%); 1451/136182 objects misplaced (1.065%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:10.911 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/461146124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:10.916 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:25:10.960 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:10.993 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:25:11.035 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:25:11.067 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.067 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.067 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.067 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.067 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.068 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.068 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.068 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:25:11.097 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:11.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:11.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:11.184 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:10 smithi107 ceph-mon[220921]: pgmap v863: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 3280/136182 objects degraded (2.409%); 1451/136182 objects misplaced (1.065%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:11.185 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/461146124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:11.206 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:11.282 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:10 smithi150 ceph-mon[194178]: pgmap v863: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 3280/136182 objects degraded (2.409%); 1451/136182 objects misplaced (1.065%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:25:11.282 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/461146124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:11.289 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:11.337 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:11.366 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:11.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:11.471 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:11.519 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:11.588 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:11.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:11.702 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:11.784 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:11.967 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:25:12.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[212750]: pgmap v864: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 3280/133440 objects degraded (2.458%); 1072/133440 objects misplaced (0.803%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:25:12.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/978305869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:12.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:12.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[220921]: pgmap v864: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 3280/133440 objects degraded (2.458%); 1072/133440 objects misplaced (0.803%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:25:12.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/978305869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:12.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:12.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:12] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-10-09T06:25:12.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:11 smithi150 ceph-mon[194178]: pgmap v864: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 826 op/s; 3280/133440 objects degraded (2.458%); 1072/133440 objects misplaced (0.803%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T06:25:12.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/978305869' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:12.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:25:13.279 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:13.286 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:13.286 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.286 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:13.286 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T06:25:13.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3280/133440 objects degraded (2.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:13.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[220921]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:13.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:13.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:13.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:13.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T06:25:13.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.289 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3280/133440 objects degraded (2.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:13.289 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:12 smithi107 ceph-mon[212750]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.388 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:25:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: osdmap e282: 8 total, 8 up, 7 in 2023-10-09T06:25:13.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3280/133440 objects degraded (2.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:13.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:12 smithi150 ceph-mon[194178]: failed to encode map e282 with expected crc 2023-10-09T06:25:13.442 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:13.526 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:13.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:13.636 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:13.684 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:13.729 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:13.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:13.834 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:13.882 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:13.943 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:13.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:14.051 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:14.094 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:13 smithi150 ceph-mon[194178]: pgmap v866: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/124458 objects degraded (2.457%); 961/124458 objects misplaced (0.772%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:14.106 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:14.183 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:14.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:14.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:13 smithi107 ceph-mon[220921]: pgmap v866: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/124458 objects degraded (2.457%); 961/124458 objects misplaced (0.772%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:14.262 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:13 smithi107 ceph-mon[212750]: pgmap v866: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/124458 objects degraded (2.457%); 961/124458 objects misplaced (0.772%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:14.293 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:14.339 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:14.425 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:14.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:14.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:14.618 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:15.391 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:25:15.391 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:25:15.391 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:25:15.391 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:25:15.392 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:25:15.427 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:15.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:15.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:15.537 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:15.573 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:15.883 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:15.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:15.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:25:15.997 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:16.033 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:16.091 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:16.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:16.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:16.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:16.282 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:16.296 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:25:16.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:16 smithi150 ceph-mon[194178]: pgmap v867: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/123699 objects degraded (2.472%); 961/123699 objects misplaced (0.777%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:16.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:16 smithi107 ceph-mon[212750]: pgmap v867: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/123699 objects degraded (2.472%); 961/123699 objects misplaced (0.777%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:16.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:16 smithi107 ceph-mon[220921]: pgmap v867: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 3058/123699 objects degraded (2.472%); 961/123699 objects misplaced (0.777%); 5.9 MiB/s, 95 objects/s recovering 2023-10-09T06:25:16.723 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:16.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:16.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:16.910 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:16.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:25:17.085 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:17.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:17.192 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:17.255 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:17.297 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:17.495 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:17.523 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:17.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:17.638 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:17.673 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:17.754 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:17.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:17.762 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:25:17.782 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:25:17.815 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:17.815 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:17.815 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3058/123699 objects degraded (2.472%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:17.815 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:17.816 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:17.816 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3058/123699 objects degraded (2.472%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:17.859 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:17.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:17.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:17.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3058/123699 objects degraded (2.472%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:17.907 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:17.971 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:17.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:18.083 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:18.173 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:18.651 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:18.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:18.662 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:18 smithi107 ceph-mon[220921]: pgmap v868: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2464/117378 objects degraded (2.099%); 701/117378 objects misplaced (0.597%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:18.663 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:18 smithi107 ceph-mon[220921]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:18.663 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:18 smithi107 ceph-mon[212750]: pgmap v868: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2464/117378 objects degraded (2.099%); 701/117378 objects misplaced (0.597%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:18.664 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:18 smithi107 ceph-mon[212750]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:18.763 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:18.789 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:18.831 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:18.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:18 smithi150 ceph-mon[194178]: pgmap v868: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 836 op/s; 2464/117378 objects degraded (2.099%); 701/117378 objects misplaced (0.597%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:18 smithi150 ceph-mon[194178]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:18.936 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:18.964 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:20.609 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:20 smithi107 ceph-mon[212750]: pgmap v869: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 2464/110577 objects degraded (2.228%); 701/110577 objects misplaced (0.634%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:20.619 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:20.619 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:20.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:20.641 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":868,"stamp":"2023-10-09T06:25:17.107597+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2562987960,"num_objects":39126,"num_object_clones":0,"num_object_copies":117378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2464,"num_objects_misplaced":701,"num_objects_unfound":0,"num_objects_dirty":39126,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150505,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9389,"num_bytes_recovered":592980058,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":196316,"ondisk_log_size":196316,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13085908,"kb_used_data":8525468,"kb_used_omap":20,"kb_used_meta":4560043,"kb_avail":643130156,"statfs":{"total":671965249536,"available":658565279744,"internally_reserved":0,"allocated":8730079232,"data_stored":8722694854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4669484686},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,4,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-548667392,"num_objects":-8372,"num_object_clones":0,"num_object_copies":-25116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-816,"num_objects_misplaced":-750,"num_objects_unfound":0,"num_objects_dirty":-8372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8372,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1068,"num_bytes_recovered":49152000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002915"},"pg_stats":[{"pgid":"10.13","version":"282'7914","reported_seq":16419,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.783855+0000","last_change":"2023-10-09T06:24:52.774608+0000","last_active":"2023-10-09T06:25:16.783855+0000","last_peered":"2023-10-09T06:25:16.783855+0000","last_clean":"2023-10-09T06:25:16.783855+0000","last_became_active":"2023-10-09T06:24:52.762870+0000","last_became_peered":"2023-10-09T06:24:52.762870+0000","last_unstale":"2023-10-09T06:25:16.783855+0000","last_undegraded":"2023-10-09T06:25:16.783855+0000","last_fullsized":"2023-10-09T06:25:16.783855+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78774272,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2810,"num_bytes_recovered":184156160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.834705+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:24:59.834705+0000","last_peered":"2023-10-09T06:24:59.834705+0000","last_clean":"2023-10-09T06:24:59.834705+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:24:59.834705+0000","last_undegraded":"2023-10-09T06:24:59.834705+0000","last_fullsized":"2023-10-09T06:24:59.834705+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":219,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.865626+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:24:59.865626+0000","last_peered":"2023-10-09T06:24:59.865626+0000","last_clean":"2023-10-09T06:24:59.865626+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:24:59.865626+0000","last_undegraded":"2023-10-09T06:24:59.865626+0000","last_fullsized":"2023-10-09T06:24:59.865626+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"282'7797","reported_seq":13371,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.982845+0000","last_change":"2023-10-09T06:25:08.543126+0000","last_active":"2023-10-09T06:25:12.982845+0000","last_peered":"2023-10-09T06:25:12.982845+0000","last_clean":"2023-10-09T06:25:12.982845+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:25:12.982845+0000","last_undegraded":"2023-10-09T06:25:12.982845+0000","last_fullsized":"2023-10-09T06:25:12.982845+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7797,"log_dups_size":0,"ondisk_log_size":7797,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5239,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":254,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:06.889357+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:25:06.889357+0000","last_peered":"2023-10-09T06:25:06.889357+0000","last_clean":"2023-10-09T06:25:06.889357+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:25:06.889357+0000","last_undegraded":"2023-10-09T06:25:06.889357+0000","last_fullsized":"2023-10-09T06:25:06.889357+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.837954+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:24:59.837954+0000","last_peered":"2023-10-09T06:24:59.837954+0000","last_clean":"2023-10-09T06:24:59.837954+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:24:59.837954+0000","last_undegraded":"2023-10-09T06:24:59.837954+0000","last_fullsized":"2023-10-09T06:24:59.837954+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"282'7953","reported_seq":11506,"reported_epoch":282,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:25:15.305560+0000","last_change":"2023-10-09T06:24:51.087762+0000","last_active":"2023-10-09T06:25:15.305560+0000","last_peered":"2023-10-09T06:25:15.305560+0000","last_clean":"2023-10-09T06:23:34.462031+0000","last_became_active":"2023-10-09T06:23:38.083231+0000","last_became_peered":"2023-10-09T06:23:38.083231+0000","last_unstale":"2023-10-09T06:25:15.305560+0000","last_undegraded":"2023-10-09T06:23:38.014323+0000","last_fullsized":"2023-10-09T06:23:37.903678+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79233024,"num_objects":1209,"num_object_clones":0,"num_object_copies":3627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5335,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1743,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,7","objects":147}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.852183+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:24:59.852183+0000","last_peered":"2023-10-09T06:24:59.852183+0000","last_clean":"2023-10-09T06:24:59.852183+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:24:59.852183+0000","last_undegraded":"2023-10-09T06:24:59.852183+0000","last_fullsized":"2023-10-09T06:24:59.852183+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838153+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:24:59.838153+0000","last_peered":"2023-10-09T06:24:59.838153+0000","last_clean":"2023-10-09T06:24:59.838153+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:24:59.838153+0000","last_undegraded":"2023-10-09T06:24:59.838153+0000","last_fullsized":"2023-10-09T06:24:59.838153+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":243,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:10.316895+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:25:10.316895+0000","last_peered":"2023-10-09T06:25:10.316895+0000","last_clean":"2023-10-09T06:25:10.316895+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:25:10.316895+0000","last_undegraded":"2023-10-09T06:25:10.316895+0000","last_fullsized":"2023-10-09T06:25:10.316895+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":219,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.852112+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:24:59.852112+0000","last_peered":"2023-10-09T06:24:59.852112+0000","last_clean":"2023-10-09T06:24:59.852112+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:24:59.852112+0000","last_undegraded":"2023-10-09T06:24:59.852112+0000","last_fullsized":"2023-10-09T06:24:59.852112+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":210,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840555+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:24:59.840555+0000","last_peered":"2023-10-09T06:24:59.840555+0000","last_clean":"2023-10-09T06:24:59.840555+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:24:59.840555+0000","last_undegraded":"2023-10-09T06:24:59.840555+0000","last_fullsized":"2023-10-09T06:24:59.840555+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":217,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.850748+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:24:59.850748+0000","last_peered":"2023-10-09T06:24:59.850748+0000","last_clean":"2023-10-09T06:24:59.850748+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:24:59.850748+0000","last_undegraded":"2023-10-09T06:24:59.850748+0000","last_fullsized":"2023-10-09T06:24:59.850748+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":131,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840435+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:24:59.840435+0000","last_peered":"2023-10-09T06:24:59.840435+0000","last_clean":"2023-10-09T06:24:59.840435+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:24:59.840435+0000","last_undegraded":"2023-10-09T06:24:59.840435+0000","last_fullsized":"2023-10-09T06:24:59.840435+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":213,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.839967+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:24:59.839967+0000","last_peered":"2023-10-09T06:24:59.839967+0000","last_clean":"2023-10-09T06:24:59.839967+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:24:59.839967+0000","last_undegraded":"2023-10-09T06:24:59.839967+0000","last_fullsized":"2023-10-09T06:24:59.839967+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":288,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835838+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:24:59.835838+0000","last_peered":"2023-10-09T06:24:59.835838+0000","last_clean":"2023-10-09T06:24:59.835838+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:24:59.835838+0000","last_undegraded":"2023-10-09T06:24:59.835838+0000","last_fullsized":"2023-10-09T06:24:59.835838+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835215+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:24:59.835215+0000","last_peered":"2023-10-09T06:24:59.835215+0000","last_clean":"2023-10-09T06:24:59.835215+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:24:59.835215+0000","last_undegraded":"2023-10-09T06:24:59.835215+0000","last_fullsized":"2023-10-09T06:24:59.835215+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":131,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840351+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:24:59.840351+0000","last_peered":"2023-10-09T06:24:59.840351+0000","last_clean":"2023-10-09T06:24:59.840351+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:24:59.840351+0000","last_undegraded":"2023-10-09T06:24:59.840351+0000","last_fullsized":"2023-10-09T06:24:59.840351+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.852210+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:24:59.852210+0000","last_peered":"2023-10-09T06:24:59.852210+0000","last_clean":"2023-10-09T06:24:59.852210+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:24:59.852210+0000","last_undegraded":"2023-10-09T06:24:59.852210+0000","last_fullsized":"2023-10-09T06:24:59.852210+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840977+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:24:59.840977+0000","last_peered":"2023-10-09T06:24:59.840977+0000","last_clean":"2023-10-09T06:24:59.840977+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:24:59.840977+0000","last_undegraded":"2023-10-09T06:24:59.840977+0000","last_fullsized":"2023-10-09T06:24:59.840977+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":209,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840332+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:24:59.840332+0000","last_peered":"2023-10-09T06:24:59.840332+0000","last_clean":"2023-10-09T06:24:59.840332+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:24:59.840332+0000","last_undegraded":"2023-10-09T06:24:59.840332+0000","last_fullsized":"2023-10-09T06:24:59.840332+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"282'7827","reported_seq":7889,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.128354+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:25:15.128354+0000","last_peered":"2023-10-09T06:25:15.128354+0000","last_clean":"2023-10-09T06:25:15.128354+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:25:15.128354+0000","last_undegraded":"2023-10-09T06:25:15.128354+0000","last_fullsized":"2023-10-09T06:25:15.128354+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7827,"log_dups_size":0,"ondisk_log_size":7827,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76709888,"num_objects":1171,"num_object_clones":0,"num_object_copies":3513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5275,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840133+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:24:59.840133+0000","last_peered":"2023-10-09T06:24:59.840133+0000","last_clean":"2023-10-09T06:24:59.840133+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:24:59.840133+0000","last_undegraded":"2023-10-09T06:24:59.840133+0000","last_fullsized":"2023-10-09T06:24:59.840133+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:10.316868+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:25:10.316868+0000","last_peered":"2023-10-09T06:25:10.316868+0000","last_clean":"2023-10-09T06:25:10.316868+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:25:10.316868+0000","last_undegraded":"2023-10-09T06:25:10.316868+0000","last_fullsized":"2023-10-09T06:25:10.316868+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"282'7905","reported_seq":7968,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.227335+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:25:16.227335+0000","last_peered":"2023-10-09T06:25:16.227335+0000","last_clean":"2023-10-09T06:25:16.227335+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:25:16.227335+0000","last_undegraded":"2023-10-09T06:25:16.227335+0000","last_fullsized":"2023-10-09T06:25:16.227335+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7905,"log_dups_size":0,"ondisk_log_size":7905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76513280,"num_objects":1168,"num_object_clones":0,"num_object_copies":3504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5340,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":270,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.839264+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:24:59.839264+0000","last_peered":"2023-10-09T06:24:59.839264+0000","last_clean":"2023-10-09T06:24:59.839264+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:24:59.839264+0000","last_undegraded":"2023-10-09T06:24:59.839264+0000","last_fullsized":"2023-10-09T06:24:59.839264+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.864943+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:24:59.864943+0000","last_peered":"2023-10-09T06:24:59.864943+0000","last_clean":"2023-10-09T06:24:59.864943+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:24:59.864943+0000","last_undegraded":"2023-10-09T06:24:59.864943+0000","last_fullsized":"2023-10-09T06:24:59.864943+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"282'7913","reported_seq":8045,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.239785+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:25:15.239785+0000","last_peered":"2023-10-09T06:25:15.239785+0000","last_clean":"2023-10-09T06:25:15.239785+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:25:15.239785+0000","last_undegraded":"2023-10-09T06:25:15.239785+0000","last_fullsized":"2023-10-09T06:25:15.239785+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":7913,"log_dups_size":0,"ondisk_log_size":7913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78249984,"num_objects":1194,"num_object_clones":0,"num_object_copies":3582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2651,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":143,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840528+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:24:59.840528+0000","last_peered":"2023-10-09T06:24:59.840528+0000","last_clean":"2023-10-09T06:24:59.840528+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:24:59.840528+0000","last_undegraded":"2023-10-09T06:24:59.840528+0000","last_fullsized":"2023-10-09T06:24:59.840528+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.841262+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:24:59.841262+0000","last_peered":"2023-10-09T06:24:59.841262+0000","last_clean":"2023-10-09T06:24:59.841262+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:24:59.841262+0000","last_undegraded":"2023-10-09T06:24:59.841262+0000","last_fullsized":"2023-10-09T06:24:59.841262+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"282'9584","reported_seq":9763,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.742800+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:25:12.742800+0000","last_peered":"2023-10-09T06:25:12.742800+0000","last_clean":"2023-10-09T06:25:12.742800+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:25:12.742800+0000","last_undegraded":"2023-10-09T06:25:12.742800+0000","last_fullsized":"2023-10-09T06:25:12.742800+0000","mapping_epoch":238,"log_start":"244'1992","ondisk_log_start":"244'1992","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":7592,"log_dups_size":0,"ondisk_log_size":7592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79560704,"num_objects":1214,"num_object_clones":0,"num_object_copies":3642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2482,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":129,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840479+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:24:59.840479+0000","last_peered":"2023-10-09T06:24:59.840479+0000","last_clean":"2023-10-09T06:24:59.840479+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:24:59.840479+0000","last_undegraded":"2023-10-09T06:24:59.840479+0000","last_fullsized":"2023-10-09T06:24:59.840479+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":259,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835261+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:24:59.835261+0000","last_peered":"2023-10-09T06:24:59.835261+0000","last_clean":"2023-10-09T06:24:59.835261+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:24:59.835261+0000","last_undegraded":"2023-10-09T06:24:59.835261+0000","last_fullsized":"2023-10-09T06:24:59.835261+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"282'8051","reported_seq":8115,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.265884+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:25:15.265884+0000","last_peered":"2023-10-09T06:25:15.265884+0000","last_clean":"2023-10-09T06:25:15.265884+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:25:15.265884+0000","last_undegraded":"2023-10-09T06:25:15.265884+0000","last_fullsized":"2023-10-09T06:25:15.265884+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8051,"log_dups_size":0,"ondisk_log_size":8051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72122368,"num_objects":1100,"num_object_clones":0,"num_object_copies":3300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5448,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":143,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840351+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:24:59.840351+0000","last_peered":"2023-10-09T06:24:59.840351+0000","last_clean":"2023-10-09T06:24:59.840351+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:24:59.840351+0000","last_undegraded":"2023-10-09T06:24:59.840351+0000","last_fullsized":"2023-10-09T06:24:59.840351+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838686+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:24:59.838686+0000","last_peered":"2023-10-09T06:24:59.838686+0000","last_clean":"2023-10-09T06:24:59.838686+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:24:59.838686+0000","last_undegraded":"2023-10-09T06:24:59.838686+0000","last_fullsized":"2023-10-09T06:24:59.838686+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"282'7896","reported_seq":7960,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.246315+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:25:16.246315+0000","last_peered":"2023-10-09T06:25:16.246315+0000","last_clean":"2023-10-09T06:25:16.246315+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:25:16.246315+0000","last_undegraded":"2023-10-09T06:25:16.246315+0000","last_fullsized":"2023-10-09T06:25:16.246315+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7896,"log_dups_size":0,"ondisk_log_size":7896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69468160,"num_objects":1060,"num_object_clones":0,"num_object_copies":3180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5354,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.852085+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:24:59.852085+0000","last_peered":"2023-10-09T06:24:59.852085+0000","last_clean":"2023-10-09T06:24:59.852085+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:24:59.852085+0000","last_undegraded":"2023-10-09T06:24:59.852085+0000","last_fullsized":"2023-10-09T06:24:59.852085+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":234,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838082+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:24:59.838082+0000","last_peered":"2023-10-09T06:24:59.838082+0000","last_clean":"2023-10-09T06:24:59.838082+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:24:59.838082+0000","last_undegraded":"2023-10-09T06:24:59.838082+0000","last_fullsized":"2023-10-09T06:24:59.838082+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"282'7877","reported_seq":8011,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.868963+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:25:16.868963+0000","last_peered":"2023-10-09T06:25:16.868963+0000","last_clean":"2023-10-09T06:25:16.868963+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:25:16.868963+0000","last_undegraded":"2023-10-09T06:25:16.868963+0000","last_fullsized":"2023-10-09T06:25:16.868963+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":7877,"log_dups_size":0,"ondisk_log_size":7877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78577664,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2583,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":574,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840005+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:24:59.840005+0000","last_peered":"2023-10-09T06:24:59.840005+0000","last_clean":"2023-10-09T06:24:59.840005+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:24:59.840005+0000","last_undegraded":"2023-10-09T06:24:59.840005+0000","last_fullsized":"2023-10-09T06:24:59.840005+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":245,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.851760+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:24:59.851760+0000","last_peered":"2023-10-09T06:24:59.851760+0000","last_clean":"2023-10-09T06:24:59.851760+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:24:59.851760+0000","last_undegraded":"2023-10-09T06:24:59.851760+0000","last_fullsized":"2023-10-09T06:24:59.851760+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835637+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:24:59.835637+0000","last_peered":"2023-10-09T06:24:59.835637+0000","last_clean":"2023-10-09T06:24:59.835637+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:24:59.835637+0000","last_undegraded":"2023-10-09T06:24:59.835637+0000","last_fullsized":"2023-10-09T06:24:59.835637+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"282'8030","reported_seq":8094,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.229789+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:25:16.229789+0000","last_peered":"2023-10-09T06:25:16.229789+0000","last_clean":"2023-10-09T06:25:16.229789+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:25:16.229789+0000","last_undegraded":"2023-10-09T06:25:16.229789+0000","last_fullsized":"2023-10-09T06:25:16.229789+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8030,"log_dups_size":0,"ondisk_log_size":8030,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79429632,"num_objects":1212,"num_object_clones":0,"num_object_copies":3636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5450,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":565,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.834328+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:24:59.834328+0000","last_peered":"2023-10-09T06:24:59.834328+0000","last_clean":"2023-10-09T06:24:59.834328+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:24:59.834328+0000","last_undegraded":"2023-10-09T06:24:59.834328+0000","last_fullsized":"2023-10-09T06:24:59.834328+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838490+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:24:59.838490+0000","last_peered":"2023-10-09T06:24:59.838490+0000","last_clean":"2023-10-09T06:24:59.838490+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:24:59.838490+0000","last_undegraded":"2023-10-09T06:24:59.838490+0000","last_fullsized":"2023-10-09T06:24:59.838490+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":259,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835655+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:24:59.835655+0000","last_peered":"2023-10-09T06:24:59.835655+0000","last_clean":"2023-10-09T06:24:59.835655+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:24:59.835655+0000","last_undegraded":"2023-10-09T06:24:59.835655+0000","last_fullsized":"2023-10-09T06:24:59.835655+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"282'7798","reported_seq":7975,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.828743+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:25:12.828743+0000","last_peered":"2023-10-09T06:25:12.828743+0000","last_clean":"2023-10-09T06:25:12.828743+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:25:12.828743+0000","last_undegraded":"2023-10-09T06:25:12.828743+0000","last_fullsized":"2023-10-09T06:25:12.828743+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":7798,"log_dups_size":0,"ondisk_log_size":7798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2663,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":263,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835051+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:24:59.835051+0000","last_peered":"2023-10-09T06:24:59.835051+0000","last_clean":"2023-10-09T06:24:59.835051+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:24:59.835051+0000","last_undegraded":"2023-10-09T06:24:59.835051+0000","last_fullsized":"2023-10-09T06:24:59.835051+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835880+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:24:59.835880+0000","last_peered":"2023-10-09T06:24:59.835880+0000","last_clean":"2023-10-09T06:24:59.835880+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:24:59.835880+0000","last_undegraded":"2023-10-09T06:24:59.835880+0000","last_fullsized":"2023-10-09T06:24:59.835880+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"282'8004","reported_seq":8188,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.963784+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:25:12.963784+0000","last_peered":"2023-10-09T06:25:12.963784+0000","last_clean":"2023-10-09T06:25:12.963784+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:25:12.963784+0000","last_undegraded":"2023-10-09T06:25:12.963784+0000","last_fullsized":"2023-10-09T06:25:12.963784+0000","mapping_epoch":238,"log_start":"242'63","ondisk_log_start":"242'63","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":7941,"log_dups_size":0,"ondisk_log_size":7941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2579,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":561,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.836517+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:24:59.836517+0000","last_peered":"2023-10-09T06:24:59.836517+0000","last_clean":"2023-10-09T06:24:59.836517+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:24:59.836517+0000","last_undegraded":"2023-10-09T06:24:59.836517+0000","last_fullsized":"2023-10-09T06:24:59.836517+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":238,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.852044+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:24:59.852044+0000","last_peered":"2023-10-09T06:24:59.852044+0000","last_clean":"2023-10-09T06:24:59.852044+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:24:59.852044+0000","last_undegraded":"2023-10-09T06:24:59.852044+0000","last_fullsized":"2023-10-09T06:24:59.852044+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":213,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838348+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:24:59.838348+0000","last_peered":"2023-10-09T06:24:59.838348+0000","last_clean":"2023-10-09T06:24:59.838348+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:24:59.838348+0000","last_undegraded":"2023-10-09T06:24:59.838348+0000","last_fullsized":"2023-10-09T06:24:59.838348+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"282'10211","reported_seq":10263,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:13.035934+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:25:13.035934+0000","last_peered":"2023-10-09T06:25:13.035934+0000","last_clean":"2023-10-09T06:25:13.035934+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:25:13.035934+0000","last_undegraded":"2023-10-09T06:25:13.035934+0000","last_fullsized":"2023-10-09T06:25:13.035934+0000","mapping_epoch":238,"log_start":"248'3500","ondisk_log_start":"248'3500","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10211,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":504,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838232+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:24:59.838232+0000","last_peered":"2023-10-09T06:24:59.838232+0000","last_clean":"2023-10-09T06:24:59.838232+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:24:59.838232+0000","last_undegraded":"2023-10-09T06:24:59.838232+0000","last_fullsized":"2023-10-09T06:24:59.838232+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":264,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838045+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:24:59.838045+0000","last_peered":"2023-10-09T06:24:59.838045+0000","last_clean":"2023-10-09T06:24:59.838045+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:24:59.838045+0000","last_undegraded":"2023-10-09T06:24:59.838045+0000","last_fullsized":"2023-10-09T06:24:59.838045+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.865501+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:24:59.865501+0000","last_peered":"2023-10-09T06:24:59.865501+0000","last_clean":"2023-10-09T06:24:59.865501+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:24:59.865501+0000","last_undegraded":"2023-10-09T06:24:59.865501+0000","last_fullsized":"2023-10-09T06:24:59.865501+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":207,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.839571+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:24:59.839571+0000","last_peered":"2023-10-09T06:24:59.839571+0000","last_clean":"2023-10-09T06:24:59.839571+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:24:59.839571+0000","last_undegraded":"2023-10-09T06:24:59.839571+0000","last_fullsized":"2023-10-09T06:24:59.839571+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"282'10619","reported_seq":10671,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.872852+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:25:12.872852+0000","last_peered":"2023-10-09T06:25:12.872852+0000","last_clean":"2023-10-09T06:25:12.872852+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:25:12.872852+0000","last_undegraded":"2023-10-09T06:25:12.872852+0000","last_fullsized":"2023-10-09T06:25:12.872852+0000","mapping_epoch":238,"log_start":"250'4600","ondisk_log_start":"250'4600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10619,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":566,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.842375+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:24:59.842375+0000","last_peered":"2023-10-09T06:24:59.842375+0000","last_clean":"2023-10-09T06:24:59.842375+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:24:59.842375+0000","last_undegraded":"2023-10-09T06:24:59.842375+0000","last_fullsized":"2023-10-09T06:24:59.842375+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":138,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.839691+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:24:59.839691+0000","last_peered":"2023-10-09T06:24:59.839691+0000","last_clean":"2023-10-09T06:24:59.839691+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:24:59.839691+0000","last_undegraded":"2023-10-09T06:24:59.839691+0000","last_fullsized":"2023-10-09T06:24:59.839691+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":243,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835091+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:24:59.835091+0000","last_peered":"2023-10-09T06:24:59.835091+0000","last_clean":"2023-10-09T06:24:59.835091+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:24:59.835091+0000","last_undegraded":"2023-10-09T06:24:59.835091+0000","last_fullsized":"2023-10-09T06:24:59.835091+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":228,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.837975+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:24:59.837975+0000","last_peered":"2023-10-09T06:24:59.837975+0000","last_clean":"2023-10-09T06:24:59.837975+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:24:59.837975+0000","last_undegraded":"2023-10-09T06:24:59.837975+0000","last_fullsized":"2023-10-09T06:24:59.837975+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"282'7981","reported_seq":8109,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.181864+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:25:16.181864+0000","last_peered":"2023-10-09T06:25:16.181864+0000","last_clean":"2023-10-09T06:25:16.181864+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:25:16.181864+0000","last_undegraded":"2023-10-09T06:25:16.181864+0000","last_fullsized":"2023-10-09T06:25:16.181864+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":7981,"log_dups_size":0,"ondisk_log_size":7981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80216064,"num_objects":1224,"num_object_clones":0,"num_object_copies":3672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2809,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":574,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840095+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:24:59.840095+0000","last_peered":"2023-10-09T06:24:59.840095+0000","last_clean":"2023-10-09T06:24:59.840095+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:24:59.840095+0000","last_undegraded":"2023-10-09T06:24:59.840095+0000","last_fullsized":"2023-10-09T06:24:59.840095+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835118+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:24:59.835118+0000","last_peered":"2023-10-09T06:24:59.835118+0000","last_clean":"2023-10-09T06:24:59.835118+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:24:59.835118+0000","last_undegraded":"2023-10-09T06:24:59.835118+0000","last_fullsized":"2023-10-09T06:24:59.835118+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":255,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:01.883814+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:25:01.883814+0000","last_peered":"2023-10-09T06:25:01.883814+0000","last_clean":"2023-10-09T06:25:01.883814+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:25:01.883814+0000","last_undegraded":"2023-10-09T06:25:01.883814+0000","last_fullsized":"2023-10-09T06:25:01.883814+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838413+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:24:59.838413+0000","last_peered":"2023-10-09T06:24:59.838413+0000","last_clean":"2023-10-09T06:24:59.838413+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:24:59.838413+0000","last_undegraded":"2023-10-09T06:24:59.838413+0000","last_fullsized":"2023-10-09T06:24:59.838413+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"282'10519","reported_seq":10570,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.861381+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:25:12.861381+0000","last_peered":"2023-10-09T06:25:12.861381+0000","last_clean":"2023-10-09T06:25:12.861381+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:25:12.861381+0000","last_undegraded":"2023-10-09T06:25:12.861381+0000","last_fullsized":"2023-10-09T06:25:12.861381+0000","mapping_epoch":238,"log_start":"249'4500","ondisk_log_start":"249'4500","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163774464,"num_objects":2499,"num_object_clones":0,"num_object_copies":7497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10519,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":214,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835220+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:24:59.835220+0000","last_peered":"2023-10-09T06:24:59.835220+0000","last_clean":"2023-10-09T06:24:59.835220+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:24:59.835220+0000","last_undegraded":"2023-10-09T06:24:59.835220+0000","last_fullsized":"2023-10-09T06:24:59.835220+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":555,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.841321+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:24:59.841321+0000","last_peered":"2023-10-09T06:24:59.841321+0000","last_clean":"2023-10-09T06:24:59.841321+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:24:59.841321+0000","last_undegraded":"2023-10-09T06:24:59.841321+0000","last_fullsized":"2023-10-09T06:24:59.841321+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":209,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:03.805851+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:25:03.805851+0000","last_peered":"2023-10-09T06:25:03.805851+0000","last_clean":"2023-10-09T06:25:03.805851+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:25:03.805851+0000","last_undegraded":"2023-10-09T06:25:03.805851+0000","last_fullsized":"2023-10-09T06:25:03.805851+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":244,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838375+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:24:59.838375+0000","last_peered":"2023-10-09T06:24:59.838375+0000","last_clean":"2023-10-09T06:24:59.838375+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:24:59.838375+0000","last_undegraded":"2023-10-09T06:24:59.838375+0000","last_fullsized":"2023-10-09T06:24:59.838375+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"282'7815","reported_seq":7942,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.195412+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:25:15.195412+0000","last_peered":"2023-10-09T06:25:15.195412+0000","last_clean":"2023-10-09T06:25:15.195412+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:25:15.195412+0000","last_undegraded":"2023-10-09T06:25:15.195412+0000","last_fullsized":"2023-10-09T06:25:15.195412+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":7815,"log_dups_size":0,"ondisk_log_size":7815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563008,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2637,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":564,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838644+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:24:59.838644+0000","last_peered":"2023-10-09T06:24:59.838644+0000","last_clean":"2023-10-09T06:24:59.838644+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:24:59.838644+0000","last_undegraded":"2023-10-09T06:24:59.838644+0000","last_fullsized":"2023-10-09T06:24:59.838644+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":125,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.841217+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:24:59.841217+0000","last_peered":"2023-10-09T06:24:59.841217+0000","last_clean":"2023-10-09T06:24:59.841217+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:24:59.841217+0000","last_undegraded":"2023-10-09T06:24:59.841217+0000","last_fullsized":"2023-10-09T06:24:59.841217+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":235,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.842282+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:24:59.842282+0000","last_peered":"2023-10-09T06:24:59.842282+0000","last_clean":"2023-10-09T06:24:59.842282+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:24:59.842282+0000","last_undegraded":"2023-10-09T06:24:59.842282+0000","last_fullsized":"2023-10-09T06:24:59.842282+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":257,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.837993+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:24:59.837993+0000","last_peered":"2023-10-09T06:24:59.837993+0000","last_clean":"2023-10-09T06:24:59.837993+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:24:59.837993+0000","last_undegraded":"2023-10-09T06:24:59.837993+0000","last_fullsized":"2023-10-09T06:24:59.837993+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"282'10640","reported_seq":10693,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.893021+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:25:12.893021+0000","last_peered":"2023-10-09T06:25:12.893021+0000","last_clean":"2023-10-09T06:25:12.893021+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:25:12.893021+0000","last_undegraded":"2023-10-09T06:25:12.893021+0000","last_fullsized":"2023-10-09T06:25:12.893021+0000","mapping_epoch":238,"log_start":"249'4600","ondisk_log_start":"249'4600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6040,"log_dups_size":0,"ondisk_log_size":6040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624152,"num_objects":2574,"num_object_clones":0,"num_object_copies":7722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2574,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10640,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":504,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838068+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:24:59.838068+0000","last_peered":"2023-10-09T06:24:59.838068+0000","last_clean":"2023-10-09T06:24:59.838068+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:24:59.838068+0000","last_undegraded":"2023-10-09T06:24:59.838068+0000","last_fullsized":"2023-10-09T06:24:59.838068+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":125,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840404+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:24:59.840404+0000","last_peered":"2023-10-09T06:24:59.840404+0000","last_clean":"2023-10-09T06:24:59.840404+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:24:59.840404+0000","last_undegraded":"2023-10-09T06:24:59.840404+0000","last_fullsized":"2023-10-09T06:24:59.840404+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.837904+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:24:59.837904+0000","last_peered":"2023-10-09T06:24:59.837904+0000","last_clean":"2023-10-09T06:24:59.837904+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:24:59.837904+0000","last_undegraded":"2023-10-09T06:24:59.837904+0000","last_fullsized":"2023-10-09T06:24:59.837904+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838244+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:24:59.838244+0000","last_peered":"2023-10-09T06:24:59.838244+0000","last_clean":"2023-10-09T06:24:59.838244+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:24:59.838244+0000","last_undegraded":"2023-10-09T06:24:59.838244+0000","last_fullsized":"2023-10-09T06:24:59.838244+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"282'10413","reported_seq":10465,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:13.028161+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:25:13.028161+0000","last_peered":"2023-10-09T06:25:13.028161+0000","last_clean":"2023-10-09T06:25:13.028161+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:25:13.028161+0000","last_undegraded":"2023-10-09T06:25:13.028161+0000","last_fullsized":"2023-10-09T06:25:13.028161+0000","mapping_epoch":238,"log_start":"252'5000","ondisk_log_start":"252'5000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5413,"log_dups_size":0,"ondisk_log_size":5413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160104448,"num_objects":2443,"num_object_clones":0,"num_object_copies":7329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10413,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":565,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835563+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:24:59.835563+0000","last_peered":"2023-10-09T06:24:59.835563+0000","last_clean":"2023-10-09T06:24:59.835563+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:24:59.835563+0000","last_undegraded":"2023-10-09T06:24:59.835563+0000","last_fullsized":"2023-10-09T06:24:59.835563+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":271,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.865007+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:24:59.865007+0000","last_peered":"2023-10-09T06:24:59.865007+0000","last_clean":"2023-10-09T06:24:59.865007+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:24:59.865007+0000","last_undegraded":"2023-10-09T06:24:59.865007+0000","last_fullsized":"2023-10-09T06:24:59.865007+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":240,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838764+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:24:59.838764+0000","last_peered":"2023-10-09T06:24:59.838764+0000","last_clean":"2023-10-09T06:24:59.838764+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:24:59.838764+0000","last_undegraded":"2023-10-09T06:24:59.838764+0000","last_fullsized":"2023-10-09T06:24:59.838764+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.851971+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:24:59.851971+0000","last_peered":"2023-10-09T06:24:59.851971+0000","last_clean":"2023-10-09T06:24:59.851971+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:24:59.851971+0000","last_undegraded":"2023-10-09T06:24:59.851971+0000","last_fullsized":"2023-10-09T06:24:59.851971+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"282'10603","reported_seq":10655,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.886426+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:25:12.886426+0000","last_peered":"2023-10-09T06:25:12.886426+0000","last_clean":"2023-10-09T06:25:12.886426+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:25:12.886426+0000","last_undegraded":"2023-10-09T06:25:12.886426+0000","last_fullsized":"2023-10-09T06:25:12.886426+0000","mapping_epoch":238,"log_start":"249'4600","ondisk_log_start":"249'4600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10603,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":585,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.864307+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:24:59.864307+0000","last_peered":"2023-10-09T06:24:59.864307+0000","last_clean":"2023-10-09T06:24:59.864307+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:24:59.864307+0000","last_undegraded":"2023-10-09T06:24:59.864307+0000","last_fullsized":"2023-10-09T06:24:59.864307+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":266,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838544+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:24:59.838544+0000","last_peered":"2023-10-09T06:24:59.838544+0000","last_clean":"2023-10-09T06:24:59.838544+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:24:59.838544+0000","last_undegraded":"2023-10-09T06:24:59.838544+0000","last_fullsized":"2023-10-09T06:24:59.838544+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":238,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.837908+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:24:59.837908+0000","last_peered":"2023-10-09T06:24:59.837908+0000","last_clean":"2023-10-09T06:24:59.837908+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:24:59.837908+0000","last_undegraded":"2023-10-09T06:24:59.837908+0000","last_fullsized":"2023-10-09T06:24:59.837908+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":235,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835189+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:24:59.835189+0000","last_peered":"2023-10-09T06:24:59.835189+0000","last_clean":"2023-10-09T06:24:59.835189+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:24:59.835189+0000","last_undegraded":"2023-10-09T06:24:59.835189+0000","last_fullsized":"2023-10-09T06:24:59.835189+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"282'8817","reported_seq":17322,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:25:16.867858+0000","last_change":"2023-10-09T06:23:40.105141+0000","last_active":"2023-10-09T06:25:16.867858+0000","last_peered":"2023-10-09T06:25:16.867858+0000","last_clean":"2023-10-09T06:23:37.701201+0000","last_became_active":"2023-10-09T06:23:40.099906+0000","last_became_peered":"2023-10-09T06:23:40.099906+0000","last_unstale":"2023-10-09T06:25:16.867858+0000","last_undegraded":"2023-10-09T06:23:39.992159+0000","last_fullsized":"2023-10-09T06:23:39.910881+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":8817,"log_dups_size":0,"ondisk_log_size":8817,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74874880,"num_objects":1143,"num_object_clones":0,"num_object_copies":3429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1402,"num_objects_misplaced":701,"num_objects_unfound":0,"num_objects_dirty":1143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5313,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2085,"num_bytes_recovered":136642560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[1,2],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":701},{"shards":"1,2","objects":442}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":240,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.838198+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:24:59.838198+0000","last_peered":"2023-10-09T06:24:59.838198+0000","last_clean":"2023-10-09T06:24:59.838198+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:24:59.838198+0000","last_undegraded":"2023-10-09T06:24:59.838198+0000","last_fullsized":"2023-10-09T06:24:59.838198+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":264,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.385995+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:25:11.385995+0000","last_peered":"2023-10-09T06:25:11.385995+0000","last_clean":"2023-10-09T06:25:11.385995+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:25:11.385995+0000","last_undegraded":"2023-10-09T06:25:11.385995+0000","last_fullsized":"2023-10-09T06:25:11.385995+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"282'7822","reported_seq":7885,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:16.239768+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:25:16.239768+0000","last_peered":"2023-10-09T06:25:16.239768+0000","last_clean":"2023-10-09T06:25:16.239768+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:25:16.239768+0000","last_undegraded":"2023-10-09T06:25:16.239768+0000","last_fullsized":"2023-10-09T06:25:16.239768+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7822,"log_dups_size":0,"ondisk_log_size":7822,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81395712,"num_objects":1242,"num_object_clones":0,"num_object_copies":3726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":87,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835528+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:24:59.835528+0000","last_peered":"2023-10-09T06:24:59.835528+0000","last_clean":"2023-10-09T06:24:59.835528+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:24:59.835528+0000","last_undegraded":"2023-10-09T06:24:59.835528+0000","last_fullsized":"2023-10-09T06:24:59.835528+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":269,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835046+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:24:59.835046+0000","last_peered":"2023-10-09T06:24:59.835046+0000","last_clean":"2023-10-09T06:24:59.835046+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:24:59.835046+0000","last_undegraded":"2023-10-09T06:24:59.835046+0000","last_fullsized":"2023-10-09T06:24:59.835046+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"282'7914","reported_seq":7977,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.091711+0000","last_change":"2023-10-09T06:23:26.834184+0000","last_active":"2023-10-09T06:25:15.091711+0000","last_peered":"2023-10-09T06:25:15.091711+0000","last_clean":"2023-10-09T06:25:15.091711+0000","last_became_active":"2023-10-09T06:23:26.832172+0000","last_became_peered":"2023-10-09T06:23:26.832172+0000","last_unstale":"2023-10-09T06:25:15.091711+0000","last_undegraded":"2023-10-09T06:25:15.091711+0000","last_fullsized":"2023-10-09T06:25:15.091711+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84246528,"num_objects":1286,"num_object_clones":0,"num_object_copies":3858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5312,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":87,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.835694+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:24:59.835694+0000","last_peered":"2023-10-09T06:24:59.835694+0000","last_clean":"2023-10-09T06:24:59.835694+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:24:59.835694+0000","last_undegraded":"2023-10-09T06:24:59.835694+0000","last_fullsized":"2023-10-09T06:24:59.835694+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":259,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.834764+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:24:59.834764+0000","last_peered":"2023-10-09T06:24:59.834764+0000","last_clean":"2023-10-09T06:24:59.834764+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:24:59.834764+0000","last_undegraded":"2023-10-09T06:24:59.834764+0000","last_fullsized":"2023-10-09T06:24:59.834764+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"282'7661","reported_seq":7723,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:12.886516+0000","last_change":"2023-10-09T06:23:26.832837+0000","last_active":"2023-10-09T06:25:12.886516+0000","last_peered":"2023-10-09T06:25:12.886516+0000","last_clean":"2023-10-09T06:25:12.886516+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:25:12.886516+0000","last_undegraded":"2023-10-09T06:25:12.886516+0000","last_fullsized":"2023-10-09T06:25:12.886516+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7661,"log_dups_size":0,"ondisk_log_size":7661,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81657856,"num_objects":1246,"num_object_clones":0,"num_object_copies":3738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5132,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":240,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.851413+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:24:59.851413+0000","last_peered":"2023-10-09T06:24:59.851413+0000","last_clean":"2023-10-09T06:24:59.851413+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:24:59.851413+0000","last_undegraded":"2023-10-09T06:24:59.851413+0000","last_fullsized":"2023-10-09T06:24:59.851413+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":254,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.850958+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:24:59.850958+0000","last_peered":"2023-10-09T06:24:59.850958+0000","last_clean":"2023-10-09T06:24:59.850958+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:24:59.850958+0000","last_undegraded":"2023-10-09T06:24:59.850958+0000","last_fullsized":"2023-10-09T06:24:59.850958+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"282'7781","reported_seq":7844,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:15.294538+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:25:15.294538+0000","last_peered":"2023-10-09T06:25:15.294538+0000","last_clean":"2023-10-09T06:25:15.294538+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:25:15.294538+0000","last_undegraded":"2023-10-09T06:25:15.294538+0000","last_fullsized":"2023-10-09T06:25:15.294538+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7781,"log_dups_size":0,"ondisk_log_size":7781,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82345984,"num_objects":1257,"num_object_clones":0,"num_object_copies":3771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5223,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":254,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.840440+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:24:59.840440+0000","last_peered":"2023-10-09T06:24:59.840440+0000","last_clean":"2023-10-09T06:24:59.840440+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:24:59.840440+0000","last_undegraded":"2023-10-09T06:24:59.840440+0000","last_fullsized":"2023-10-09T06:24:59.840440+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":242,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-10-09T06:24:59.834229+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:24:59.834229+0000","last_peered":"2023-10-09T06:24:59.834229+0000","last_clean":"2023-10-09T06:24:59.834229+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:24:59.834229+0000","last_undegraded":"2023-10-09T06:24:59.834229+0000","last_fullsized":"2023-10-09T06:24:59.834229+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2562392088,"num_objects":39102,"num_object_clones":0,"num_object_copies":117306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2464,"num_objects_misplaced":701,"num_objects_unfound":0,"num_objects_dirty":39102,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150444,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9361,"num_bytes_recovered":592642048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8589684736,"data_stored":8589672520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194490,"ondisk_log_size":194490,"up":78,"acting":76,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234335,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1732300,"kb_used_data":836100,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":92012852,"statfs":{"total":95995035648,"available":94221160448,"internally_reserved":0,"allocated":856166400,"data_stored":855388335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3399999999999999,"5min":7.7510000000000003,"15min":3.4870000000000001},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":60.524999999999999,"5min":219.41900000000001,"15min":219.41900000000001},"last":2.419},{"interface":"front","average":{"1min":4.4130000000000003,"5min":8.1159999999999997,"15min":3.5529999999999999},"min":{"1min":0.27900000000000003,"5min":0.218,"15min":0.218},"max":{"1min":60.539000000000001,"5min":219.399,"15min":219.399},"last":0.38400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":3.7959999999999998,"15min":2.6680000000000001},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.7720000000000002,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.984},{"interface":"front","average":{"1min":2.04,"5min":3.6400000000000001,"15min":2.4049999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":12.805999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":2.4569999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":17.384,"5min":10.077999999999999,"15min":4.5700000000000003},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":131.773,"5min":215.34800000000001,"15min":215.34800000000001},"last":1.546},{"interface":"front","average":{"1min":18.056999999999999,"5min":10.329000000000001,"15min":4.742},"min":{"1min":0.35599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":132.28100000000001,"5min":215.869,"15min":215.869},"last":6.274}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.9889999999999999,"15min":1.542},"min":{"1min":0.40999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.2149999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.012,"5min":3.1389999999999998,"15min":1.599},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.4550000000000001,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.40500000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.161,"15min":1.673},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":94.105999999999995,"5min":94.105999999999995,"15min":94.105999999999995},"last":2.641},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.218,"15min":1.4379999999999999},"min":{"1min":0.31900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":93.713999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.32100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.429,"15min":1.714},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9220000000000002,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.341,"5min":1.3140000000000001,"15min":1.429},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":5.7130000000000001,"5min":7.452,"15min":48.774999999999999},"last":2.6709999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.494,"15min":1.359},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7889999999999997,"5min":8.4890000000000008,"15min":11.878},"last":2.734},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.464,"15min":1.256},"min":{"1min":0.38600000000000001,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":5.2690000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.42699999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495972,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2149380,"kb_used_data":1250364,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91595772,"statfs":{"total":95995035648,"available":93794070528,"internally_reserved":0,"allocated":1280372736,"data_stored":1279603251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":3.9129999999999998,"15min":3.1339999999999999},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":28.035,"5min":110.926,"15min":110.926},"last":0.376},{"interface":"front","average":{"1min":1.635,"5min":4.0149999999999997,"15min":3.3319999999999999},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":7.0499999999999998,"5min":110.911,"15min":110.911},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":4.9089999999999998,"15min":3.242},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":12.881,"5min":114.202,"15min":114.202},"last":66.796999999999997},{"interface":"front","average":{"1min":1.4259999999999999,"5min":4.3840000000000003,"15min":3.0710000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.1790000000000003,"5min":114.179,"15min":114.179},"last":66.704999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":3.484,"15min":2.6040000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.8419999999999996,"5min":75.191999999999993,"15min":75.191999999999993},"last":52.609999999999999},{"interface":"front","average":{"1min":2.4769999999999999,"5min":3.0939999999999999,"15min":2.5059999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.215999999999999,"5min":75.120000000000005,"15min":75.120000000000005},"last":52.575000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":3.2149999999999999,"15min":1.8939999999999999},"min":{"1min":0.31900000000000001,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":11.148,"5min":110.358,"15min":110.358},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.1680000000000001,"5min":3.5870000000000002,"15min":1.986},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":11.180999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4870000000000001,"15min":1.2549999999999999},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":3.6480000000000001,"5min":17.643999999999998,"15min":17.643999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.4990000000000001,"15min":1.256},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":6.8780000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.434}]},{"osd":5,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.806,"15min":1.3460000000000001},"min":{"1min":0.39200000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":4.3620000000000001,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.597,"5min":1.6499999999999999,"15min":1.3740000000000001},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.2149999999999999,"5min":24.401,"15min":24.401},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.5139999999999998,"15min":1.5129999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":8.5879999999999992,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.4969999999999999,"15min":1.712},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":3.5390000000000001,"5min":100.995,"15min":100.995},"last":0.47099999999999997}]}]},{"osd":1,"up_from":110,"seq":472446402709,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257672,"kb_used_data":776320,"kb_used_omap":4,"kb_used_meta":481339,"kb_avail":92487480,"statfs":{"total":95995035648,"available":94707179520,"internally_reserved":0,"allocated":794951680,"data_stored":793707090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":492892046},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":20.606999999999999,"5min":7.0389999999999997,"15min":3.7789999999999999},"min":{"1min":0.255,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":145.96600000000001,"5min":148.98400000000001,"15min":148.98400000000001},"last":102.471},{"interface":"front","average":{"1min":22.132000000000001,"5min":7.8310000000000004,"15min":4.2510000000000003},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":145.982,"5min":149.02699999999999,"15min":149.02699999999999},"last":102.054}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":15.206,"5min":5.3970000000000002,"15min":3.048},"min":{"1min":0.21299999999999999,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":139.75399999999999,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":16.838000000000001,"5min":6.3470000000000004,"15min":3.8319999999999999},"min":{"1min":0.24199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":140.29900000000001,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.25900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.9089999999999998,"15min":2.327},"min":{"1min":0.30599999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.585,"5min":77.331999999999994,"15min":153.18199999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.5379999999999998,"15min":1.8340000000000001},"min":{"1min":0.31,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.754,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.31}]},{"osd":4,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":16.911999999999999,"5min":6.9329999999999998,"15min":3.8330000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":99.391000000000005,"5min":199.43799999999999,"15min":199.43799999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":16.670999999999999,"5min":6.5439999999999996,"15min":3.7650000000000001},"min":{"1min":0.311,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":99.462000000000003,"5min":199.32900000000001,"15min":199.32900000000001},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":7.665,"5min":3.1459999999999999,"15min":2.7210000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":77.436999999999998,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":8.4740000000000002,"5min":3.7610000000000001,"15min":3.0059999999999998},"min":{"1min":0.371,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":77.388000000000005,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.59299999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":9.5429999999999993,"5min":3.9060000000000001,"15min":3.0030000000000001},"min":{"1min":0.38700000000000001,"5min":0.34999999999999998,"15min":0.28399999999999997},"max":{"1min":77.292000000000002,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":8.7949999999999999,"5min":3.8450000000000002,"15min":2.7669999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.374},"max":{"1min":77.378,"5min":77.378,"15min":77.378},"last":0.623}]},{"osd":7,"last update":"Mon Oct 9 06:24:23 2023","interfaces":[{"interface":"back","average":{"1min":8.6129999999999995,"5min":3.6909999999999998,"15min":2.7799999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":77.456999999999994,"5min":77.456999999999994,"15min":77.456999999999994},"last":0.65600000000000003},{"interface":"front","average":{"1min":8.1199999999999992,"5min":3.117,"15min":2.2759999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":77.359999999999999,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.60299999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024142,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343092,"kb_used_data":1664684,"kb_used_omap":5,"kb_used_meta":678202,"kb_avail":91402060,"statfs":{"total":95995035648,"available":93595709440,"internally_reserved":0,"allocated":1704636416,"data_stored":1703178682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":694479482},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":4.673,"15min":2.8130000000000002},"min":{"1min":0.19500000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":13.916,"5min":185.17400000000001,"15min":185.17400000000001},"last":3.0489999999999999},{"interface":"front","average":{"1min":2.9609999999999999,"5min":4.7080000000000002,"15min":3.3500000000000001},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":13.945,"5min":186.691,"15min":186.691},"last":4.9729999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.2349999999999999,"15min":2.2599999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":13.211,"5min":13.211,"15min":17.588999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.472,"15min":2.391},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":11.956,"5min":13.081,"15min":21.198},"last":2.649}]},{"osd":3,"last update":"Mon Oct 9 06:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":6.3200000000000003,"15min":2.726},"min":{"1min":0.28499999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.375,"5min":185.89400000000001,"15min":185.89400000000001},"last":2.8050000000000002},{"interface":"front","average":{"1min":3.2120000000000002,"5min":6.6360000000000001,"15min":2.8530000000000002},"min":{"1min":0.27000000000000002,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":13.138999999999999,"5min":186.72900000000001,"15min":186.72900000000001},"last":4.9480000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":1.599,"15min":1.627},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9850000000000003,"5min":7.9660000000000002,"15min":15.835000000000001},"last":2.4020000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.7829999999999999,"15min":2.0590000000000002},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.25700000000000001},"max":{"1min":5.1310000000000002,"5min":8.4890000000000008,"15min":14.792999999999999},"last":4.9139999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.1699999999999999,"15min":2.036},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.0780000000000003,"5min":62.984999999999999,"15min":62.984999999999999},"last":2.4420000000000002},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.8479999999999999,"15min":2.5920000000000001},"min":{"1min":0.35199999999999998,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":3.8610000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":1.5269999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":2.3420000000000001,"15min":2.3759999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.4710000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":1.5589999999999999},{"interface":"front","average":{"1min":1.669,"5min":1.7949999999999999,"15min":1.9430000000000001},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7439999999999998,"5min":10.606,"15min":15.893000000000001},"last":2.423}]},{"osd":7,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.516,"15min":2.3279999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28899999999999998},"max":{"1min":7.5010000000000003,"5min":62.262,"15min":62.262},"last":1.3440000000000001},{"interface":"front","average":{"1min":2.3340000000000001,"5min":2.9740000000000002,"15min":2.645},"min":{"1min":0.377,"5min":0.377,"15min":0.314},"max":{"1min":14.167,"5min":62.229999999999997,"15min":62.229999999999997},"last":1.671}]}]},{"osd":2,"up_from":128,"seq":549755814011,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577524,"kb_used_data":1241260,"kb_used_omap":7,"kb_used_meta":336248,"kb_avail":92167628,"statfs":{"total":95995035648,"available":94379651072,"internally_reserved":0,"allocated":1271050240,"data_stored":1269821944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":344318033},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":3.802,"15min":3.4159999999999999},"min":{"1min":0.20799999999999999,"5min":0.157,"15min":0.157},"max":{"1min":10.356,"5min":110.611,"15min":163.40100000000001},"last":1.9139999999999999},{"interface":"front","average":{"1min":2.5099999999999998,"5min":3.968,"15min":3.8540000000000001},"min":{"1min":0.24099999999999999,"5min":0.182,"15min":0.182},"max":{"1min":9.0679999999999996,"5min":110.66500000000001,"15min":163.363},"last":0.246}]},{"osd":1,"last update":"Mon Oct 9 06:24:27 2023","interfaces":[{"interface":"back","average":{"1min":13.189,"5min":4.4189999999999996,"15min":3.573},"min":{"1min":0.26100000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":205.148,"5min":205.148,"15min":222.50299999999999},"last":1.401},{"interface":"front","average":{"1min":12.093999999999999,"5min":4.5579999999999998,"15min":4.0209999999999999},"min":{"1min":0.316,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":205.267,"5min":205.267,"15min":222.19399999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.629,"15min":1.6759999999999999},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":16.331,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.9950000000000001,"5min":2.8159999999999998,"15min":1.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":16.364000000000001,"5min":69.906999999999996,"15min":81.061999999999998},"last":1.1279999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":7.7530000000000001,"5min":2.7330000000000001,"15min":2.1339999999999999},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":90.938999999999993,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.42199999999999999},{"interface":"front","average":{"1min":7.2480000000000002,"5min":3.1110000000000002,"15min":2.4169999999999998},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":90.950000000000003,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":1.5409999999999999,"15min":1.579},"min":{"1min":0.41499999999999998,"5min":0.317,"15min":0.309},"max":{"1min":8.9589999999999996,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.675,"5min":2.0259999999999998,"15min":2.1899999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.4320000000000004,"5min":10.548,"15min":10.548},"last":2.3799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":9.2420000000000009,"5min":3.2189999999999999,"15min":2.4630000000000001},"min":{"1min":0.36199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":138.291,"5min":138.291,"15min":138.291},"last":0.52100000000000002},{"interface":"front","average":{"1min":8.9420000000000002,"5min":2.9399999999999999,"15min":1.9990000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":138.01599999999999,"5min":138.01599999999999,"15min":138.01599999999999},"last":1.4690000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.7949999999999999,"15min":1.651},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":1.4379999999999999},{"interface":"front","average":{"1min":2.73,"5min":1.7809999999999999,"15min":2.3399999999999999},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.9960000000000004,"5min":8.9960000000000004,"15min":9.3249999999999993},"last":0.53400000000000003}]}]},{"osd":3,"up_from":149,"seq":639950127208,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1246672,"kb_used_data":864264,"kb_used_omap":2,"kb_used_meta":382333,"kb_avail":92498480,"statfs":{"total":95995035648,"available":94718443520,"internally_reserved":0,"allocated":885006336,"data_stored":883870334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":391509045},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":8.4339999999999993,"5min":8.0909999999999993,"15min":3.198},"min":{"1min":0.308,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":105.252,"5min":187.852,"15min":187.852},"last":1.546},{"interface":"front","average":{"1min":8.1020000000000003,"5min":8.7400000000000002,"15min":3.7010000000000001},"min":{"1min":0.245,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":105.331,"5min":187.923,"15min":187.923},"last":2.5209999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3650000000000002,"5min":6.1289999999999996,"15min":2.8679999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.223},"max":{"1min":22.045999999999999,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":3.77,"5min":3.7010000000000001,"15min":1.847},"min":{"1min":0.26600000000000001,"5min":0.216,"15min":0.216},"max":{"1min":31.053000000000001,"5min":67.028000000000006,"15min":67.028000000000006},"last":1.6000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":5.5049999999999999,"15min":2.6110000000000002},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.891999999999999,"5min":187.76400000000001,"15min":187.76400000000001},"last":12.57},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.8470000000000004,"15min":2.008},"min":{"1min":0.23400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.202,"5min":187.79499999999999,"15min":187.79499999999999},"last":6.0449999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.6639999999999999,"15min":1.05},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":4.2590000000000003,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.8779999999999999,"5min":3.5790000000000002,"15min":1.992},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1370000000000005,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.78300000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.7749999999999999,"15min":2.1320000000000001},"min":{"1min":0.379,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":8.218,"5min":187.833,"15min":187.833},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.995,"5min":3.4350000000000001,"15min":2.0619999999999998},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.7249999999999996,"5min":187.952,"15min":187.952},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":3.762,"15min":2.1030000000000002},"min":{"1min":0.34899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1809999999999992,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.2989999999999999,"5min":3.3460000000000001,"15min":1.998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7030000000000003,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.734,"15min":1.7889999999999999},"min":{"1min":0.37,"5min":0.35599999999999998,"15min":0.34100000000000003},"max":{"1min":4.2960000000000003,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":2.0169999999999999,"5min":3.754,"15min":2.0920000000000001},"min":{"1min":0.38800000000000001,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":8.2509999999999994,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.75600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019244,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2779268,"kb_used_data":1892476,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":90965884,"statfs":{"total":95995035648,"available":93149065216,"internally_reserved":0,"allocated":1937895424,"data_stored":1937125218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":3.8069999999999999,"15min":2.3140000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.18,"5min":76.944999999999993,"15min":76.944999999999993},"last":2.7480000000000002},{"interface":"front","average":{"1min":5.444,"5min":4.2999999999999998,"15min":2.706},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":48.287999999999997,"5min":76.988,"15min":76.988},"last":2.7669999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":2.3130000000000002,"15min":2.093},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.2370000000000001,"5min":37.383000000000003,"15min":91.849999999999994},"last":2.7029999999999998},{"interface":"front","average":{"1min":1.599,"5min":2.6379999999999999,"15min":2.1899999999999999},"min":{"1min":0.26000000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.4199999999999999,"5min":37.436999999999998,"15min":91.885000000000005},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":2.2839999999999998,"15min":1.8919999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":4.4829999999999997,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.657,"5min":2.5329999999999999,"15min":2.0630000000000002},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":5.3079999999999998,"5min":69.676000000000002,"15min":69.676000000000002},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":3.6480000000000001,"15min":2.1459999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.262,"5min":57.826999999999998,"15min":91.697000000000003},"last":1.27},{"interface":"front","average":{"1min":1.7669999999999999,"5min":3.6680000000000001,"15min":2.081},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":6.3250000000000002,"5min":57.853000000000002,"15min":94.356999999999999},"last":0.46700000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.268,"15min":1.0620000000000001},"min":{"1min":0.29799999999999999,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":4.4400000000000004,"5min":10.673,"15min":10.673},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.202,"15min":1.2050000000000001},"min":{"1min":0.316,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":2.9990000000000001,"5min":12.552,"15min":12.552},"last":0.501}]},{"osd":6,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.77400000000000002,"15min":0.83499999999999996},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.8740000000000001,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.212,"15min":1.244},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5419999999999998,"5min":15.321999999999999,"15min":15.321999999999999},"last":1.6180000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.956,"15min":1.49},"min":{"1min":0.34499999999999997,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":5.4480000000000004,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.857,"15min":1.4770000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0640000000000001,"5min":62.573,"15min":62.573},"last":0.42599999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757609,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1663700992,"data_stored":1663696920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":785055744,"data_stored":785055744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1246691328,"data_stored":1246691328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":874250240,"data_stored":874250240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1914769408,"data_stored":1914765336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1261637632,"data_stored":1261633560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":843579392,"data_stored":843579392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:25:20.737 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:20.824 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:20.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:20 smithi107 ceph-mon[220921]: pgmap v869: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 2464/110577 objects degraded (2.228%); 701/110577 objects misplaced (0.634%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:20.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:20 smithi150 ceph-mon[194178]: pgmap v869: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 2464/110577 objects degraded (2.228%); 701/110577 objects misplaced (0.634%); 4.7 MiB/s, 106 objects/s recovering 2023-10-09T06:25:21.102 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:21.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:21.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:21.135 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:25:21.182 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:25:21.219 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.228 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:25:21.267 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:21.269 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:25:21.326 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:25:21.348 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:21.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:21.398 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:25:21.408 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:25:21.447 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:25:21.455 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:21.546 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:21.689 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:21.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:21.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:21.883 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3413720827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:21.886 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3413720827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3413720827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:21.960 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:21.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:22.069 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:22.118 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:22.174 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:22.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:22.279 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:22.304 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:22] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-10-09T06:25:22.307 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:22.533 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:22.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:22.642 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[212750]: pgmap v870: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 2009/108066 objects degraded (1.859%); 701/108066 objects misplaced (0.649%); 2.3 MiB/s, 114 objects/s recovering 2023-10-09T06:25:22.670 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/799984235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.670 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.670 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2009/108066 objects degraded (1.859%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:22.671 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:22.671 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:25:22.705 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:22.765 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:22.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:22.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:22.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:22 smithi150 ceph-mon[194178]: pgmap v870: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 2009/108066 objects degraded (1.859%); 701/108066 objects misplaced (0.649%); 2.3 MiB/s, 114 objects/s recovering 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/799984235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2009/108066 objects degraded (1.859%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[220921]: pgmap v870: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 845 op/s; 2009/108066 objects degraded (1.859%); 701/108066 objects misplaced (0.649%); 2.3 MiB/s, 114 objects/s recovering 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/799984235' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:22.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2009/108066 objects degraded (1.859%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:22.918 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:23.024 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:23.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:23.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:23.136 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:23.183 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:23.280 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:23.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:23.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:23.388 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:23.500 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:23.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:23.595 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:23.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:23.648 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:25:23.696 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:23.723 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:23 smithi150 ceph-mon[194178]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:23.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[212750]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: osdmap e283: 8 total, 8 up, 7 in 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.907 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:23.908 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:23 smithi107 ceph-mon[220921]: failed to encode map e283 with expected crc 2023-10-09T06:25:24.109 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:24.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:24.223 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:24.261 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:24.335 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:24.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:24.446 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:24.498 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:24.588 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:24.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:24.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:24 smithi107 ceph-mon[212750]: pgmap v872: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1757/99495 objects degraded (1.766%); 575/99495 objects misplaced (0.578%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:24.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:24 smithi107 ceph-mon[220921]: pgmap v872: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1757/99495 objects degraded (1.766%); 575/99495 objects misplaced (0.578%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:24.701 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:24.720 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:24.720 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:24.735 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:24.770 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:24 smithi150 ceph-mon[194178]: pgmap v872: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1757/99495 objects degraded (1.766%); 575/99495 objects misplaced (0.578%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:24.973 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:24.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:25.086 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:25.140 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:25.193 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:25.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:25.300 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:25.353 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:25.367 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:25:25.652 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:25.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:25.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:25.773 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:25.854 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:25:25.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:25.978 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:25.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:26.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:26.198 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:26.267 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:25:26.812 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:26.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:26.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:26 smithi107 ceph-mon[212750]: pgmap v873: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 1757/98754 objects degraded (1.779%); 575/98754 objects misplaced (0.582%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:26.918 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:26 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:26.920 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:26.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:26 smithi107 ceph-mon[220921]: pgmap v873: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 1757/98754 objects degraded (1.779%); 575/98754 objects misplaced (0.582%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:26.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:26 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:26.972 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:27.075 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:27.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:26 smithi150 ceph-mon[194178]: pgmap v873: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 830 op/s; 1757/98754 objects degraded (1.779%); 575/98754 objects misplaced (0.582%); 2.4 MiB/s, 115 objects/s recovering 2023-10-09T06:25:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:26 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:25:27.184 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:27.227 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:27.261 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:27.410 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:27.485 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:27.485 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:27.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:27.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:27.591 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:27.625 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:27.756 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:27.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:27.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[220921]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T06:25:27.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[220921]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[220921]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[220921]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.766 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1757/98754 objects degraded (1.779%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:27.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[212750]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T06:25:27.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[212750]: failed to encode map e284 with expected crc 2023-10-09T06:25:27.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1757/98754 objects degraded (1.779%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:27.844 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:25:27.866 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:27.872 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:25:27.910 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:27.989 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:28.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:28.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:27 smithi150 ceph-mon[194178]: osdmap e284: 8 total, 8 up, 7 in 2023-10-09T06:25:28.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:27 smithi150 ceph-mon[194178]: failed to encode map e284 with expected crc 2023-10-09T06:25:28.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:27 smithi150 ceph-mon[194178]: failed to encode map e284 with expected crc 2023-10-09T06:25:28.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:27 smithi150 ceph-mon[194178]: failed to encode map e284 with expected crc 2023-10-09T06:25:28.009 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1757/98754 objects degraded (1.779%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:25:28.109 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:28.229 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: pgmap v875: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 93/93105 objects degraded (0.100%); 1335/93105 objects misplaced (1.434%); 2.0 MiB/s, 153 objects/s recovering 2023-10-09T06:25:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T06:25:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[220921]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: pgmap v875: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 93/93105 objects degraded (0.100%); 1335/93105 objects misplaced (1.434%); 2.0 MiB/s, 153 objects/s recovering 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: failed to encode map e285 with expected crc 2023-10-09T06:25:28.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:28 smithi107 ceph-mon[212750]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:28.966 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:28.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:29.073 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:29.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: pgmap v875: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 93/93105 objects degraded (0.100%); 1335/93105 objects misplaced (1.434%); 2.0 MiB/s, 153 objects/s recovering 2023-10-09T06:25:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: failed to encode map e285 with expected crc 2023-10-09T06:25:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: osdmap e285: 8 total, 8 up, 7 in 2023-10-09T06:25:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: failed to encode map e285 with expected crc 2023-10-09T06:25:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: failed to encode map e285 with expected crc 2023-10-09T06:25:29.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:28 smithi150 ceph-mon[194178]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:29.101 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:29.199 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:29.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:29.305 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:29.382 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:29.998 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":876,"stamp":"2023-10-09T06:25:27.682939+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2032736184,"num_objects":31035,"num_object_clones":0,"num_object_copies":93105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":1335,"num_objects_unfound":0,"num_objects_dirty":31035,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158596,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10614,"num_bytes_recovered":609757274,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200215,"ondisk_log_size":200215,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":334,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11513732,"kb_used_data":6937164,"kb_used_omap":20,"kb_used_meta":4576427,"kb_avail":644702332,"statfs":{"total":671965249536,"available":660175187968,"internally_reserved":0,"allocated":7103655936,"data_stored":7096256602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4686261902},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-326828032,"num_objects":-4987,"num_object_clones":0,"num_object_copies":-14961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1916,"num_objects_misplaced":634,"num_objects_unfound":0,"num_objects_dirty":-4987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4987,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":770,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.573884"},"pg_stats":[{"pgid":"10.13","version":"284'8171","reported_seq":16678,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.865585+0000","last_change":"2023-10-09T06:24:52.774608+0000","last_active":"2023-10-09T06:25:26.865585+0000","last_peered":"2023-10-09T06:25:26.865585+0000","last_clean":"2023-10-09T06:25:26.865585+0000","last_became_active":"2023-10-09T06:24:52.762870+0000","last_became_peered":"2023-10-09T06:24:52.762870+0000","last_unstale":"2023-10-09T06:25:26.865585+0000","last_undegraded":"2023-10-09T06:25:26.865585+0000","last_fullsized":"2023-10-09T06:25:26.865585+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61931520,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5591,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2810,"num_bytes_recovered":184156160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932530+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:25:11.932530+0000","last_peered":"2023-10-09T06:25:11.932530+0000","last_clean":"2023-10-09T06:25:11.932530+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:25:11.932530+0000","last_undegraded":"2023-10-09T06:25:11.932530+0000","last_fullsized":"2023-10-09T06:25:11.932530+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.960859+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:25:11.960859+0000","last_peered":"2023-10-09T06:25:11.960859+0000","last_clean":"2023-10-09T06:25:11.960859+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:25:11.960859+0000","last_undegraded":"2023-10-09T06:25:11.960859+0000","last_fullsized":"2023-10-09T06:25:11.960859+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"283'8028","reported_seq":13603,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:23.002474+0000","last_change":"2023-10-09T06:25:08.543126+0000","last_active":"2023-10-09T06:25:23.002474+0000","last_peered":"2023-10-09T06:25:23.002474+0000","last_clean":"2023-10-09T06:25:23.002474+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:25:23.002474+0000","last_undegraded":"2023-10-09T06:25:23.002474+0000","last_fullsized":"2023-10-09T06:25:23.002474+0000","mapping_epoch":269,"log_start":"242'157","ondisk_log_start":"242'157","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7871,"log_dups_size":0,"ondisk_log_size":7871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5470,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":256,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.643494+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:25:22.643494+0000","last_peered":"2023-10-09T06:25:22.643494+0000","last_clean":"2023-10-09T06:25:22.643494+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:25:22.643494+0000","last_undegraded":"2023-10-09T06:25:22.643494+0000","last_fullsized":"2023-10-09T06:25:22.643494+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":254,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925404+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:25:11.925404+0000","last_peered":"2023-10-09T06:25:11.925404+0000","last_clean":"2023-10-09T06:25:11.925404+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:25:11.925404+0000","last_undegraded":"2023-10-09T06:25:11.925404+0000","last_fullsized":"2023-10-09T06:25:11.925404+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"283'8214","reported_seq":13706,"reported_epoch":283,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:25:25.318392+0000","last_change":"2023-10-09T06:24:51.087762+0000","last_active":"2023-10-09T06:25:25.318392+0000","last_peered":"2023-10-09T06:25:25.318392+0000","last_clean":"2023-10-09T06:23:34.462031+0000","last_became_active":"2023-10-09T06:23:38.083231+0000","last_became_peered":"2023-10-09T06:23:38.083231+0000","last_unstale":"2023-10-09T06:25:25.318392+0000","last_undegraded":"2023-10-09T06:23:38.014323+0000","last_fullsized":"2023-10-09T06:23:37.903678+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8214,"log_dups_size":0,"ondisk_log_size":8214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62128128,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5596,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2712,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,7","objects":855}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963457+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:25:11.963457+0000","last_peered":"2023-10-09T06:25:11.963457+0000","last_clean":"2023-10-09T06:25:11.963457+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:25:11.963457+0000","last_undegraded":"2023-10-09T06:25:11.963457+0000","last_fullsized":"2023-10-09T06:25:11.963457+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931456+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:25:11.931456+0000","last_peered":"2023-10-09T06:25:11.931456+0000","last_clean":"2023-10-09T06:25:11.931456+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:25:11.931456+0000","last_undegraded":"2023-10-09T06:25:11.931456+0000","last_fullsized":"2023-10-09T06:25:11.931456+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":244,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925578+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:25:11.925578+0000","last_peered":"2023-10-09T06:25:11.925578+0000","last_clean":"2023-10-09T06:25:11.925578+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:25:11.925578+0000","last_undegraded":"2023-10-09T06:25:11.925578+0000","last_fullsized":"2023-10-09T06:25:11.925578+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963365+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:25:11.963365+0000","last_peered":"2023-10-09T06:25:11.963365+0000","last_clean":"2023-10-09T06:25:11.963365+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:25:11.963365+0000","last_undegraded":"2023-10-09T06:25:11.963365+0000","last_fullsized":"2023-10-09T06:25:11.963365+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":211,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937755+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:25:11.937755+0000","last_peered":"2023-10-09T06:25:11.937755+0000","last_clean":"2023-10-09T06:25:11.937755+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:25:11.937755+0000","last_undegraded":"2023-10-09T06:25:11.937755+0000","last_fullsized":"2023-10-09T06:25:11.937755+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":218,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.962732+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:25:11.962732+0000","last_peered":"2023-10-09T06:25:11.962732+0000","last_clean":"2023-10-09T06:25:11.962732+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:25:11.962732+0000","last_undegraded":"2023-10-09T06:25:11.962732+0000","last_fullsized":"2023-10-09T06:25:11.962732+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":132,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937445+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:25:11.937445+0000","last_peered":"2023-10-09T06:25:11.937445+0000","last_clean":"2023-10-09T06:25:11.937445+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:25:11.937445+0000","last_undegraded":"2023-10-09T06:25:11.937445+0000","last_fullsized":"2023-10-09T06:25:11.937445+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":215,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.652590+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:25:22.652590+0000","last_peered":"2023-10-09T06:25:22.652590+0000","last_clean":"2023-10-09T06:25:22.652590+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:25:22.652590+0000","last_undegraded":"2023-10-09T06:25:22.652590+0000","last_fullsized":"2023-10-09T06:25:22.652590+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":289,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932963+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:25:11.932963+0000","last_peered":"2023-10-09T06:25:11.932963+0000","last_clean":"2023-10-09T06:25:11.932963+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:25:11.932963+0000","last_undegraded":"2023-10-09T06:25:11.932963+0000","last_fullsized":"2023-10-09T06:25:11.932963+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":253,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932809+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:25:11.932809+0000","last_peered":"2023-10-09T06:25:11.932809+0000","last_clean":"2023-10-09T06:25:11.932809+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:25:11.932809+0000","last_undegraded":"2023-10-09T06:25:11.932809+0000","last_fullsized":"2023-10-09T06:25:11.932809+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":132,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937606+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:25:11.937606+0000","last_peered":"2023-10-09T06:25:11.937606+0000","last_clean":"2023-10-09T06:25:11.937606+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:25:11.937606+0000","last_undegraded":"2023-10-09T06:25:11.937606+0000","last_fullsized":"2023-10-09T06:25:11.937606+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.962899+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:25:11.962899+0000","last_peered":"2023-10-09T06:25:11.962899+0000","last_clean":"2023-10-09T06:25:11.962899+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:25:11.962899+0000","last_undegraded":"2023-10-09T06:25:11.962899+0000","last_fullsized":"2023-10-09T06:25:11.962899+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.927520+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:25:11.927520+0000","last_peered":"2023-10-09T06:25:11.927520+0000","last_clean":"2023-10-09T06:25:11.927520+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:25:11.927520+0000","last_undegraded":"2023-10-09T06:25:11.927520+0000","last_fullsized":"2023-10-09T06:25:11.927520+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937292+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:25:11.937292+0000","last_peered":"2023-10-09T06:25:11.937292+0000","last_clean":"2023-10-09T06:25:11.937292+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:25:11.937292+0000","last_undegraded":"2023-10-09T06:25:11.937292+0000","last_fullsized":"2023-10-09T06:25:11.937292+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"283'8073","reported_seq":8136,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:25.138164+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:25:25.138164+0000","last_peered":"2023-10-09T06:25:25.138164+0000","last_clean":"2023-10-09T06:25:25.138164+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:25:25.138164+0000","last_undegraded":"2023-10-09T06:25:25.138164+0000","last_fullsized":"2023-10-09T06:25:25.138164+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8073,"log_dups_size":0,"ondisk_log_size":8073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60588032,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":126,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.943885+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:25:11.943885+0000","last_peered":"2023-10-09T06:25:11.943885+0000","last_clean":"2023-10-09T06:25:11.943885+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:25:11.943885+0000","last_undegraded":"2023-10-09T06:25:11.943885+0000","last_fullsized":"2023-10-09T06:25:11.943885+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":253,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925535+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:25:11.925535+0000","last_peered":"2023-10-09T06:25:11.925535+0000","last_clean":"2023-10-09T06:25:11.925535+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:25:11.925535+0000","last_undegraded":"2023-10-09T06:25:11.925535+0000","last_fullsized":"2023-10-09T06:25:11.925535+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"283'8147","reported_seq":8211,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.309558+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:25:26.309558+0000","last_peered":"2023-10-09T06:25:26.309558+0000","last_clean":"2023-10-09T06:25:26.309558+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:25:26.309558+0000","last_undegraded":"2023-10-09T06:25:26.309558+0000","last_fullsized":"2023-10-09T06:25:26.309558+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8147,"log_dups_size":0,"ondisk_log_size":8147,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60653568,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5582,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":271,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.930787+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:25:11.930787+0000","last_peered":"2023-10-09T06:25:11.930787+0000","last_clean":"2023-10-09T06:25:11.930787+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:25:11.930787+0000","last_undegraded":"2023-10-09T06:25:11.930787+0000","last_fullsized":"2023-10-09T06:25:11.930787+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.962426+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:25:11.962426+0000","last_peered":"2023-10-09T06:25:11.962426+0000","last_clean":"2023-10-09T06:25:11.962426+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:25:11.962426+0000","last_undegraded":"2023-10-09T06:25:11.962426+0000","last_fullsized":"2023-10-09T06:25:11.962426+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"283'8159","reported_seq":8292,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:25.147300+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:25:25.147300+0000","last_peered":"2023-10-09T06:25:25.147300+0000","last_clean":"2023-10-09T06:25:25.147300+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:25:25.147300+0000","last_undegraded":"2023-10-09T06:25:25.147300+0000","last_fullsized":"2023-10-09T06:25:25.147300+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":8159,"log_dups_size":0,"ondisk_log_size":8159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62128128,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2897,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":144,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937675+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:25:11.937675+0000","last_peered":"2023-10-09T06:25:11.937675+0000","last_clean":"2023-10-09T06:25:11.937675+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:25:11.937675+0000","last_undegraded":"2023-10-09T06:25:11.937675+0000","last_fullsized":"2023-10-09T06:25:11.937675+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":193,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937220+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:25:11.937220+0000","last_peered":"2023-10-09T06:25:11.937220+0000","last_clean":"2023-10-09T06:25:11.937220+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:25:11.937220+0000","last_undegraded":"2023-10-09T06:25:11.937220+0000","last_fullsized":"2023-10-09T06:25:11.937220+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"283'9824","reported_seq":10004,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.889237+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:25:22.889237+0000","last_peered":"2023-10-09T06:25:22.889237+0000","last_clean":"2023-10-09T06:25:22.889237+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:25:22.889237+0000","last_undegraded":"2023-10-09T06:25:22.889237+0000","last_fullsized":"2023-10-09T06:25:22.889237+0000","mapping_epoch":238,"log_start":"246'2476","ondisk_log_start":"246'2476","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63832064,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2722,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":130,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937676+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:25:11.937676+0000","last_peered":"2023-10-09T06:25:11.937676+0000","last_clean":"2023-10-09T06:25:11.937676+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:25:11.937676+0000","last_undegraded":"2023-10-09T06:25:11.937676+0000","last_fullsized":"2023-10-09T06:25:11.937676+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932566+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:25:11.932566+0000","last_peered":"2023-10-09T06:25:11.932566+0000","last_clean":"2023-10-09T06:25:11.932566+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:25:11.932566+0000","last_undegraded":"2023-10-09T06:25:11.932566+0000","last_fullsized":"2023-10-09T06:25:11.932566+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"283'8305","reported_seq":8370,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:25.081320+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:25:25.081320+0000","last_peered":"2023-10-09T06:25:25.081320+0000","last_clean":"2023-10-09T06:25:25.081320+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:25:25.081320+0000","last_undegraded":"2023-10-09T06:25:25.081320+0000","last_fullsized":"2023-10-09T06:25:25.081320+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8305,"log_dups_size":0,"ondisk_log_size":8305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55476224,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":144,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937212+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:25:11.937212+0000","last_peered":"2023-10-09T06:25:11.937212+0000","last_clean":"2023-10-09T06:25:11.937212+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:25:11.937212+0000","last_undegraded":"2023-10-09T06:25:11.937212+0000","last_fullsized":"2023-10-09T06:25:11.937212+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931375+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:25:11.931375+0000","last_peered":"2023-10-09T06:25:11.931375+0000","last_clean":"2023-10-09T06:25:11.931375+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:25:11.931375+0000","last_undegraded":"2023-10-09T06:25:11.931375+0000","last_fullsized":"2023-10-09T06:25:11.931375+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"283'8160","reported_seq":8225,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.305887+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:25:26.305887+0000","last_peered":"2023-10-09T06:25:26.305887+0000","last_clean":"2023-10-09T06:25:26.305887+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:25:26.305887+0000","last_undegraded":"2023-10-09T06:25:26.305887+0000","last_fullsized":"2023-10-09T06:25:26.305887+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8160,"log_dups_size":0,"ondisk_log_size":8160,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52166656,"num_objects":796,"num_object_clones":0,"num_object_copies":2388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5618,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963647+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:25:11.963647+0000","last_peered":"2023-10-09T06:25:11.963647+0000","last_clean":"2023-10-09T06:25:11.963647+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:25:11.963647+0000","last_undegraded":"2023-10-09T06:25:11.963647+0000","last_fullsized":"2023-10-09T06:25:11.963647+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":235,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.927771+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:25:11.927771+0000","last_peered":"2023-10-09T06:25:11.927771+0000","last_clean":"2023-10-09T06:25:11.927771+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:25:11.927771+0000","last_undegraded":"2023-10-09T06:25:11.927771+0000","last_fullsized":"2023-10-09T06:25:11.927771+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"284'8130","reported_seq":8266,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.816926+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:25:26.816926+0000","last_peered":"2023-10-09T06:25:26.816926+0000","last_clean":"2023-10-09T06:25:26.816926+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:25:26.816926+0000","last_undegraded":"2023-10-09T06:25:26.816926+0000","last_fullsized":"2023-10-09T06:25:26.816926+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61997056,"num_objects":946,"num_object_clones":0,"num_object_copies":2838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2836,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":575,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.944501+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:25:11.944501+0000","last_peered":"2023-10-09T06:25:11.944501+0000","last_clean":"2023-10-09T06:25:11.944501+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:25:11.944501+0000","last_undegraded":"2023-10-09T06:25:11.944501+0000","last_fullsized":"2023-10-09T06:25:11.944501+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":246,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.960513+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:25:11.960513+0000","last_peered":"2023-10-09T06:25:11.960513+0000","last_clean":"2023-10-09T06:25:11.960513+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:25:11.960513+0000","last_undegraded":"2023-10-09T06:25:11.960513+0000","last_fullsized":"2023-10-09T06:25:11.960513+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932920+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:25:11.932920+0000","last_peered":"2023-10-09T06:25:11.932920+0000","last_clean":"2023-10-09T06:25:11.932920+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:25:11.932920+0000","last_undegraded":"2023-10-09T06:25:11.932920+0000","last_fullsized":"2023-10-09T06:25:11.932920+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"283'8296","reported_seq":8361,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.342354+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:25:26.342354+0000","last_peered":"2023-10-09T06:25:26.342354+0000","last_clean":"2023-10-09T06:25:26.342354+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:25:26.342354+0000","last_undegraded":"2023-10-09T06:25:26.342354+0000","last_fullsized":"2023-10-09T06:25:26.342354+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8296,"log_dups_size":0,"ondisk_log_size":8296,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61997056,"num_objects":946,"num_object_clones":0,"num_object_copies":2838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5716,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":566,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.935108+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:25:11.935108+0000","last_peered":"2023-10-09T06:25:11.935108+0000","last_clean":"2023-10-09T06:25:11.935108+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:25:11.935108+0000","last_undegraded":"2023-10-09T06:25:11.935108+0000","last_fullsized":"2023-10-09T06:25:11.935108+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931047+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:25:11.931047+0000","last_peered":"2023-10-09T06:25:11.931047+0000","last_clean":"2023-10-09T06:25:11.931047+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:25:11.931047+0000","last_undegraded":"2023-10-09T06:25:11.931047+0000","last_fullsized":"2023-10-09T06:25:11.931047+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932785+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:25:11.932785+0000","last_peered":"2023-10-09T06:25:11.932785+0000","last_clean":"2023-10-09T06:25:11.932785+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:25:11.932785+0000","last_undegraded":"2023-10-09T06:25:11.932785+0000","last_fullsized":"2023-10-09T06:25:11.932785+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"283'8063","reported_seq":8241,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.823146+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:25:22.823146+0000","last_peered":"2023-10-09T06:25:22.823146+0000","last_clean":"2023-10-09T06:25:22.823146+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:25:22.823146+0000","last_undegraded":"2023-10-09T06:25:22.823146+0000","last_fullsized":"2023-10-09T06:25:22.823146+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65404928,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2928,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":264,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932484+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:25:11.932484+0000","last_peered":"2023-10-09T06:25:11.932484+0000","last_clean":"2023-10-09T06:25:11.932484+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:25:11.932484+0000","last_undegraded":"2023-10-09T06:25:11.932484+0000","last_fullsized":"2023-10-09T06:25:11.932484+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.933016+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:25:11.933016+0000","last_peered":"2023-10-09T06:25:11.933016+0000","last_clean":"2023-10-09T06:25:11.933016+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:25:11.933016+0000","last_undegraded":"2023-10-09T06:25:11.933016+0000","last_fullsized":"2023-10-09T06:25:11.933016+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"283'8261","reported_seq":8446,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.998990+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:25:22.998990+0000","last_peered":"2023-10-09T06:25:22.998990+0000","last_clean":"2023-10-09T06:25:22.998990+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:25:22.998990+0000","last_undegraded":"2023-10-09T06:25:22.998990+0000","last_fullsized":"2023-10-09T06:25:22.998990+0000","mapping_epoch":238,"log_start":"242'614","ondisk_log_start":"242'614","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":7647,"log_dups_size":0,"ondisk_log_size":7647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2836,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":562,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.934848+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:25:11.934848+0000","last_peered":"2023-10-09T06:25:11.934848+0000","last_clean":"2023-10-09T06:25:11.934848+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:25:11.934848+0000","last_undegraded":"2023-10-09T06:25:11.934848+0000","last_fullsized":"2023-10-09T06:25:11.934848+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":240,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.642954+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:25:22.642954+0000","last_peered":"2023-10-09T06:25:22.642954+0000","last_clean":"2023-10-09T06:25:22.642954+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:25:22.642954+0000","last_undegraded":"2023-10-09T06:25:22.642954+0000","last_fullsized":"2023-10-09T06:25:22.642954+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":215,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.643998+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:25:22.643998+0000","last_peered":"2023-10-09T06:25:22.643998+0000","last_clean":"2023-10-09T06:25:22.643998+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:25:22.643998+0000","last_undegraded":"2023-10-09T06:25:22.643998+0000","last_fullsized":"2023-10-09T06:25:22.643998+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"283'10730","reported_seq":10783,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:23.126135+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:25:23.126135+0000","last_peered":"2023-10-09T06:25:23.126135+0000","last_clean":"2023-10-09T06:25:23.126135+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:25:23.126135+0000","last_undegraded":"2023-10-09T06:25:23.126135+0000","last_fullsized":"2023-10-09T06:25:23.126135+0000","mapping_epoch":238,"log_start":"249'4000","ondisk_log_start":"249'4000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129105920,"num_objects":1970,"num_object_clones":0,"num_object_copies":5910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10730,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":505,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931515+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:25:11.931515+0000","last_peered":"2023-10-09T06:25:11.931515+0000","last_clean":"2023-10-09T06:25:11.931515+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:25:11.931515+0000","last_undegraded":"2023-10-09T06:25:11.931515+0000","last_fullsized":"2023-10-09T06:25:11.931515+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":266,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.644093+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:25:22.644093+0000","last_peered":"2023-10-09T06:25:22.644093+0000","last_clean":"2023-10-09T06:25:22.644093+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:25:22.644093+0000","last_undegraded":"2023-10-09T06:25:22.644093+0000","last_fullsized":"2023-10-09T06:25:22.644093+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.960751+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:25:11.960751+0000","last_peered":"2023-10-09T06:25:11.960751+0000","last_clean":"2023-10-09T06:25:11.960751+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:25:11.960751+0000","last_undegraded":"2023-10-09T06:25:11.960751+0000","last_fullsized":"2023-10-09T06:25:11.960751+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":209,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.652194+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:25:22.652194+0000","last_peered":"2023-10-09T06:25:22.652194+0000","last_clean":"2023-10-09T06:25:22.652194+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:25:22.652194+0000","last_undegraded":"2023-10-09T06:25:22.652194+0000","last_fullsized":"2023-10-09T06:25:22.652194+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"283'11113","reported_seq":11166,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.911318+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:25:22.911318+0000","last_peered":"2023-10-09T06:25:22.911318+0000","last_clean":"2023-10-09T06:25:22.911318+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:25:22.911318+0000","last_undegraded":"2023-10-09T06:25:22.911318+0000","last_fullsized":"2023-10-09T06:25:22.911318+0000","mapping_epoch":238,"log_start":"252'5100","ondisk_log_start":"252'5100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11113,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":568,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.637543+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:25:22.637543+0000","last_peered":"2023-10-09T06:25:22.637543+0000","last_clean":"2023-10-09T06:25:22.637543+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:25:22.637543+0000","last_undegraded":"2023-10-09T06:25:22.637543+0000","last_fullsized":"2023-10-09T06:25:22.637543+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":140,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.652326+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:25:22.652326+0000","last_peered":"2023-10-09T06:25:22.652326+0000","last_clean":"2023-10-09T06:25:22.652326+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:25:22.652326+0000","last_undegraded":"2023-10-09T06:25:22.652326+0000","last_fullsized":"2023-10-09T06:25:22.652326+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":244,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932710+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:25:11.932710+0000","last_peered":"2023-10-09T06:25:11.932710+0000","last_clean":"2023-10-09T06:25:11.932710+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:25:11.932710+0000","last_undegraded":"2023-10-09T06:25:11.932710+0000","last_fullsized":"2023-10-09T06:25:11.932710+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.644018+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:25:22.644018+0000","last_peered":"2023-10-09T06:25:22.644018+0000","last_clean":"2023-10-09T06:25:22.644018+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:25:22.644018+0000","last_undegraded":"2023-10-09T06:25:22.644018+0000","last_fullsized":"2023-10-09T06:25:22.644018+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"283'8239","reported_seq":8368,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.363462+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:25:26.363462+0000","last_peered":"2023-10-09T06:25:26.363462+0000","last_clean":"2023-10-09T06:25:26.363462+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:25:26.363462+0000","last_undegraded":"2023-10-09T06:25:26.363462+0000","last_fullsized":"2023-10-09T06:25:26.363462+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":8239,"log_dups_size":0,"ondisk_log_size":8239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63307776,"num_objects":966,"num_object_clones":0,"num_object_copies":2898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3067,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":575,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937790+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:25:11.937790+0000","last_peered":"2023-10-09T06:25:11.937790+0000","last_clean":"2023-10-09T06:25:11.937790+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:25:11.937790+0000","last_undegraded":"2023-10-09T06:25:11.937790+0000","last_fullsized":"2023-10-09T06:25:11.937790+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":248,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.934780+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:25:11.934780+0000","last_peered":"2023-10-09T06:25:11.934780+0000","last_clean":"2023-10-09T06:25:11.934780+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:25:11.934780+0000","last_undegraded":"2023-10-09T06:25:11.934780+0000","last_fullsized":"2023-10-09T06:25:11.934780+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":257,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.643774+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:25:22.643774+0000","last_peered":"2023-10-09T06:25:22.643774+0000","last_clean":"2023-10-09T06:25:22.643774+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:25:22.643774+0000","last_undegraded":"2023-10-09T06:25:22.643774+0000","last_fullsized":"2023-10-09T06:25:22.643774+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":252,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.644070+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:25:22.644070+0000","last_peered":"2023-10-09T06:25:22.644070+0000","last_clean":"2023-10-09T06:25:22.644070+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:25:22.644070+0000","last_undegraded":"2023-10-09T06:25:22.644070+0000","last_fullsized":"2023-10-09T06:25:22.644070+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"283'11047","reported_seq":11100,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.890803+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:25:22.890803+0000","last_peered":"2023-10-09T06:25:22.890803+0000","last_clean":"2023-10-09T06:25:22.890803+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:25:22.890803+0000","last_undegraded":"2023-10-09T06:25:22.890803+0000","last_fullsized":"2023-10-09T06:25:22.890803+0000","mapping_epoch":238,"log_start":"252'5000","ondisk_log_start":"252'5000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6047,"log_dups_size":0,"ondisk_log_size":6047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11047,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":215,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937955+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:25:11.937955+0000","last_peered":"2023-10-09T06:25:11.937955+0000","last_clean":"2023-10-09T06:25:11.937955+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:25:11.937955+0000","last_undegraded":"2023-10-09T06:25:11.937955+0000","last_fullsized":"2023-10-09T06:25:11.937955+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":556,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937353+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:25:11.937353+0000","last_peered":"2023-10-09T06:25:11.937353+0000","last_clean":"2023-10-09T06:25:11.937353+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:25:11.937353+0000","last_undegraded":"2023-10-09T06:25:11.937353+0000","last_fullsized":"2023-10-09T06:25:11.937353+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":210,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.964412+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:25:11.964412+0000","last_peered":"2023-10-09T06:25:11.964412+0000","last_clean":"2023-10-09T06:25:11.964412+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:25:11.964412+0000","last_undegraded":"2023-10-09T06:25:11.964412+0000","last_fullsized":"2023-10-09T06:25:11.964412+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":245,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931214+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:25:11.931214+0000","last_peered":"2023-10-09T06:25:11.931214+0000","last_clean":"2023-10-09T06:25:11.931214+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:25:11.931214+0000","last_undegraded":"2023-10-09T06:25:11.931214+0000","last_fullsized":"2023-10-09T06:25:11.931214+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"283'8038","reported_seq":8166,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:24.993362+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:25:24.993362+0000","last_peered":"2023-10-09T06:25:24.993362+0000","last_clean":"2023-10-09T06:25:24.993362+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:25:24.993362+0000","last_undegraded":"2023-10-09T06:25:24.993362+0000","last_fullsized":"2023-10-09T06:25:24.993362+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":8038,"log_dups_size":0,"ondisk_log_size":8038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60948480,"num_objects":930,"num_object_clones":0,"num_object_copies":2790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2860,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":566,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.649904+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:25:22.649904+0000","last_peered":"2023-10-09T06:25:22.649904+0000","last_clean":"2023-10-09T06:25:22.649904+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:25:22.649904+0000","last_undegraded":"2023-10-09T06:25:22.649904+0000","last_fullsized":"2023-10-09T06:25:22.649904+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":126,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937098+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:25:11.937098+0000","last_peered":"2023-10-09T06:25:11.937098+0000","last_clean":"2023-10-09T06:25:11.937098+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:25:11.937098+0000","last_undegraded":"2023-10-09T06:25:11.937098+0000","last_fullsized":"2023-10-09T06:25:11.937098+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925966+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:25:11.925966+0000","last_peered":"2023-10-09T06:25:11.925966+0000","last_clean":"2023-10-09T06:25:11.925966+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:25:11.925966+0000","last_undegraded":"2023-10-09T06:25:11.925966+0000","last_fullsized":"2023-10-09T06:25:11.925966+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":258,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925470+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:25:11.925470+0000","last_peered":"2023-10-09T06:25:11.925470+0000","last_clean":"2023-10-09T06:25:11.925470+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:25:11.925470+0000","last_undegraded":"2023-10-09T06:25:11.925470+0000","last_fullsized":"2023-10-09T06:25:11.925470+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"283'11161","reported_seq":11215,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.916148+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:25:22.916148+0000","last_peered":"2023-10-09T06:25:22.916148+0000","last_clean":"2023-10-09T06:25:22.916148+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:25:22.916148+0000","last_undegraded":"2023-10-09T06:25:22.916148+0000","last_fullsized":"2023-10-09T06:25:22.916148+0000","mapping_epoch":238,"log_start":"253'5100","ondisk_log_start":"253'5100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134479896,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11161,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":505,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931143+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:25:11.931143+0000","last_peered":"2023-10-09T06:25:11.931143+0000","last_clean":"2023-10-09T06:25:11.931143+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:25:11.931143+0000","last_undegraded":"2023-10-09T06:25:11.931143+0000","last_fullsized":"2023-10-09T06:25:11.931143+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":126,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.937363+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:25:11.937363+0000","last_peered":"2023-10-09T06:25:11.937363+0000","last_clean":"2023-10-09T06:25:11.937363+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:25:11.937363+0000","last_undegraded":"2023-10-09T06:25:11.937363+0000","last_fullsized":"2023-10-09T06:25:11.937363+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":265,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.643943+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:25:22.643943+0000","last_peered":"2023-10-09T06:25:22.643943+0000","last_clean":"2023-10-09T06:25:22.643943+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:25:22.643943+0000","last_undegraded":"2023-10-09T06:25:22.643943+0000","last_fullsized":"2023-10-09T06:25:22.643943+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":248,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.928007+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:25:11.928007+0000","last_peered":"2023-10-09T06:25:11.928007+0000","last_clean":"2023-10-09T06:25:11.928007+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:25:11.928007+0000","last_undegraded":"2023-10-09T06:25:11.928007+0000","last_fullsized":"2023-10-09T06:25:11.928007+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"283'10889","reported_seq":10942,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:23.006969+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:25:23.006969+0000","last_peered":"2023-10-09T06:25:23.006969+0000","last_clean":"2023-10-09T06:25:23.006969+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:25:23.006969+0000","last_undegraded":"2023-10-09T06:25:23.006969+0000","last_fullsized":"2023-10-09T06:25:23.006969+0000","mapping_epoch":238,"log_start":"257'5500","ondisk_log_start":"257'5500","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5389,"log_dups_size":0,"ondisk_log_size":5389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128909312,"num_objects":1967,"num_object_clones":0,"num_object_copies":5901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10889,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":566,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932677+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:25:11.932677+0000","last_peered":"2023-10-09T06:25:11.932677+0000","last_clean":"2023-10-09T06:25:11.932677+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:25:11.932677+0000","last_undegraded":"2023-10-09T06:25:11.932677+0000","last_fullsized":"2023-10-09T06:25:11.932677+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":273,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.645971+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:25:22.645971+0000","last_peered":"2023-10-09T06:25:22.645971+0000","last_clean":"2023-10-09T06:25:22.645971+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:25:22.645971+0000","last_undegraded":"2023-10-09T06:25:22.645971+0000","last_fullsized":"2023-10-09T06:25:22.645971+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":242,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.646071+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:25:22.646071+0000","last_peered":"2023-10-09T06:25:22.646071+0000","last_clean":"2023-10-09T06:25:22.646071+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:25:22.646071+0000","last_undegraded":"2023-10-09T06:25:22.646071+0000","last_fullsized":"2023-10-09T06:25:22.646071+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963149+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:25:11.963149+0000","last_peered":"2023-10-09T06:25:11.963149+0000","last_clean":"2023-10-09T06:25:11.963149+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:25:11.963149+0000","last_undegraded":"2023-10-09T06:25:11.963149+0000","last_fullsized":"2023-10-09T06:25:11.963149+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"283'11131","reported_seq":11184,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.905957+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:25:22.905957+0000","last_peered":"2023-10-09T06:25:22.905957+0000","last_clean":"2023-10-09T06:25:22.905957+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:25:22.905957+0000","last_undegraded":"2023-10-09T06:25:22.905957+0000","last_fullsized":"2023-10-09T06:25:22.905957+0000","mapping_epoch":238,"log_start":"252'5100","ondisk_log_start":"252'5100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6031,"log_dups_size":0,"ondisk_log_size":6031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131530752,"num_objects":2007,"num_object_clones":0,"num_object_copies":6021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11131,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":586,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963734+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:25:11.963734+0000","last_peered":"2023-10-09T06:25:11.963734+0000","last_clean":"2023-10-09T06:25:11.963734+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:25:11.963734+0000","last_undegraded":"2023-10-09T06:25:11.963734+0000","last_fullsized":"2023-10-09T06:25:11.963734+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":267,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.931621+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:25:11.931621+0000","last_peered":"2023-10-09T06:25:11.931621+0000","last_clean":"2023-10-09T06:25:11.931621+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:25:11.931621+0000","last_undegraded":"2023-10-09T06:25:11.931621+0000","last_fullsized":"2023-10-09T06:25:11.931621+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":239,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925332+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:25:11.925332+0000","last_peered":"2023-10-09T06:25:11.925332+0000","last_clean":"2023-10-09T06:25:11.925332+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:25:11.925332+0000","last_undegraded":"2023-10-09T06:25:11.925332+0000","last_fullsized":"2023-10-09T06:25:11.925332+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932565+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:25:11.932565+0000","last_peered":"2023-10-09T06:25:11.932565+0000","last_clean":"2023-10-09T06:25:11.932565+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:25:11.932565+0000","last_undegraded":"2023-10-09T06:25:11.932565+0000","last_fullsized":"2023-10-09T06:25:11.932565+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"284'9057","reported_seq":18587,"reported_epoch":284,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:25:26.893902+0000","last_change":"2023-10-09T06:25:26.168793+0000","last_active":"2023-10-09T06:25:26.893902+0000","last_peered":"2023-10-09T06:25:26.893902+0000","last_clean":"2023-10-09T06:23:37.701201+0000","last_became_active":"2023-10-09T06:23:40.099906+0000","last_became_peered":"2023-10-09T06:23:40.099906+0000","last_unstale":"2023-10-09T06:25:26.893902+0000","last_undegraded":"2023-10-09T06:25:26.893902+0000","last_fullsized":"2023-10-09T06:23:39.910881+0000","mapping_epoch":262,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":9057,"log_dups_size":0,"ondisk_log_size":9057,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59146240,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1335,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153419776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[1,2],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":445},{"shards":"1,2","objects":458}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":241,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.927929+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:25:11.927929+0000","last_peered":"2023-10-09T06:25:11.927929+0000","last_clean":"2023-10-09T06:25:11.927929+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:25:11.927929+0000","last_undegraded":"2023-10-09T06:25:11.927929+0000","last_fullsized":"2023-10-09T06:25:11.927929+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":265,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.934624+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:25:11.934624+0000","last_peered":"2023-10-09T06:25:11.934624+0000","last_clean":"2023-10-09T06:25:11.934624+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:25:11.934624+0000","last_undegraded":"2023-10-09T06:25:11.934624+0000","last_fullsized":"2023-10-09T06:25:11.934624+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"283'8043","reported_seq":8106,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:26.336022+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:25:26.336022+0000","last_peered":"2023-10-09T06:25:26.336022+0000","last_clean":"2023-10-09T06:25:26.336022+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:25:26.336022+0000","last_undegraded":"2023-10-09T06:25:26.336022+0000","last_fullsized":"2023-10-09T06:25:26.336022+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8043,"log_dups_size":0,"ondisk_log_size":8043,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66912256,"num_objects":1021,"num_object_clones":0,"num_object_copies":3063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":88,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932720+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:25:11.932720+0000","last_peered":"2023-10-09T06:25:11.932720+0000","last_clean":"2023-10-09T06:25:11.932720+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:25:11.932720+0000","last_undegraded":"2023-10-09T06:25:11.932720+0000","last_fullsized":"2023-10-09T06:25:11.932720+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":270,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932647+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:25:11.932647+0000","last_peered":"2023-10-09T06:25:11.932647+0000","last_clean":"2023-10-09T06:25:11.932647+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:25:11.932647+0000","last_undegraded":"2023-10-09T06:25:11.932647+0000","last_fullsized":"2023-10-09T06:25:11.932647+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"283'8159","reported_seq":8223,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:25.142570+0000","last_change":"2023-10-09T06:23:26.834184+0000","last_active":"2023-10-09T06:25:25.142570+0000","last_peered":"2023-10-09T06:25:25.142570+0000","last_clean":"2023-10-09T06:25:25.142570+0000","last_became_active":"2023-10-09T06:23:26.832172+0000","last_became_peered":"2023-10-09T06:23:26.832172+0000","last_unstale":"2023-10-09T06:25:25.142570+0000","last_undegraded":"2023-10-09T06:25:25.142570+0000","last_fullsized":"2023-10-09T06:25:25.142570+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8159,"log_dups_size":0,"ondisk_log_size":8159,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68190208,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5557,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":88,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932821+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:25:11.932821+0000","last_peered":"2023-10-09T06:25:11.932821+0000","last_clean":"2023-10-09T06:25:11.932821+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:25:11.932821+0000","last_undegraded":"2023-10-09T06:25:11.932821+0000","last_fullsized":"2023-10-09T06:25:11.932821+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932809+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:25:11.932809+0000","last_peered":"2023-10-09T06:25:11.932809+0000","last_clean":"2023-10-09T06:25:11.932809+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:25:11.932809+0000","last_undegraded":"2023-10-09T06:25:11.932809+0000","last_fullsized":"2023-10-09T06:25:11.932809+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"283'7958","reported_seq":8021,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:22.724079+0000","last_change":"2023-10-09T06:23:26.832837+0000","last_active":"2023-10-09T06:25:22.724079+0000","last_peered":"2023-10-09T06:25:22.724079+0000","last_clean":"2023-10-09T06:25:22.724079+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:25:22.724079+0000","last_undegraded":"2023-10-09T06:25:22.724079+0000","last_fullsized":"2023-10-09T06:25:22.724079+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5429,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":241,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.960214+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:25:11.960214+0000","last_peered":"2023-10-09T06:25:11.960214+0000","last_clean":"2023-10-09T06:25:11.960214+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:25:11.960214+0000","last_undegraded":"2023-10-09T06:25:11.960214+0000","last_fullsized":"2023-10-09T06:25:11.960214+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":255,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.963576+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:25:11.963576+0000","last_peered":"2023-10-09T06:25:11.963576+0000","last_clean":"2023-10-09T06:25:11.963576+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:25:11.963576+0000","last_undegraded":"2023-10-09T06:25:11.963576+0000","last_fullsized":"2023-10-09T06:25:11.963576+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"283'8040","reported_seq":8104,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T06:25:25.003563+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:25:25.003563+0000","last_peered":"2023-10-09T06:25:25.003563+0000","last_clean":"2023-10-09T06:25:25.003563+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:25:25.003563+0000","last_undegraded":"2023-10-09T06:25:25.003563+0000","last_fullsized":"2023-10-09T06:25:25.003563+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8040,"log_dups_size":0,"ondisk_log_size":8040,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65372160,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5482,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":255,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.925454+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:25:11.925454+0000","last_peered":"2023-10-09T06:25:11.925454+0000","last_clean":"2023-10-09T06:25:11.925454+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:25:11.925454+0000","last_undegraded":"2023-10-09T06:25:11.925454+0000","last_fullsized":"2023-10-09T06:25:11.925454+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T06:25:11.932217+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:25:11.932217+0000","last_peered":"2023-10-09T06:25:11.932217+0000","last_clean":"2023-10-09T06:25:11.932217+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:25:11.932217+0000","last_undegraded":"2023-10-09T06:25:11.932217+0000","last_fullsized":"2023-10-09T06:25:11.932217+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2032140312,"num_objects":31011,"num_object_clones":0,"num_object_copies":93033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":1335,"num_objects_unfound":0,"num_objects_dirty":31011,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158535,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10586,"num_bytes_recovered":609419264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7002730496,"data_stored":7002718280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":198389,"ondisk_log_size":198389,"up":78,"acting":76,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234337,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1581708,"kb_used_data":673284,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":92163444,"statfs":{"total":95995035648,"available":94375366656,"internally_reserved":0,"allocated":689442816,"data_stored":688664751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3399999999999999,"5min":7.7510000000000003,"15min":3.4870000000000001},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":60.524999999999999,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.4130000000000003,"5min":8.1159999999999997,"15min":3.5529999999999999},"min":{"1min":0.27900000000000003,"5min":0.218,"15min":0.218},"max":{"1min":60.539000000000001,"5min":219.399,"15min":219.399},"last":0.58199999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:25:20 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":3.1869999999999998,"15min":2.6659999999999999},"min":{"1min":0.253,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.242,"5min":141.44300000000001,"15min":141.44300000000001},"last":3.5219999999999998},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.4289999999999998,"15min":2.4630000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.7629999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.88700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":17.384,"5min":10.077999999999999,"15min":4.5700000000000003},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":131.773,"5min":215.34800000000001,"15min":215.34800000000001},"last":3.7090000000000001},{"interface":"front","average":{"1min":18.056999999999999,"5min":10.329000000000001,"15min":4.742},"min":{"1min":0.35599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":132.28100000000001,"5min":215.869,"15min":215.869},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.9889999999999999,"15min":1.542},"min":{"1min":0.40999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.2149999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.77000000000000002},{"interface":"front","average":{"1min":2.012,"5min":3.1389999999999998,"15min":1.599},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.4550000000000001,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.44500000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.161,"15min":1.673},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":94.105999999999995,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.32800000000000001},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.218,"15min":1.4379999999999999},"min":{"1min":0.31900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":93.713999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.429,"15min":1.714},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9220000000000002,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.341,"5min":1.3140000000000001,"15min":1.429},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":5.7130000000000001,"5min":7.452,"15min":48.774999999999999},"last":0.41399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.494,"15min":1.359},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7889999999999997,"5min":8.4890000000000008,"15min":11.878},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.464,"15min":1.256},"min":{"1min":0.38600000000000001,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":5.2690000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.50900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495974,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1908484,"kb_used_data":1005308,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91836668,"statfs":{"total":95995035648,"available":94040748032,"internally_reserved":0,"allocated":1029435392,"data_stored":1028665907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":3.9129999999999998,"15min":3.1339999999999999},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":28.035,"5min":110.926,"15min":110.926},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.635,"5min":4.0149999999999997,"15min":3.3319999999999999},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":7.0499999999999998,"5min":110.911,"15min":110.911},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:19 2023","interfaces":[{"interface":"back","average":{"1min":6.5010000000000003,"5min":2.8690000000000002,"15min":3.5310000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":66.796999999999997,"5min":66.796999999999997,"15min":114.202},"last":1.135},{"interface":"front","average":{"1min":6.5460000000000003,"5min":2.5630000000000002,"15min":3.3530000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":66.704999999999998,"5min":66.704999999999998,"15min":114.179},"last":1.173}]},{"osd":2,"last update":"Mon Oct 9 06:24:30 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":3.484,"15min":2.6040000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.8419999999999996,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.4769999999999999,"5min":3.0939999999999999,"15min":2.5059999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.215999999999999,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":3.2149999999999999,"15min":1.8939999999999999},"min":{"1min":0.31900000000000001,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":11.148,"5min":110.358,"15min":110.358},"last":0.41799999999999998},{"interface":"front","average":{"1min":3.1680000000000001,"5min":3.5870000000000002,"15min":1.986},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":11.180999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":2.2040000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4870000000000001,"15min":1.2549999999999999},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":3.6480000000000001,"5min":17.643999999999998,"15min":17.643999999999998},"last":1.4390000000000001},{"interface":"front","average":{"1min":2.141,"5min":1.4990000000000001,"15min":1.256},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":6.8780000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.36799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.806,"15min":1.3460000000000001},"min":{"1min":0.39200000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":4.3620000000000001,"5min":22.379999999999999,"15min":22.379999999999999},"last":1.645},{"interface":"front","average":{"1min":1.597,"5min":1.6499999999999999,"15min":1.3740000000000001},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.2149999999999999,"5min":24.401,"15min":24.401},"last":2.2250000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.5139999999999998,"15min":1.5129999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":8.5879999999999992,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.4969999999999999,"15min":1.712},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":3.5390000000000001,"5min":100.995,"15min":100.995},"last":0.57899999999999996}]}]},{"osd":1,"up_from":110,"seq":472446402711,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122260,"kb_used_data":649100,"kb_used_omap":4,"kb_used_meta":473147,"kb_avail":92622892,"statfs":{"total":95995035648,"available":94845841408,"internally_reserved":0,"allocated":664678400,"data_stored":663430071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":484503438},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5919999999999996,"5min":6.9119999999999999,"15min":4.032},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":102.471,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.223},{"interface":"front","average":{"1min":7.7779999999999996,"5min":7.9870000000000001,"15min":4.5599999999999996},"min":{"1min":0.20399999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":102.054,"5min":145.982,"15min":149.02699999999999},"last":6.7510000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":15.206,"5min":5.3970000000000002,"15min":3.048},"min":{"1min":0.21299999999999999,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":139.75399999999999,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":16.838000000000001,"5min":6.3470000000000004,"15min":3.8319999999999999},"min":{"1min":0.24199999999999999,"5min":0.219,"15min":0.219},"max":{"1min":140.29900000000001,"5min":140.29900000000001,"15min":140.29900000000001},"last":6.7149999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.9089999999999998,"15min":2.327},"min":{"1min":0.30599999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.585,"5min":77.331999999999994,"15min":153.18199999999999},"last":10.355},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.5379999999999998,"15min":1.8340000000000001},"min":{"1min":0.31,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.754,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":5.266,"15min":3.7469999999999999},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.1760000000000002,"5min":99.391000000000005,"15min":199.43799999999999},"last":3.2240000000000002},{"interface":"front","average":{"1min":1.6100000000000001,"5min":5.1180000000000003,"15min":3.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.3170000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":3.2930000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.5859999999999999,"15min":2.5499999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7169999999999996,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.441},{"interface":"front","average":{"1min":1.9890000000000001,"5min":3.5169999999999999,"15min":2.911},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":6.7789999999999999,"5min":77.388000000000005,"15min":77.388000000000005},"last":6.7789999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":4.1269999999999998,"15min":2.948},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":6.6799999999999997,"5min":77.292000000000002,"15min":77.292000000000002},"last":6.6799999999999997},{"interface":"front","average":{"1min":2.3559999999999999,"5min":3.8130000000000002,"15min":2.7429999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9290000000000003,"5min":77.378,"15min":77.378},"last":6.9290000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5289999999999999,"15min":2.706},"min":{"1min":0.36099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.2069999999999999,"5min":77.456999999999994,"15min":77.456999999999994},"last":3.2519999999999998},{"interface":"front","average":{"1min":1.105,"5min":2.9929999999999999,"15min":2.2040000000000002},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6900000000000004,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.41799999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024144,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2025664,"kb_used_data":1351544,"kb_used_omap":5,"kb_used_meta":674106,"kb_avail":91719488,"statfs":{"total":95995035648,"available":93920755712,"internally_reserved":0,"allocated":1383981056,"data_stored":1382519583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":690285178},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.3769999999999998,"15min":2.8170000000000002},"min":{"1min":0.193,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.5369999999999999,"5min":185.17400000000001,"15min":185.17400000000001},"last":0.315},{"interface":"front","average":{"1min":2.6280000000000001,"5min":4.5570000000000004,"15min":3.351},"min":{"1min":0.254,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":6.6319999999999997,"5min":186.691,"15min":186.691},"last":4.1299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9249999999999998,"5min":2.2349999999999999,"15min":2.2599999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":13.211,"5min":13.211,"15min":17.588999999999999},"last":3.919},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.472,"15min":2.391},"min":{"1min":0.247,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":11.956,"5min":13.081,"15min":21.198},"last":0.40100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":6.3200000000000003,"15min":2.726},"min":{"1min":0.28499999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.375,"5min":185.89400000000001,"15min":185.89400000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.2120000000000002,"5min":6.6360000000000001,"15min":2.8530000000000002},"min":{"1min":0.27000000000000002,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":13.138999999999999,"5min":186.72900000000001,"15min":186.72900000000001},"last":3.9929999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":1.599,"15min":1.627},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9850000000000003,"5min":7.9660000000000002,"15min":15.835000000000001},"last":3.024},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.7829999999999999,"15min":2.0590000000000002},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.25700000000000001},"max":{"1min":5.1310000000000002,"5min":8.4890000000000008,"15min":14.792999999999999},"last":4.9770000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.1699999999999999,"15min":2.036},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.0780000000000003,"5min":62.984999999999999,"15min":62.984999999999999},"last":3.8450000000000002},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.8479999999999999,"15min":2.5920000000000001},"min":{"1min":0.35199999999999998,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":3.8610000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":2.3420000000000001,"15min":2.3759999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.4710000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":4.0609999999999999},{"interface":"front","average":{"1min":1.669,"5min":1.7949999999999999,"15min":1.9430000000000001},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7439999999999998,"5min":10.606,"15min":15.893000000000001},"last":0.59299999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.516,"15min":2.3279999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28899999999999998},"max":{"1min":7.5010000000000003,"5min":62.262,"15min":62.262},"last":4.5019999999999998},{"interface":"front","average":{"1min":2.3340000000000001,"5min":2.9740000000000002,"15min":2.645},"min":{"1min":0.377,"5min":0.377,"15min":0.314},"max":{"1min":14.167,"5min":62.229999999999997,"15min":62.229999999999997},"last":1.0609999999999999}]}]},{"osd":2,"up_from":128,"seq":549755814013,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1305856,"kb_used_data":969592,"kb_used_omap":7,"kb_used_meta":336248,"kb_avail":92439296,"statfs":{"total":95995035648,"available":94657839104,"internally_reserved":0,"allocated":992862208,"data_stored":991630173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":344318033},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":3.802,"15min":3.4159999999999999},"min":{"1min":0.20799999999999999,"5min":0.157,"15min":0.157},"max":{"1min":10.356,"5min":110.611,"15min":163.40100000000001},"last":1.526},{"interface":"front","average":{"1min":2.5099999999999998,"5min":3.968,"15min":3.8540000000000001},"min":{"1min":0.24099999999999999,"5min":0.182,"15min":0.182},"max":{"1min":9.0679999999999996,"5min":110.66500000000001,"15min":163.363},"last":5.6079999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:24:27 2023","interfaces":[{"interface":"back","average":{"1min":13.189,"5min":4.4189999999999996,"15min":3.573},"min":{"1min":0.26100000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":205.148,"5min":205.148,"15min":222.50299999999999},"last":1.5669999999999999},{"interface":"front","average":{"1min":12.093999999999999,"5min":4.5579999999999998,"15min":4.0209999999999999},"min":{"1min":0.316,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":205.267,"5min":205.267,"15min":222.19399999999999},"last":11.811}]},{"osd":3,"last update":"Mon Oct 9 06:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.629,"15min":1.6759999999999999},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":16.331,"5min":69.936999999999998,"15min":81.183999999999997},"last":5.5659999999999998},{"interface":"front","average":{"1min":2.9950000000000001,"5min":2.8159999999999998,"15min":1.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":16.364000000000001,"5min":69.906999999999996,"15min":81.061999999999998},"last":5.5869999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":7.7530000000000001,"5min":2.7330000000000001,"15min":2.1339999999999999},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":90.938999999999993,"5min":90.938999999999993,"15min":90.938999999999993},"last":5.4630000000000001},{"interface":"front","average":{"1min":7.2480000000000002,"5min":3.1110000000000002,"15min":2.4169999999999998},"min":{"1min":0.313,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":90.950000000000003,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.26900000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":1.5409999999999999,"15min":1.579},"min":{"1min":0.41499999999999998,"5min":0.317,"15min":0.309},"max":{"1min":8.9589999999999996,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":5.508},{"interface":"front","average":{"1min":1.675,"5min":2.0259999999999998,"15min":2.1899999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.4320000000000004,"5min":10.548,"15min":10.548},"last":0.33600000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":9.2420000000000009,"5min":3.2189999999999999,"15min":2.4630000000000001},"min":{"1min":0.36199999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":138.291,"5min":138.291,"15min":138.291},"last":0.30199999999999999},{"interface":"front","average":{"1min":8.9420000000000002,"5min":2.9399999999999999,"15min":1.9990000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":138.01599999999999,"5min":138.01599999999999,"15min":138.01599999999999},"last":1.4910000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.7949999999999999,"15min":1.651},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":1.5469999999999999},{"interface":"front","average":{"1min":2.73,"5min":1.7809999999999999,"15min":2.3399999999999999},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.9960000000000004,"5min":8.9960000000000004,"15min":9.3249999999999993},"last":5.5359999999999996}]}]},{"osd":3,"up_from":149,"seq":639950127210,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089180,"kb_used_data":694548,"kb_used_omap":2,"kb_used_meta":394621,"kb_avail":92655972,"statfs":{"total":95995035648,"available":94879715328,"internally_reserved":0,"allocated":711217152,"data_stored":710077411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":404091957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":8.4339999999999993,"5min":8.0909999999999993,"15min":3.198},"min":{"1min":0.308,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":105.252,"5min":187.852,"15min":187.852},"last":0.97399999999999998},{"interface":"front","average":{"1min":8.1020000000000003,"5min":8.7400000000000002,"15min":3.7010000000000001},"min":{"1min":0.245,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":105.331,"5min":187.923,"15min":187.923},"last":0.95999999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3650000000000002,"5min":6.1289999999999996,"15min":2.8679999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.223},"max":{"1min":22.045999999999999,"5min":187.89099999999999,"15min":187.89099999999999},"last":1.704},{"interface":"front","average":{"1min":3.77,"5min":3.7010000000000001,"15min":1.847},"min":{"1min":0.26600000000000001,"5min":0.216,"15min":0.216},"max":{"1min":31.053000000000001,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.93300000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":5.5049999999999999,"15min":2.6110000000000002},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.891999999999999,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.8470000000000004,"15min":2.008},"min":{"1min":0.23400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.202,"5min":187.79499999999999,"15min":187.79499999999999},"last":1.633}]},{"osd":4,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.6639999999999999,"15min":1.05},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":4.2590000000000003,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.8779999999999999,"5min":3.5790000000000002,"15min":1.992},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1370000000000005,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.7749999999999999,"15min":2.1320000000000001},"min":{"1min":0.379,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":8.218,"5min":187.833,"15min":187.833},"last":1.266},{"interface":"front","average":{"1min":0.995,"5min":3.4350000000000001,"15min":2.0619999999999998},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.7249999999999996,"5min":187.952,"15min":187.952},"last":1.3009999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":3.762,"15min":2.1030000000000002},"min":{"1min":0.34899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1809999999999992,"5min":187.78200000000001,"15min":187.78200000000001},"last":1.6499999999999999},{"interface":"front","average":{"1min":1.2989999999999999,"5min":3.3460000000000001,"15min":1.998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7030000000000003,"5min":187.81800000000001,"15min":187.81800000000001},"last":3.0369999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.734,"15min":1.7889999999999999},"min":{"1min":0.37,"5min":0.35599999999999998,"15min":0.34100000000000003},"max":{"1min":4.2960000000000003,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.0169999999999999,"5min":3.754,"15min":2.0920000000000001},"min":{"1min":0.38800000000000001,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":8.2509999999999994,"5min":187.86500000000001,"15min":187.86500000000001},"last":2.9169999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019246,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2480580,"kb_used_data":1593788,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":91264572,"statfs":{"total":95995035648,"available":93454921728,"internally_reserved":0,"allocated":1632038912,"data_stored":1631268706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":3.8069999999999999,"15min":2.3140000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.18,"5min":76.944999999999993,"15min":76.944999999999993},"last":1.131},{"interface":"front","average":{"1min":5.444,"5min":4.2999999999999998,"15min":2.706},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":48.287999999999997,"5min":76.988,"15min":76.988},"last":1.1060000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.9890000000000001,"15min":2.0699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.8069999999999999,"5min":37.383000000000003,"15min":91.849999999999994},"last":2.8069999999999999},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.1869999999999998,"15min":2.177},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.5869999999999997,"5min":37.436999999999998,"15min":91.885000000000005},"last":1.401}]},{"osd":2,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.9240000000000004,"5min":2.8490000000000002,"15min":2.093},"min":{"1min":0.41599999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":44.807000000000002,"5min":69.640000000000001,"15min":69.640000000000001},"last":3.0019999999999998},{"interface":"front","average":{"1min":5.2370000000000001,"5min":3.0139999999999998,"15min":2.278},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":44.697000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":13.861000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":3.6480000000000001,"15min":2.1459999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.262,"5min":57.826999999999998,"15min":91.697000000000003},"last":1.464},{"interface":"front","average":{"1min":1.7669999999999999,"5min":3.6680000000000001,"15min":2.081},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":6.3250000000000002,"5min":57.853000000000002,"15min":94.356999999999999},"last":1.444}]},{"osd":5,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1619999999999999,"15min":1.107},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":4.2270000000000003,"5min":10.673,"15min":10.673},"last":1.341},{"interface":"front","average":{"1min":1.21,"5min":1.014,"15min":1.23},"min":{"1min":0.29599999999999999,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":4.6189999999999998,"5min":9.6379999999999999,"15min":12.552},"last":0.35699999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.77400000000000002,"15min":0.83499999999999996},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.8740000000000001,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":2.1179999999999999},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.212,"15min":1.244},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5419999999999998,"5min":15.321999999999999,"15min":15.321999999999999},"last":1.3740000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.956,"15min":1.49},"min":{"1min":0.34499999999999997,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":5.4480000000000004,"5min":62.658000000000001,"15min":62.658000000000001},"last":1.431},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.857,"15min":1.4770000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0640000000000001,"5min":62.573,"15min":62.573},"last":0.33600000000000002}]}]},{"osd":5,"up_from":0,"seq":163208757611,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1356075008,"data_stored":1356070936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":656080896,"data_stored":656080896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":989265920,"data_stored":989265920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":705822720,"data_stored":705822720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1608519680,"data_stored":1608515608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1009455104,"data_stored":1009451032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":677511168,"data_stored":677511168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:25:30.005 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:25:30.052 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:30.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:30.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:30.186 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:30.258 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:30.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:30.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:30.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:30.423 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[212750]: pgmap v877: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 93/87696 objects degraded (0.106%); 1335/87696 objects misplaced (1.522%); 1.3 MiB/s, 99 objects/s recovering 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[220921]: pgmap v877: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 93/87696 objects degraded (0.106%); 1335/87696 objects misplaced (1.522%); 1.3 MiB/s, 99 objects/s recovering 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:30.693 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:30 smithi150 ceph-mon[194178]: pgmap v877: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 890 op/s; 93/87696 objects degraded (0.106%); 1335/87696 objects misplaced (1.522%); 1.3 MiB/s, 99 objects/s recovering 2023-10-09T06:25:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:31.569 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:31.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:31.678 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:31.706 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:31.748 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:31.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:31.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:31.854 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:31.939 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:32.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:25:32.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:32.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:25:32.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:32.017 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:32] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-10-09T06:25:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:25:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:32.181 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:32.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:32.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:32.294 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:32.324 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:32.389 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:32.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:32.496 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:32.524 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:32.574 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:32.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:32.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:32.601 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:32.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:32.712 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: pgmap v878: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1335/85383 objects misplaced (1.564%); 1.0 MiB/s, 92 objects/s recovering 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T06:25:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:32 smithi150 ceph-mon[194178]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.937 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:32.937 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:25:32.941 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:32.941 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:32.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:32.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:32.961 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: pgmap v878: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1335/85383 objects misplaced (1.564%); 1.0 MiB/s, 92 objects/s recovering 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[220921]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: pgmap v878: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 1.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 1335/85383 objects misplaced (1.564%); 1.0 MiB/s, 92 objects/s recovering 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: osdmap e286: 8 total, 8 up, 7 in 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: osdmap e287: 8 total, 8 up, 7 in 2023-10-09T06:25:32.976 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:32 smithi107 ceph-mon[212750]: failed to encode map e286 with expected crc 2023-10-09T06:25:32.981 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:25:32.999 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:25:33.018 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:25:33.037 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:25:33.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:33.056 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:25:33.073 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:33.082 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.083 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:25:33.114 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:33.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:33.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:33.222 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:33.248 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:33.290 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:33.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:33.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:33.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:33.430 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:33.710 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: failed to encode map e287 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4260934431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: failed to encode map e287 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: failed to encode map e287 with expected crc 2023-10-09T06:25:33.711 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4260934431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:33.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: failed to encode map e288 with expected crc 2023-10-09T06:25:33.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T06:25:33.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: failed to encode map e288 with expected crc 2023-10-09T06:25:33.712 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:33 smithi107 ceph-mon[220921]: failed to encode map e288 with expected crc 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: failed to encode map e287 with expected crc 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: failed to encode map e287 with expected crc 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4260934431' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: failed to encode map e288 with expected crc 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: osdmap e288: 8 total, 8 up, 7 in 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: failed to encode map e288 with expected crc 2023-10-09T06:25:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:33 smithi150 ceph-mon[194178]: failed to encode map e288 with expected crc 2023-10-09T06:25:35.128 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:35.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:35.137 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: pgmap v881: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/75297 objects misplaced (0.409%); 3.6 MiB/s, 72 objects/s recovering 2023-10-09T06:25:35.137 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: failed to encode map e288 with expected crc 2023-10-09T06:25:35.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T06:25:35.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: pgmap v881: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/75297 objects misplaced (0.409%); 3.6 MiB/s, 72 objects/s recovering 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: failed to encode map e288 with expected crc 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.139 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: pgmap v881: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/75297 objects misplaced (0.409%); 3.6 MiB/s, 72 objects/s recovering 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: failed to encode map e288 with expected crc 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: osdmap e289: 8 total, 8 up, 7 in 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: failed to encode map e289 with expected crc 2023-10-09T06:25:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:25:35.207 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:25:35.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:35.273 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:35.355 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:35.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:35.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:35.469 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:35.547 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[212750]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[212750]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:36.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:25:36.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[220921]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T06:25:36.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:35 smithi107 ceph-mon[220921]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:35 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:25:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3848646655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:25:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:35 smithi150 ceph-mon[194178]: osdmap e290: 8 total, 8 up, 7 in 2023-10-09T06:25:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:35 smithi150 ceph-mon[194178]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:25:36.711 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:36.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:36.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[220921]: pgmap v884: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/74703 objects misplaced (0.412%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[220921]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[220921]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[212750]: pgmap v884: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/74703 objects misplaced (0.412%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[212750]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:36 smithi107 ceph-mon[212750]: failed to encode map e290 with expected crc 2023-10-09T06:25:36.941 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:36 smithi150 ceph-mon[194178]: pgmap v884: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 308/74703 objects misplaced (0.412%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:25:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:36 smithi150 ceph-mon[194178]: failed to encode map e290 with expected crc 2023-10-09T06:25:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:36 smithi150 ceph-mon[194178]: failed to encode map e290 with expected crc 2023-10-09T06:25:37.422 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:25:37.447 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:37.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:37.556 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:37.588 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:37.649 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:37.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:37.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:37.796 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:37.861 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:37.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:37.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:37.971 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:38.006 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:38.070 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:38.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:38.175 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:38.210 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:38.268 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:38.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:38.374 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:38.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:25:38.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[212750]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 42 MiB/s wr, 1.76k op/s; 16/73722 objects degraded (0.022%); 1758/73722 objects misplaced (2.385%); 856 KiB/s, 13 objects/s recovering 2023-10-09T06:25:38.406 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/73722 objects degraded (0.022%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:38.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:25:38.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[220921]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 42 MiB/s wr, 1.76k op/s; 16/73722 objects degraded (0.022%); 1758/73722 objects misplaced (2.385%); 856 KiB/s, 13 objects/s recovering 2023-10-09T06:25:38.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:38 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/73722 objects degraded (0.022%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:38.452 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:38.597 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:38.623 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:38.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:38 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:25:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:38 smithi150 ceph-mon[194178]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 42 MiB/s wr, 1.76k op/s; 16/73722 objects degraded (0.022%); 1758/73722 objects misplaced (2.385%); 856 KiB/s, 13 objects/s recovering 2023-10-09T06:25:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:38 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/73722 objects degraded (0.022%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:25:38.735 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:38.778 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:38.840 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:38.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:38.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:39.031 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:39.107 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:39.295 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:25:39.423 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:25:39.423 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16/67983 objects degraded (0.024%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1152,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":290,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":7,"osd_in_since":1696832504,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":22661,"data_bytes":1483937720,"bytes_used":10126340096,"bytes_avail":661838909440,"bytes_total":671965249536,"degraded_objects":16,"degraded_total":67983,"degraded_ratio":0.00023535295588603033,"misplaced_objects":1450,"misplaced_total":67983,"misplaced_ratio":0.0213288616271715,"recovering_objects_per_sec":28,"recovering_bytes_per_sec":1845451,"recovering_keys_per_sec":0,"num_objects_recovered":170,"num_bytes_recovered":11075584,"num_keys_recovered":0,"write_bytes_sec":36019054,"read_op_per_sec":0,"write_op_per_sec":1505},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T06:22:27.038508+0000","services":{}},"progress_events":{"be02b243-5ad1-4e62-ae2b-ff1425dec846":{"message":"Global Recovery Event (115s)\n [===========================.] (remaining: 2s)","progress":0.98165136575698853,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:25:40.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:40 smithi107 ceph-mon[220921]: pgmap v887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 34 MiB/s wr, 1.50k op/s; 16/67983 objects degraded (0.024%); 1450/67983 objects misplaced (2.133%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:25:40.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/362626127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:40.212 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:40 smithi107 ceph-mon[212750]: pgmap v887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 34 MiB/s wr, 1.50k op/s; 16/67983 objects degraded (0.024%); 1450/67983 objects misplaced (2.133%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:25:40.212 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/362626127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:40.320 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:40.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:40.434 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:40.474 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:40.584 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:40.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:40 smithi150 ceph-mon[194178]: pgmap v887: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.4 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 34 MiB/s wr, 1.50k op/s; 16/67983 objects degraded (0.024%); 1450/67983 objects misplaced (2.133%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:25:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/362626127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:25:40.707 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:40.743 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:40.807 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:40.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:40.919 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:40.963 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:41.026 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:41.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:41.138 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:41.202 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:41 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:25:41.630 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:41 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:25:41.632 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:41 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:25:41.993 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:41.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:42.098 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:42.139 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:42.231 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:42.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:42.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:42.394 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:42] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-10-09T06:25:42.394 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:42 smithi107 ceph-mon[212750]: pgmap v888: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 26 MiB/s wr, 1.23k op/s; 16/65268 objects degraded (0.025%); 1162/65268 objects misplaced (1.780%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:25:42.395 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:42 smithi107 ceph-mon[220921]: pgmap v888: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 26 MiB/s wr, 1.23k op/s; 16/65268 objects degraded (0.025%); 1162/65268 objects misplaced (1.780%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:25:42.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:42 smithi150 ceph-mon[194178]: pgmap v888: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.3 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 26 MiB/s wr, 1.23k op/s; 16/65268 objects degraded (0.025%); 1162/65268 objects misplaced (1.780%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:25:42.432 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:42.436 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:25:42.437 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:25:42.844 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:42.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:42.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:42.966 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:43.044 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:43.294 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:43 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/56706 objects degraded (0.028%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:43.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:43 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/56706 objects degraded (0.028%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:43.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:43 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/56706 objects degraded (0.028%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:44.125 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:44.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:44.232 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:44.291 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:44.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[220921]: pgmap v889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 23 MiB/s wr, 1.41k op/s; 16/56706 objects degraded (0.028%); 1162/56706 objects misplaced (2.049%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:25:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.411 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:44.418 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[212750]: pgmap v889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 23 MiB/s wr, 1.41k op/s; 16/56706 objects degraded (0.028%); 1162/56706 objects misplaced (2.049%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:25:44.418 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.418 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:44 smithi150 ceph-mon[194178]: pgmap v889: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 23 MiB/s wr, 1.41k op/s; 16/56706 objects degraded (0.028%); 1162/56706 objects misplaced (2.049%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:25:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:44.677 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:44.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:44.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:44.886 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:25:44.886 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:25:44.886 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:25:44.886 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:25:44.886 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:25:44.904 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:45.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:25:45.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:45.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:45.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:45.313 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/334620088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:45.590 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:25:45.590 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:45.590 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:45.590 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:45.590 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/334620088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:25:45.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.605 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:25:45.620 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:45 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:25:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:25:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:25:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:25:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/334620088' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:45.850 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:45.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:45.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:45.960 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:46.053 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[220921]: pgmap v890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 21 MiB/s wr, 1.28k op/s; 16/56040 objects degraded (0.029%); 1162/56040 objects misplaced (2.074%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:25:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:25:47.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[212750]: pgmap v890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 21 MiB/s wr, 1.28k op/s; 16/56040 objects degraded (0.029%); 1162/56040 objects misplaced (2.074%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:25:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:25:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:46 smithi150 ceph-mon[194178]: pgmap v890: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.1 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 21 MiB/s wr, 1.28k op/s; 16/56040 objects degraded (0.029%); 1162/56040 objects misplaced (2.074%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:25:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:46 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:25:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:25:47.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:25:48.047 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:48.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:48.155 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:48.193 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:48.247 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:48 smithi107 ceph-mon[212750]: pgmap v891: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 1.22k op/s; 16/50916 objects degraded (0.031%); 870/50916 objects misplaced (1.709%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:25:48.248 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:48 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:25:48.248 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:48 smithi107 ceph-mon[220921]: pgmap v891: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 1.22k op/s; 16/50916 objects degraded (0.031%); 870/50916 objects misplaced (1.709%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:25:48.248 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:48 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:25:48.251 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:48.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:48.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:48.358 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:48.386 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:48.450 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:48.455 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:48 smithi150 ceph-mon[194178]: pgmap v891: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 1.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 17 MiB/s wr, 1.22k op/s; 16/50916 objects degraded (0.031%); 870/50916 objects misplaced (1.709%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:25:48.455 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:48 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:25:48.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:48.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:48.557 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:48.603 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:48.676 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:48.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:48.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:48.786 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:48.856 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:49.271 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:49.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:49.381 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:49.446 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:49.652 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:49 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/44373 objects degraded (0.036%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:49.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:49 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/44373 objects degraded (0.036%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:49.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:49 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/44373 objects degraded (0.036%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:49.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:49.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:49.759 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:49.853 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:50.107 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:50.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:50.232 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:50.261 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:25:50.313 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:50.331 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:50.370 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:50 smithi107 ceph-mon[212750]: pgmap v892: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 923 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 16/44373 objects degraded (0.036%); 870/44373 objects misplaced (1.961%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:50.370 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:50 smithi107 ceph-mon[220921]: pgmap v892: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 923 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 16/44373 objects degraded (0.036%); 870/44373 objects misplaced (1.961%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:50.499 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:50.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:50.617 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:50.645 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:50 smithi150 ceph-mon[194178]: pgmap v892: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 923 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 16/44373 objects degraded (0.036%); 870/44373 objects misplaced (1.961%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:50.647 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:50.706 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:50.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:50.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:50.814 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:50.848 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:50.903 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:50.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:50.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:51.009 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:51.044 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:51.104 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:51.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:51.211 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:51.256 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:51.312 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2149427910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.312 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:51 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.330 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:51.339 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:25:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:51.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:51.346 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:25:51.377 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:25:51.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:51.478 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:51.550 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:51.557 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2149427910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.557 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:51 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:51.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2149427910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:51 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:25:51.660 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:51.721 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:52.306 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: pgmap v893: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 866 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 16/41625 objects degraded (0.038%); 559/41625 objects misplaced (1.343%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:25:52.306 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[220921]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: pgmap v893: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 866 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 16/41625 objects degraded (0.038%); 559/41625 objects misplaced (1.343%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:52 smithi107 ceph-mon[212750]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.308 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:25:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:25:52] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2023-10-09T06:25:52.616 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:52.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: pgmap v893: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 866 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 16/41625 objects degraded (0.038%); 559/41625 objects misplaced (1.343%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: osdmap e291: 8 total, 8 up, 7 in 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:52 smithi150 ceph-mon[194178]: failed to encode map e291 with expected crc 2023-10-09T06:25:52.726 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:52.788 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:53.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:53 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:25:53.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:53 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:25:53.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:53 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:25:53.887 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:53.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:53.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:54.058 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:54.099 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":894,"stamp":"2023-10-09T06:25:51.306549+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":908138424,"num_objects":13875,"num_object_clones":0,"num_object_copies":41625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":559,"num_objects_unfound":0,"num_objects_dirty":13875,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175756,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11664,"num_bytes_recovered":672409690,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203450,"ondisk_log_size":203450,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8184824,"kb_used_data":3558016,"kb_used_omap":20,"kb_used_meta":4626347,"kb_avail":648031240,"statfs":{"total":671965249536,"available":663583989760,"internally_reserved":0,"allocated":3643408384,"data_stored":3635930315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4737379982},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,3,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-516489216,"num_objects":-7881,"num_object_clones":0,"num_object_copies":-23643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-603,"num_objects_unfound":0,"num_objects_dirty":-7881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7881,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":301,"num_bytes_recovered":19726336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.188987"},"pg_stats":[{"pgid":"10.13","version":"290'8622","reported_seq":17135,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.864520+0000","last_change":"2023-10-09T06:24:52.774608+0000","last_active":"2023-10-09T06:25:46.864520+0000","last_peered":"2023-10-09T06:25:46.864520+0000","last_clean":"2023-10-09T06:25:46.864520+0000","last_became_active":"2023-10-09T06:24:52.762870+0000","last_became_peered":"2023-10-09T06:24:52.762870+0000","last_unstale":"2023-10-09T06:25:46.864520+0000","last_undegraded":"2023-10-09T06:25:46.864520+0000","last_fullsized":"2023-10-09T06:25:46.864520+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8622,"log_dups_size":0,"ondisk_log_size":8622,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6042,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2810,"num_bytes_recovered":184156160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":269,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.237890+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:25:34.237890+0000","last_peered":"2023-10-09T06:25:34.237890+0000","last_clean":"2023-10-09T06:25:34.237890+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:25:34.237890+0000","last_undegraded":"2023-10-09T06:25:34.237890+0000","last_fullsized":"2023-10-09T06:25:34.237890+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":227,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.242733+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:25:34.242733+0000","last_peered":"2023-10-09T06:25:34.242733+0000","last_clean":"2023-10-09T06:25:34.242733+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:25:34.242733+0000","last_undegraded":"2023-10-09T06:25:34.242733+0000","last_fullsized":"2023-10-09T06:25:34.242733+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"290'8566","reported_seq":14148,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:48.010917+0000","last_change":"2023-10-09T06:25:08.543126+0000","last_active":"2023-10-09T06:25:48.010917+0000","last_peered":"2023-10-09T06:25:48.010917+0000","last_clean":"2023-10-09T06:25:48.010917+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:25:48.010917+0000","last_undegraded":"2023-10-09T06:25:48.010917+0000","last_fullsized":"2023-10-09T06:25:48.010917+0000","mapping_epoch":269,"log_start":"243'1231","ondisk_log_start":"243'1231","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31588352,"num_objects":482,"num_object_clones":0,"num_object_copies":1446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6008,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196430+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:25:34.196430+0000","last_peered":"2023-10-09T06:25:34.196430+0000","last_clean":"2023-10-09T06:25:34.196430+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:25:34.196430+0000","last_undegraded":"2023-10-09T06:25:34.196430+0000","last_fullsized":"2023-10-09T06:25:34.196430+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":261,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195219+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:25:34.195219+0000","last_peered":"2023-10-09T06:25:34.195219+0000","last_clean":"2023-10-09T06:25:34.195219+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:25:34.195219+0000","last_undegraded":"2023-10-09T06:25:34.195219+0000","last_fullsized":"2023-10-09T06:25:34.195219+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"290'8775","reported_seq":14470,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.336850+0000","last_change":"2023-10-09T06:25:27.722935+0000","last_active":"2023-10-09T06:25:50.336850+0000","last_peered":"2023-10-09T06:25:50.336850+0000","last_clean":"2023-10-09T06:25:50.336850+0000","last_became_active":"2023-10-09T06:25:27.703733+0000","last_became_peered":"2023-10-09T06:25:27.703733+0000","last_unstale":"2023-10-09T06:25:50.336850+0000","last_undegraded":"2023-10-09T06:25:50.336850+0000","last_fullsized":"2023-10-09T06:25:50.336850+0000","mapping_epoch":284,"log_start":"242'278","ondisk_log_start":"242'278","created":254,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8497,"log_dups_size":0,"ondisk_log_size":8497,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6157,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2805,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":245,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233534+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:25:34.233534+0000","last_peered":"2023-10-09T06:25:34.233534+0000","last_clean":"2023-10-09T06:25:34.233534+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:25:34.233534+0000","last_undegraded":"2023-10-09T06:25:34.233534+0000","last_fullsized":"2023-10-09T06:25:34.233534+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198700+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:25:34.198700+0000","last_peered":"2023-10-09T06:25:34.198700+0000","last_clean":"2023-10-09T06:25:34.198700+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:25:34.198700+0000","last_undegraded":"2023-10-09T06:25:34.198700+0000","last_fullsized":"2023-10-09T06:25:34.198700+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195524+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:25:34.195524+0000","last_peered":"2023-10-09T06:25:34.195524+0000","last_clean":"2023-10-09T06:25:34.195524+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:25:34.195524+0000","last_undegraded":"2023-10-09T06:25:34.195524+0000","last_fullsized":"2023-10-09T06:25:34.195524+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233451+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:25:34.233451+0000","last_peered":"2023-10-09T06:25:34.233451+0000","last_clean":"2023-10-09T06:25:34.233451+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:25:34.233451+0000","last_undegraded":"2023-10-09T06:25:34.233451+0000","last_fullsized":"2023-10-09T06:25:34.233451+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200231+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:25:34.200231+0000","last_peered":"2023-10-09T06:25:34.200231+0000","last_clean":"2023-10-09T06:25:34.200231+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:25:34.200231+0000","last_undegraded":"2023-10-09T06:25:34.200231+0000","last_fullsized":"2023-10-09T06:25:34.200231+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":225,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233749+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:25:34.233749+0000","last_peered":"2023-10-09T06:25:34.233749+0000","last_clean":"2023-10-09T06:25:34.233749+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:25:34.233749+0000","last_undegraded":"2023-10-09T06:25:34.233749+0000","last_fullsized":"2023-10-09T06:25:34.233749+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":139,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200179+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:25:34.200179+0000","last_peered":"2023-10-09T06:25:34.200179+0000","last_clean":"2023-10-09T06:25:34.200179+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:25:34.200179+0000","last_undegraded":"2023-10-09T06:25:34.200179+0000","last_fullsized":"2023-10-09T06:25:34.200179+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":221,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240214+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:25:34.240214+0000","last_peered":"2023-10-09T06:25:34.240214+0000","last_clean":"2023-10-09T06:25:34.240214+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:25:34.240214+0000","last_undegraded":"2023-10-09T06:25:34.240214+0000","last_fullsized":"2023-10-09T06:25:34.240214+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":296,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241165+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:25:34.241165+0000","last_peered":"2023-10-09T06:25:34.241165+0000","last_clean":"2023-10-09T06:25:34.241165+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:25:34.241165+0000","last_undegraded":"2023-10-09T06:25:34.241165+0000","last_fullsized":"2023-10-09T06:25:34.241165+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":260,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238596+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:25:34.238596+0000","last_peered":"2023-10-09T06:25:34.238596+0000","last_clean":"2023-10-09T06:25:34.238596+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:25:34.238596+0000","last_undegraded":"2023-10-09T06:25:34.238596+0000","last_fullsized":"2023-10-09T06:25:34.238596+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":139,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199934+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:25:34.199934+0000","last_peered":"2023-10-09T06:25:34.199934+0000","last_clean":"2023-10-09T06:25:34.199934+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:25:34.199934+0000","last_undegraded":"2023-10-09T06:25:34.199934+0000","last_fullsized":"2023-10-09T06:25:34.199934+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.234533+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:25:34.234533+0000","last_peered":"2023-10-09T06:25:34.234533+0000","last_clean":"2023-10-09T06:25:34.234533+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:25:34.234533+0000","last_undegraded":"2023-10-09T06:25:34.234533+0000","last_fullsized":"2023-10-09T06:25:34.234533+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":260,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.194796+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:25:34.194796+0000","last_peered":"2023-10-09T06:25:34.194796+0000","last_clean":"2023-10-09T06:25:34.194796+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:25:34.194796+0000","last_undegraded":"2023-10-09T06:25:34.194796+0000","last_fullsized":"2023-10-09T06:25:34.194796+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":217,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200038+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:25:34.200038+0000","last_peered":"2023-10-09T06:25:34.200038+0000","last_clean":"2023-10-09T06:25:34.200038+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:25:34.200038+0000","last_undegraded":"2023-10-09T06:25:34.200038+0000","last_fullsized":"2023-10-09T06:25:34.200038+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"290'8622","reported_seq":8692,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.305480+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:25:50.305480+0000","last_peered":"2023-10-09T06:25:50.305480+0000","last_clean":"2023-10-09T06:25:50.305480+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:25:50.305480+0000","last_undegraded":"2023-10-09T06:25:50.305480+0000","last_fullsized":"2023-10-09T06:25:50.305480+0000","mapping_epoch":254,"log_start":"242'99","ondisk_log_start":"242'99","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8523,"log_dups_size":0,"ondisk_log_size":8523,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24608768,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6070,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":133,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199800+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:25:34.199800+0000","last_peered":"2023-10-09T06:25:34.199800+0000","last_clean":"2023-10-09T06:25:34.199800+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:25:34.199800+0000","last_undegraded":"2023-10-09T06:25:34.199800+0000","last_fullsized":"2023-10-09T06:25:34.199800+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195311+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:25:34.195311+0000","last_peered":"2023-10-09T06:25:34.195311+0000","last_clean":"2023-10-09T06:25:34.195311+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:25:34.195311+0000","last_undegraded":"2023-10-09T06:25:34.195311+0000","last_fullsized":"2023-10-09T06:25:34.195311+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"290'8570","reported_seq":8641,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.481513+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:25:46.481513+0000","last_peered":"2023-10-09T06:25:46.481513+0000","last_clean":"2023-10-09T06:25:46.481513+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:25:46.481513+0000","last_undegraded":"2023-10-09T06:25:46.481513+0000","last_fullsized":"2023-10-09T06:25:46.481513+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8570,"log_dups_size":0,"ondisk_log_size":8570,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32931840,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6005,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":278,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.197966+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:25:34.197966+0000","last_peered":"2023-10-09T06:25:34.197966+0000","last_clean":"2023-10-09T06:25:34.197966+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:25:34.197966+0000","last_undegraded":"2023-10-09T06:25:34.197966+0000","last_fullsized":"2023-10-09T06:25:34.197966+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":245,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.243337+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:25:34.243337+0000","last_peered":"2023-10-09T06:25:34.243337+0000","last_clean":"2023-10-09T06:25:34.243337+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:25:34.243337+0000","last_undegraded":"2023-10-09T06:25:34.243337+0000","last_fullsized":"2023-10-09T06:25:34.243337+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"290'8722","reported_seq":8862,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.268425+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:25:50.268425+0000","last_peered":"2023-10-09T06:25:50.268425+0000","last_clean":"2023-10-09T06:25:50.268425+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:25:50.268425+0000","last_undegraded":"2023-10-09T06:25:50.268425+0000","last_fullsized":"2023-10-09T06:25:50.268425+0000","mapping_epoch":238,"log_start":"242'422","ondisk_log_start":"242'422","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":8300,"log_dups_size":0,"ondisk_log_size":8300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25231360,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":151,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200233+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:25:34.200233+0000","last_peered":"2023-10-09T06:25:34.200233+0000","last_clean":"2023-10-09T06:25:34.200233+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:25:34.200233+0000","last_undegraded":"2023-10-09T06:25:34.200233+0000","last_fullsized":"2023-10-09T06:25:34.200233+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":200,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199810+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:25:34.199810+0000","last_peered":"2023-10-09T06:25:34.199810+0000","last_clean":"2023-10-09T06:25:34.199810+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:25:34.199810+0000","last_undegraded":"2023-10-09T06:25:34.199810+0000","last_fullsized":"2023-10-09T06:25:34.199810+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"290'10357","reported_seq":10544,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.898048+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:25:47.898048+0000","last_peered":"2023-10-09T06:25:47.898048+0000","last_clean":"2023-10-09T06:25:47.898048+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:25:47.898048+0000","last_undegraded":"2023-10-09T06:25:47.898048+0000","last_fullsized":"2023-10-09T06:25:47.898048+0000","mapping_epoch":238,"log_start":"248'3553","ondisk_log_start":"248'3553","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":6804,"log_dups_size":0,"ondisk_log_size":6804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28901376,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3255,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":137,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200137+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:25:34.200137+0000","last_peered":"2023-10-09T06:25:34.200137+0000","last_clean":"2023-10-09T06:25:34.200137+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:25:34.200137+0000","last_undegraded":"2023-10-09T06:25:34.200137+0000","last_fullsized":"2023-10-09T06:25:34.200137+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":267,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238705+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:25:34.238705+0000","last_peered":"2023-10-09T06:25:34.238705+0000","last_clean":"2023-10-09T06:25:34.238705+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:25:34.238705+0000","last_undegraded":"2023-10-09T06:25:34.238705+0000","last_fullsized":"2023-10-09T06:25:34.238705+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"290'8914","reported_seq":8986,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.359270+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:25:50.359270+0000","last_peered":"2023-10-09T06:25:50.359270+0000","last_clean":"2023-10-09T06:25:50.359270+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:25:50.359270+0000","last_undegraded":"2023-10-09T06:25:50.359270+0000","last_fullsized":"2023-10-09T06:25:50.359270+0000","mapping_epoch":238,"log_start":"242'791","ondisk_log_start":"242'791","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8123,"log_dups_size":0,"ondisk_log_size":8123,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15564800,"num_objects":237,"num_object_clones":0,"num_object_copies":711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6311,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":151,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200117+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:25:34.200117+0000","last_peered":"2023-10-09T06:25:34.200117+0000","last_clean":"2023-10-09T06:25:34.200117+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:25:34.200117+0000","last_undegraded":"2023-10-09T06:25:34.200117+0000","last_fullsized":"2023-10-09T06:25:34.200117+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":269,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198669+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:25:34.198669+0000","last_peered":"2023-10-09T06:25:34.198669+0000","last_clean":"2023-10-09T06:25:34.198669+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:25:34.198669+0000","last_undegraded":"2023-10-09T06:25:34.198669+0000","last_fullsized":"2023-10-09T06:25:34.198669+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"290'8561","reported_seq":8633,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.497426+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:25:46.497426+0000","last_peered":"2023-10-09T06:25:46.497426+0000","last_clean":"2023-10-09T06:25:46.497426+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:25:46.497426+0000","last_undegraded":"2023-10-09T06:25:46.497426+0000","last_fullsized":"2023-10-09T06:25:46.497426+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8561,"log_dups_size":0,"ondisk_log_size":8561,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6019,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.234426+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:25:34.234426+0000","last_peered":"2023-10-09T06:25:34.234426+0000","last_clean":"2023-10-09T06:25:34.234426+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:25:34.234426+0000","last_undegraded":"2023-10-09T06:25:34.234426+0000","last_fullsized":"2023-10-09T06:25:34.234426+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":242,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.194768+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:25:34.194768+0000","last_peered":"2023-10-09T06:25:34.194768+0000","last_clean":"2023-10-09T06:25:34.194768+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:25:34.194768+0000","last_undegraded":"2023-10-09T06:25:34.194768+0000","last_fullsized":"2023-10-09T06:25:34.194768+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"290'8612","reported_seq":8755,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.779824+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:25:46.779824+0000","last_peered":"2023-10-09T06:25:46.779824+0000","last_clean":"2023-10-09T06:25:46.779824+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:25:46.779824+0000","last_undegraded":"2023-10-09T06:25:46.779824+0000","last_fullsized":"2023-10-09T06:25:46.779824+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3318,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":582,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199714+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:25:34.199714+0000","last_peered":"2023-10-09T06:25:34.199714+0000","last_clean":"2023-10-09T06:25:34.199714+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:25:34.199714+0000","last_undegraded":"2023-10-09T06:25:34.199714+0000","last_fullsized":"2023-10-09T06:25:34.199714+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":254,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206805+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:25:35.206805+0000","last_peered":"2023-10-09T06:25:35.206805+0000","last_clean":"2023-10-09T06:25:35.206805+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:25:35.206805+0000","last_undegraded":"2023-10-09T06:25:35.206805+0000","last_fullsized":"2023-10-09T06:25:35.206805+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241252+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:25:34.241252+0000","last_peered":"2023-10-09T06:25:34.241252+0000","last_clean":"2023-10-09T06:25:34.241252+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:25:34.241252+0000","last_undegraded":"2023-10-09T06:25:34.241252+0000","last_fullsized":"2023-10-09T06:25:34.241252+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"290'8753","reported_seq":8825,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.511702+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:25:46.511702+0000","last_peered":"2023-10-09T06:25:46.511702+0000","last_clean":"2023-10-09T06:25:46.511702+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:25:46.511702+0000","last_undegraded":"2023-10-09T06:25:46.511702+0000","last_fullsized":"2023-10-09T06:25:46.511702+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8753,"log_dups_size":0,"ondisk_log_size":8753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32047104,"num_objects":489,"num_object_clones":0,"num_object_copies":1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6173,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":573,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199293+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:25:34.199293+0000","last_peered":"2023-10-09T06:25:34.199293+0000","last_clean":"2023-10-09T06:25:34.199293+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:25:34.199293+0000","last_undegraded":"2023-10-09T06:25:34.199293+0000","last_fullsized":"2023-10-09T06:25:34.199293+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198554+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:25:34.198554+0000","last_peered":"2023-10-09T06:25:34.198554+0000","last_clean":"2023-10-09T06:25:34.198554+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:25:34.198554+0000","last_undegraded":"2023-10-09T06:25:34.198554+0000","last_fullsized":"2023-10-09T06:25:34.198554+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":267,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241146+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:25:34.241146+0000","last_peered":"2023-10-09T06:25:34.241146+0000","last_clean":"2023-10-09T06:25:34.241146+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:25:34.241146+0000","last_undegraded":"2023-10-09T06:25:34.241146+0000","last_fullsized":"2023-10-09T06:25:34.241146+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"290'8633","reported_seq":8818,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.913894+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:25:47.913894+0000","last_peered":"2023-10-09T06:25:47.913894+0000","last_clean":"2023-10-09T06:25:47.913894+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:25:47.913894+0000","last_undegraded":"2023-10-09T06:25:47.913894+0000","last_fullsized":"2023-10-09T06:25:47.913894+0000","mapping_epoch":238,"log_start":"242'212","ondisk_log_start":"242'212","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":8421,"log_dups_size":0,"ondisk_log_size":8421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3498,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":271,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238564+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:25:34.238564+0000","last_peered":"2023-10-09T06:25:34.238564+0000","last_clean":"2023-10-09T06:25:34.238564+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:25:34.238564+0000","last_undegraded":"2023-10-09T06:25:34.238564+0000","last_fullsized":"2023-10-09T06:25:34.238564+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241904+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:25:34.241904+0000","last_peered":"2023-10-09T06:25:34.241904+0000","last_clean":"2023-10-09T06:25:34.241904+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:25:34.241904+0000","last_undegraded":"2023-10-09T06:25:34.241904+0000","last_fullsized":"2023-10-09T06:25:34.241904+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"290'8811","reported_seq":9003,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:48.013677+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:25:48.013677+0000","last_peered":"2023-10-09T06:25:48.013677+0000","last_clean":"2023-10-09T06:25:48.013677+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:25:48.013677+0000","last_undegraded":"2023-10-09T06:25:48.013677+0000","last_fullsized":"2023-10-09T06:25:48.013677+0000","mapping_epoch":238,"log_start":"244'1681","ondisk_log_start":"244'1681","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":7130,"log_dups_size":0,"ondisk_log_size":7130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3386,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":569,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199822+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:25:34.199822+0000","last_peered":"2023-10-09T06:25:34.199822+0000","last_clean":"2023-10-09T06:25:34.199822+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:25:34.199822+0000","last_undegraded":"2023-10-09T06:25:34.199822+0000","last_fullsized":"2023-10-09T06:25:34.199822+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233433+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:25:34.233433+0000","last_peered":"2023-10-09T06:25:34.233433+0000","last_clean":"2023-10-09T06:25:34.233433+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:25:34.233433+0000","last_undegraded":"2023-10-09T06:25:34.233433+0000","last_fullsized":"2023-10-09T06:25:34.233433+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":221,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196767+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:25:34.196767+0000","last_peered":"2023-10-09T06:25:34.196767+0000","last_clean":"2023-10-09T06:25:34.196767+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:25:34.196767+0000","last_undegraded":"2023-10-09T06:25:34.196767+0000","last_fullsized":"2023-10-09T06:25:34.196767+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"290'11829","reported_seq":11889,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:48.140536+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:25:48.140536+0000","last_peered":"2023-10-09T06:25:48.140536+0000","last_clean":"2023-10-09T06:25:48.140536+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:25:48.140536+0000","last_undegraded":"2023-10-09T06:25:48.140536+0000","last_fullsized":"2023-10-09T06:25:48.140536+0000","mapping_epoch":238,"log_start":"255'5100","ondisk_log_start":"255'5100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57081856,"num_objects":871,"num_object_clones":0,"num_object_copies":2613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11829,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":512,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198739+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:25:34.198739+0000","last_peered":"2023-10-09T06:25:34.198739+0000","last_clean":"2023-10-09T06:25:34.198739+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:25:34.198739+0000","last_undegraded":"2023-10-09T06:25:34.198739+0000","last_fullsized":"2023-10-09T06:25:34.198739+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":273,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240817+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:25:34.240817+0000","last_peered":"2023-10-09T06:25:34.240817+0000","last_clean":"2023-10-09T06:25:34.240817+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:25:34.240817+0000","last_undegraded":"2023-10-09T06:25:34.240817+0000","last_fullsized":"2023-10-09T06:25:34.240817+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":198,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.235808+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:25:34.235808+0000","last_peered":"2023-10-09T06:25:34.235808+0000","last_clean":"2023-10-09T06:25:34.235808+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:25:34.235808+0000","last_undegraded":"2023-10-09T06:25:34.235808+0000","last_fullsized":"2023-10-09T06:25:34.235808+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":216,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196614+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:25:34.196614+0000","last_peered":"2023-10-09T06:25:34.196614+0000","last_clean":"2023-10-09T06:25:34.196614+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:25:34.196614+0000","last_undegraded":"2023-10-09T06:25:34.196614+0000","last_fullsized":"2023-10-09T06:25:34.196614+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"290'12279","reported_seq":12337,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.918726+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:25:47.918726+0000","last_peered":"2023-10-09T06:25:47.918726+0000","last_clean":"2023-10-09T06:25:47.918726+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:25:47.918726+0000","last_undegraded":"2023-10-09T06:25:47.918726+0000","last_fullsized":"2023-10-09T06:25:47.918726+0000","mapping_epoch":238,"log_start":"267'6200","ondisk_log_start":"267'6200","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12279,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":575,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195695+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:25:34.195695+0000","last_peered":"2023-10-09T06:25:34.195695+0000","last_clean":"2023-10-09T06:25:34.195695+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:25:34.195695+0000","last_undegraded":"2023-10-09T06:25:34.195695+0000","last_fullsized":"2023-10-09T06:25:34.195695+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":146,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196734+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:25:34.196734+0000","last_peered":"2023-10-09T06:25:34.196734+0000","last_clean":"2023-10-09T06:25:34.196734+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:25:34.196734+0000","last_undegraded":"2023-10-09T06:25:34.196734+0000","last_fullsized":"2023-10-09T06:25:34.196734+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238394+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:25:34.238394+0000","last_peered":"2023-10-09T06:25:34.238394+0000","last_clean":"2023-10-09T06:25:34.238394+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:25:34.238394+0000","last_undegraded":"2023-10-09T06:25:34.238394+0000","last_fullsized":"2023-10-09T06:25:34.238394+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":237,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240736+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:25:34.240736+0000","last_peered":"2023-10-09T06:25:34.240736+0000","last_clean":"2023-10-09T06:25:34.240736+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:25:34.240736+0000","last_undegraded":"2023-10-09T06:25:34.240736+0000","last_fullsized":"2023-10-09T06:25:34.240736+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"290'8701","reported_seq":8837,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.502974+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:25:46.502974+0000","last_peered":"2023-10-09T06:25:46.502974+0000","last_clean":"2023-10-09T06:25:46.502974+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:25:46.502974+0000","last_undegraded":"2023-10-09T06:25:46.502974+0000","last_fullsized":"2023-10-09T06:25:46.502974+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":8701,"log_dups_size":0,"ondisk_log_size":8701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33030144,"num_objects":504,"num_object_clones":0,"num_object_copies":1512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3529,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":582,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199697+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:25:34.199697+0000","last_peered":"2023-10-09T06:25:34.199697+0000","last_clean":"2023-10-09T06:25:34.199697+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:25:34.199697+0000","last_undegraded":"2023-10-09T06:25:34.199697+0000","last_fullsized":"2023-10-09T06:25:34.199697+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":256,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199357+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:25:34.199357+0000","last_peered":"2023-10-09T06:25:34.199357+0000","last_clean":"2023-10-09T06:25:34.199357+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:25:34.199357+0000","last_undegraded":"2023-10-09T06:25:34.199357+0000","last_fullsized":"2023-10-09T06:25:34.199357+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":263,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196615+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:25:34.196615+0000","last_peered":"2023-10-09T06:25:34.196615+0000","last_clean":"2023-10-09T06:25:34.196615+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:25:34.196615+0000","last_undegraded":"2023-10-09T06:25:34.196615+0000","last_fullsized":"2023-10-09T06:25:34.196615+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":258,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.196826+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:25:34.196826+0000","last_peered":"2023-10-09T06:25:34.196826+0000","last_clean":"2023-10-09T06:25:34.196826+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:25:34.196826+0000","last_undegraded":"2023-10-09T06:25:34.196826+0000","last_fullsized":"2023-10-09T06:25:34.196826+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"290'12171","reported_seq":12231,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.894941+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:25:47.894941+0000","last_peered":"2023-10-09T06:25:47.894941+0000","last_clean":"2023-10-09T06:25:47.894941+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:25:47.894941+0000","last_undegraded":"2023-10-09T06:25:47.894941+0000","last_fullsized":"2023-10-09T06:25:47.894941+0000","mapping_epoch":238,"log_start":"267'6100","ondisk_log_start":"267'6100","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12171,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":222,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199420+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:25:34.199420+0000","last_peered":"2023-10-09T06:25:34.199420+0000","last_clean":"2023-10-09T06:25:34.199420+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:25:34.199420+0000","last_undegraded":"2023-10-09T06:25:34.199420+0000","last_fullsized":"2023-10-09T06:25:34.199420+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":563,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199708+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:25:34.199708+0000","last_peered":"2023-10-09T06:25:34.199708+0000","last_clean":"2023-10-09T06:25:34.199708+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:25:34.199708+0000","last_undegraded":"2023-10-09T06:25:34.199708+0000","last_fullsized":"2023-10-09T06:25:34.199708+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":217,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.234124+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:25:34.234124+0000","last_peered":"2023-10-09T06:25:34.234124+0000","last_clean":"2023-10-09T06:25:34.234124+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:25:34.234124+0000","last_undegraded":"2023-10-09T06:25:34.234124+0000","last_fullsized":"2023-10-09T06:25:34.234124+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198412+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:25:34.198412+0000","last_peered":"2023-10-09T06:25:34.198412+0000","last_clean":"2023-10-09T06:25:34.198412+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:25:34.198412+0000","last_undegraded":"2023-10-09T06:25:34.198412+0000","last_fullsized":"2023-10-09T06:25:34.198412+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"290'8620","reported_seq":8755,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.265807+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:25:50.265807+0000","last_peered":"2023-10-09T06:25:50.265807+0000","last_clean":"2023-10-09T06:25:50.265807+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:25:50.265807+0000","last_undegraded":"2023-10-09T06:25:50.265807+0000","last_fullsized":"2023-10-09T06:25:50.265807+0000","mapping_epoch":238,"log_start":"242'172","ondisk_log_start":"242'172","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":8448,"log_dups_size":0,"ondisk_log_size":8448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22806528,"num_objects":348,"num_object_clones":0,"num_object_copies":1044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3442,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":573,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240484+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:25:34.240484+0000","last_peered":"2023-10-09T06:25:34.240484+0000","last_clean":"2023-10-09T06:25:34.240484+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:25:34.240484+0000","last_undegraded":"2023-10-09T06:25:34.240484+0000","last_fullsized":"2023-10-09T06:25:34.240484+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":133,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199551+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:25:34.199551+0000","last_peered":"2023-10-09T06:25:34.199551+0000","last_clean":"2023-10-09T06:25:34.199551+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:25:34.199551+0000","last_undegraded":"2023-10-09T06:25:34.199551+0000","last_fullsized":"2023-10-09T06:25:34.199551+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":243,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195627+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:25:34.195627+0000","last_peered":"2023-10-09T06:25:34.195627+0000","last_clean":"2023-10-09T06:25:34.195627+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:25:34.195627+0000","last_undegraded":"2023-10-09T06:25:34.195627+0000","last_fullsized":"2023-10-09T06:25:34.195627+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":266,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.210997+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:25:35.210997+0000","last_peered":"2023-10-09T06:25:35.210997+0000","last_clean":"2023-10-09T06:25:35.210997+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:25:35.210997+0000","last_undegraded":"2023-10-09T06:25:35.210997+0000","last_fullsized":"2023-10-09T06:25:35.210997+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"290'12317","reported_seq":12378,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.902936+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:25:47.902936+0000","last_peered":"2023-10-09T06:25:47.902936+0000","last_clean":"2023-10-09T06:25:47.902936+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:25:47.902936+0000","last_undegraded":"2023-10-09T06:25:47.902936+0000","last_fullsized":"2023-10-09T06:25:47.902936+0000","mapping_epoch":238,"log_start":"267'6300","ondisk_log_start":"267'6300","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6017,"log_dups_size":0,"ondisk_log_size":6017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720280,"num_objects":897,"num_object_clones":0,"num_object_copies":2691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":897,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12317,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":512,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198431+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:25:34.198431+0000","last_peered":"2023-10-09T06:25:34.198431+0000","last_clean":"2023-10-09T06:25:34.198431+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:25:34.198431+0000","last_undegraded":"2023-10-09T06:25:34.198431+0000","last_fullsized":"2023-10-09T06:25:34.198431+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":133,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.200132+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:25:34.200132+0000","last_peered":"2023-10-09T06:25:34.200132+0000","last_clean":"2023-10-09T06:25:34.200132+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:25:34.200132+0000","last_undegraded":"2023-10-09T06:25:34.200132+0000","last_fullsized":"2023-10-09T06:25:34.200132+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":272,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240730+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:25:34.240730+0000","last_peered":"2023-10-09T06:25:34.240730+0000","last_clean":"2023-10-09T06:25:34.240730+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:25:34.240730+0000","last_undegraded":"2023-10-09T06:25:34.240730+0000","last_fullsized":"2023-10-09T06:25:34.240730+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.194958+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:25:34.194958+0000","last_peered":"2023-10-09T06:25:34.194958+0000","last_clean":"2023-10-09T06:25:34.194958+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:25:34.194958+0000","last_undegraded":"2023-10-09T06:25:34.194958+0000","last_fullsized":"2023-10-09T06:25:34.194958+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"290'11984","reported_seq":12044,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.946391+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:25:47.946391+0000","last_peered":"2023-10-09T06:25:47.946391+0000","last_clean":"2023-10-09T06:25:47.946391+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:25:47.946391+0000","last_undegraded":"2023-10-09T06:25:47.946391+0000","last_fullsized":"2023-10-09T06:25:47.946391+0000","mapping_epoch":238,"log_start":"270'6600","ondisk_log_start":"270'6600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11984,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":573,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241064+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:25:34.241064+0000","last_peered":"2023-10-09T06:25:34.241064+0000","last_clean":"2023-10-09T06:25:34.241064+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:25:34.241064+0000","last_undegraded":"2023-10-09T06:25:34.241064+0000","last_fullsized":"2023-10-09T06:25:34.241064+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":280,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.243415+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:25:34.243415+0000","last_peered":"2023-10-09T06:25:34.243415+0000","last_clean":"2023-10-09T06:25:34.243415+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:25:34.243415+0000","last_undegraded":"2023-10-09T06:25:34.243415+0000","last_fullsized":"2023-10-09T06:25:34.243415+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":248,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240442+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:25:34.240442+0000","last_peered":"2023-10-09T06:25:34.240442+0000","last_clean":"2023-10-09T06:25:34.240442+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:25:34.240442+0000","last_undegraded":"2023-10-09T06:25:34.240442+0000","last_fullsized":"2023-10-09T06:25:34.240442+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":269,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233373+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:25:34.233373+0000","last_peered":"2023-10-09T06:25:34.233373+0000","last_clean":"2023-10-09T06:25:34.233373+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:25:34.233373+0000","last_undegraded":"2023-10-09T06:25:34.233373+0000","last_fullsized":"2023-10-09T06:25:34.233373+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"290'12259","reported_seq":12319,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.898335+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:25:47.898335+0000","last_peered":"2023-10-09T06:25:47.898335+0000","last_clean":"2023-10-09T06:25:47.898335+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:25:47.898335+0000","last_undegraded":"2023-10-09T06:25:47.898335+0000","last_fullsized":"2023-10-09T06:25:47.898335+0000","mapping_epoch":238,"log_start":"267'6200","ondisk_log_start":"267'6200","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6059,"log_dups_size":0,"ondisk_log_size":6059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12259,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":594,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.234646+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:25:34.234646+0000","last_peered":"2023-10-09T06:25:34.234646+0000","last_clean":"2023-10-09T06:25:34.234646+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:25:34.234646+0000","last_undegraded":"2023-10-09T06:25:34.234646+0000","last_fullsized":"2023-10-09T06:25:34.234646+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":274,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.198851+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:25:34.198851+0000","last_peered":"2023-10-09T06:25:34.198851+0000","last_clean":"2023-10-09T06:25:34.198851+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:25:34.198851+0000","last_undegraded":"2023-10-09T06:25:34.198851+0000","last_fullsized":"2023-10-09T06:25:34.198851+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":247,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207567+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:25:35.207567+0000","last_peered":"2023-10-09T06:25:35.207567+0000","last_clean":"2023-10-09T06:25:35.207567+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:25:35.207567+0000","last_undegraded":"2023-10-09T06:25:35.207567+0000","last_fullsized":"2023-10-09T06:25:35.207567+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":243,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238707+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:25:34.238707+0000","last_peered":"2023-10-09T06:25:34.238707+0000","last_clean":"2023-10-09T06:25:34.238707+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:25:34.238707+0000","last_undegraded":"2023-10-09T06:25:34.238707+0000","last_fullsized":"2023-10-09T06:25:34.238707+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"290'9529","reported_seq":20639,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.885031+0000","last_change":"2023-10-09T06:25:46.179505+0000","last_active":"2023-10-09T06:25:47.885031+0000","last_peered":"2023-10-09T06:25:47.885031+0000","last_clean":"2023-10-09T06:25:47.885031+0000","last_became_active":"2023-10-09T06:25:34.201087+0000","last_became_peered":"2023-10-09T06:25:34.201087+0000","last_unstale":"2023-10-09T06:25:47.885031+0000","last_undegraded":"2023-10-09T06:25:47.885031+0000","last_fullsized":"2023-10-09T06:25:47.885031+0000","mapping_epoch":288,"log_start":"244'1855","ondisk_log_start":"244'1855","created":254,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":7674,"log_dups_size":0,"ondisk_log_size":7674,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28213248,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6025,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2786,"num_bytes_recovered":182583296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":249,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203693+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:25:35.203693+0000","last_peered":"2023-10-09T06:25:35.203693+0000","last_clean":"2023-10-09T06:25:35.203693+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:25:35.203693+0000","last_undegraded":"2023-10-09T06:25:35.203693+0000","last_fullsized":"2023-10-09T06:25:35.203693+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":272,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.199666+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:25:34.199666+0000","last_peered":"2023-10-09T06:25:34.199666+0000","last_clean":"2023-10-09T06:25:34.199666+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:25:34.199666+0000","last_undegraded":"2023-10-09T06:25:34.199666+0000","last_fullsized":"2023-10-09T06:25:34.199666+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"290'8500","reported_seq":8571,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:46.520750+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:25:46.520750+0000","last_peered":"2023-10-09T06:25:46.520750+0000","last_clean":"2023-10-09T06:25:46.520750+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:25:46.520750+0000","last_undegraded":"2023-10-09T06:25:46.520750+0000","last_fullsized":"2023-10-09T06:25:46.520750+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8500,"log_dups_size":0,"ondisk_log_size":8500,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5958,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":95,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.240998+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:25:34.240998+0000","last_peered":"2023-10-09T06:25:34.240998+0000","last_clean":"2023-10-09T06:25:34.240998+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:25:34.240998+0000","last_undegraded":"2023-10-09T06:25:34.240998+0000","last_fullsized":"2023-10-09T06:25:34.240998+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":277,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.238322+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:25:34.238322+0000","last_peered":"2023-10-09T06:25:34.238322+0000","last_clean":"2023-10-09T06:25:34.238322+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:25:34.238322+0000","last_undegraded":"2023-10-09T06:25:34.238322+0000","last_fullsized":"2023-10-09T06:25:34.238322+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"290'8736","reported_seq":10353,"reported_epoch":290,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:25:50.361313+0000","last_change":"2023-10-09T06:25:33.463651+0000","last_active":"2023-10-09T06:25:50.361313+0000","last_peered":"2023-10-09T06:25:50.361313+0000","last_clean":"2023-10-09T06:25:31.596130+0000","last_became_active":"2023-10-09T06:25:33.383675+0000","last_became_peered":"2023-10-09T06:25:33.383675+0000","last_unstale":"2023-10-09T06:25:50.361313+0000","last_undegraded":"2023-10-09T06:25:33.331302+0000","last_fullsized":"2023-10-09T06:25:33.192605+0000","mapping_epoch":287,"log_start":"242'79","ondisk_log_start":"242'79","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8657,"log_dups_size":0,"ondisk_log_size":8657,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30375936,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":559,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6134,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":512,"num_bytes_recovered":33488896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,1,3,4","objects":1},{"shards":"0,3,4","objects":271},{"shards":"0,4","objects":176}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":95,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.241190+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:25:34.241190+0000","last_peered":"2023-10-09T06:25:34.241190+0000","last_clean":"2023-10-09T06:25:34.241190+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:25:34.241190+0000","last_undegraded":"2023-10-09T06:25:34.241190+0000","last_fullsized":"2023-10-09T06:25:34.241190+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":267,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.237971+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:25:34.237971+0000","last_peered":"2023-10-09T06:25:34.237971+0000","last_clean":"2023-10-09T06:25:34.237971+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:25:34.237971+0000","last_undegraded":"2023-10-09T06:25:34.237971+0000","last_fullsized":"2023-10-09T06:25:34.237971+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"290'8531","reported_seq":8610,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:47.912446+0000","last_change":"2023-10-09T06:25:47.212370+0000","last_active":"2023-10-09T06:25:47.912446+0000","last_peered":"2023-10-09T06:25:47.912446+0000","last_clean":"2023-10-09T06:25:47.912446+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:25:47.912446+0000","last_undegraded":"2023-10-09T06:25:47.912446+0000","last_fullsized":"2023-10-09T06:25:47.912446+0000","mapping_epoch":254,"log_start":"242'87","ondisk_log_start":"242'87","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8444,"log_dups_size":0,"ondisk_log_size":8444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6002,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":248,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.242439+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:25:34.242439+0000","last_peered":"2023-10-09T06:25:34.242439+0000","last_clean":"2023-10-09T06:25:34.242439+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:25:34.242439+0000","last_undegraded":"2023-10-09T06:25:34.242439+0000","last_fullsized":"2023-10-09T06:25:34.242439+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.233950+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:25:34.233950+0000","last_peered":"2023-10-09T06:25:34.233950+0000","last_clean":"2023-10-09T06:25:34.233950+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:25:34.233950+0000","last_undegraded":"2023-10-09T06:25:34.233950+0000","last_fullsized":"2023-10-09T06:25:34.233950+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"290'8622","reported_seq":8693,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:50.294073+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:25:50.294073+0000","last_peered":"2023-10-09T06:25:50.294073+0000","last_clean":"2023-10-09T06:25:50.294073+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:25:50.294073+0000","last_undegraded":"2023-10-09T06:25:50.294073+0000","last_fullsized":"2023-10-09T06:25:50.294073+0000","mapping_epoch":254,"log_start":"242'12","ondisk_log_start":"242'12","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8610,"log_dups_size":0,"ondisk_log_size":8610,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27230208,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6064,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.195455+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:25:34.195455+0000","last_peered":"2023-10-09T06:25:34.195455+0000","last_clean":"2023-10-09T06:25:34.195455+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:25:34.195455+0000","last_undegraded":"2023-10-09T06:25:34.195455+0000","last_fullsized":"2023-10-09T06:25:34.195455+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T06:25:34.237746+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:25:34.237746+0000","last_peered":"2023-10-09T06:25:34.237746+0000","last_clean":"2023-10-09T06:25:34.237746+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:25:34.237746+0000","last_undegraded":"2023-10-09T06:25:34.237746+0000","last_fullsized":"2023-10-09T06:25:34.237746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":907542552,"num_objects":13851,"num_object_clones":0,"num_object_copies":41553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":559,"num_objects_unfound":0,"num_objects_dirty":13851,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175695,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11636,"num_bytes_recovered":672071680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3531747328,"data_stored":3531735112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201624,"ondisk_log_size":201624,"up":78,"acting":77,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234342,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1216652,"kb_used_data":316420,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92528500,"statfs":{"total":95995035648,"available":94749184000,"internally_reserved":0,"allocated":324014080,"data_stored":323235817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3399999999999999,"5min":7.7510000000000003,"15min":3.4870000000000001},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":60.524999999999999,"5min":219.41900000000001,"15min":219.41900000000001},"last":8.4909999999999997},{"interface":"front","average":{"1min":4.4130000000000003,"5min":8.1159999999999997,"15min":3.5529999999999999},"min":{"1min":0.27900000000000003,"5min":0.218,"15min":0.218},"max":{"1min":60.539000000000001,"5min":219.399,"15min":219.399},"last":8.6649999999999991}]},{"osd":1,"last update":"Mon Oct 9 06:25:20 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":3.1869999999999998,"15min":2.6659999999999999},"min":{"1min":0.253,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.242,"5min":141.44300000000001,"15min":141.44300000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.4289999999999998,"15min":2.4630000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.7629999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.48399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:25:32 2023","interfaces":[{"interface":"back","average":{"1min":12.579000000000001,"5min":12.257999999999999,"15min":5.2720000000000002},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":92.293000000000006,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":12.222,"5min":12.436,"15min":5.4109999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.727999999999994,"5min":215.869,"15min":215.869},"last":1.004}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.9889999999999999,"15min":1.542},"min":{"1min":0.40999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.2149999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":1.2849999999999999},{"interface":"front","average":{"1min":2.012,"5min":3.1389999999999998,"15min":1.599},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.4550000000000001,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.161,"15min":1.673},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":94.105999999999995,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.40400000000000003},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.218,"15min":1.4379999999999999},"min":{"1min":0.31900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":93.713999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.39000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.429,"15min":1.714},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9220000000000002,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.341,"5min":1.3140000000000001,"15min":1.429},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":5.7130000000000001,"5min":7.452,"15min":48.774999999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.494,"15min":1.359},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7889999999999997,"5min":8.4890000000000008,"15min":11.878},"last":0.443},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.464,"15min":1.256},"min":{"1min":0.38600000000000001,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":5.2690000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.45200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495979,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1374852,"kb_used_data":467644,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92370300,"statfs":{"total":95995035648,"available":94587187200,"internally_reserved":0,"allocated":478867456,"data_stored":478097773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0449999999999999,"5min":3.9129999999999998,"15min":3.1339999999999999},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":28.035,"5min":110.926,"15min":110.926},"last":1.556},{"interface":"front","average":{"1min":1.635,"5min":4.0149999999999997,"15min":3.3319999999999999},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.24399999999999999},"max":{"1min":7.0499999999999998,"5min":110.911,"15min":110.911},"last":1.6739999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:19 2023","interfaces":[{"interface":"back","average":{"1min":6.5010000000000003,"5min":2.8690000000000002,"15min":3.5310000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":66.796999999999997,"5min":66.796999999999997,"15min":114.202},"last":1.0189999999999999},{"interface":"front","average":{"1min":6.5460000000000003,"5min":2.5630000000000002,"15min":3.3530000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":66.704999999999998,"5min":66.704999999999998,"15min":114.179},"last":1.0309999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":4.0890000000000004,"15min":2.774},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":52.609999999999999,"5min":75.191999999999993,"15min":75.191999999999993},"last":3.3940000000000001},{"interface":"front","average":{"1min":5.1059999999999999,"5min":3.9119999999999999,"15min":2.6669999999999998},"min":{"1min":0.29899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.575000000000003,"5min":75.120000000000005,"15min":75.120000000000005},"last":3.4060000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":3.2149999999999999,"15min":1.8939999999999999},"min":{"1min":0.31900000000000001,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":11.148,"5min":110.358,"15min":110.358},"last":0.747},{"interface":"front","average":{"1min":3.1680000000000001,"5min":3.5870000000000002,"15min":1.986},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":11.180999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.82299999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.1299999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.1890000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.03,"5min":1.474,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":4.2119999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.456,"15min":1.3640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.2460000000000004,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.488,"15min":1.407},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.8920000000000003,"5min":24.401,"15min":24.401},"last":0.40699999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.5139999999999998,"15min":1.5129999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":8.5879999999999992,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.4969999999999999,"15min":1.712},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":3.5390000000000001,"5min":100.995,"15min":100.995},"last":0.47199999999999998}]}]},{"osd":1,"up_from":110,"seq":472446402715,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865568,"kb_used_data":392408,"kb_used_omap":4,"kb_used_meta":473147,"kb_avail":92879584,"statfs":{"total":95995035648,"available":95108694016,"internally_reserved":0,"allocated":401825792,"data_stored":400558699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":484503438},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5919999999999996,"5min":6.9119999999999999,"15min":4.032},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":102.471,"5min":145.96600000000001,"15min":148.98400000000001},"last":5.173},{"interface":"front","average":{"1min":7.7779999999999996,"5min":7.9870000000000001,"15min":4.5599999999999996},"min":{"1min":0.20399999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":102.054,"5min":145.982,"15min":149.02699999999999},"last":3.0310000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":5.6630000000000003,"15min":3.1059999999999999},"min":{"1min":0.26000000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":8.4030000000000005,"5min":139.75399999999999,"15min":139.75399999999999},"last":2.3809999999999998},{"interface":"front","average":{"1min":3.3530000000000002,"5min":6.7800000000000002,"15min":3.863},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.219},"max":{"1min":9.8859999999999992,"5min":140.29900000000001,"15min":140.29900000000001},"last":11.522}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.9089999999999998,"15min":2.327},"min":{"1min":0.30599999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.585,"5min":77.331999999999994,"15min":153.18199999999999},"last":2.4220000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.5379999999999998,"15min":1.8340000000000001},"min":{"1min":0.31,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.754,"5min":77.197000000000003,"15min":152.52000000000001},"last":2.1539999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":5.266,"15min":3.7469999999999999},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.1760000000000002,"5min":99.391000000000005,"15min":199.43799999999999},"last":2.508},{"interface":"front","average":{"1min":1.6100000000000001,"5min":5.1180000000000003,"15min":3.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.3170000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":2.04}]},{"osd":5,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.5859999999999999,"15min":2.5499999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7169999999999996,"5min":77.436999999999998,"15min":77.436999999999998},"last":2.202},{"interface":"front","average":{"1min":1.9890000000000001,"5min":3.5169999999999999,"15min":2.911},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":6.7789999999999999,"5min":77.388000000000005,"15min":77.388000000000005},"last":2.0830000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":4.1269999999999998,"15min":2.948},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":6.6799999999999997,"5min":77.292000000000002,"15min":77.292000000000002},"last":2.105},{"interface":"front","average":{"1min":2.3559999999999999,"5min":3.8130000000000002,"15min":2.7429999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9290000000000003,"5min":77.378,"15min":77.378},"last":2.4820000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5289999999999999,"15min":2.706},"min":{"1min":0.36099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.2069999999999999,"5min":77.456999999999994,"15min":77.456999999999994},"last":2.4540000000000002},{"interface":"front","average":{"1min":1.105,"5min":2.9929999999999999,"15min":2.2040000000000002},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6900000000000004,"5min":77.359999999999999,"15min":77.359999999999999},"last":2.2440000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024149,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1343700,"kb_used_data":647564,"kb_used_omap":5,"kb_used_meta":696122,"kb_avail":92401452,"statfs":{"total":95995035648,"available":94619086848,"internally_reserved":0,"allocated":663105536,"data_stored":661618175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":712829562},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.3769999999999998,"15min":2.8170000000000002},"min":{"1min":0.193,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.5369999999999999,"5min":185.17400000000001,"15min":185.17400000000001},"last":9.1820000000000004},{"interface":"front","average":{"1min":2.6280000000000001,"5min":4.5570000000000004,"15min":3.351},"min":{"1min":0.254,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":6.6319999999999997,"5min":186.691,"15min":186.691},"last":9.3369999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:25:36 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":2.4620000000000002,"15min":2.3010000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.17799999999999999},"max":{"1min":10.255000000000001,"5min":13.211,"15min":17.588999999999999},"last":1.139},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.6970000000000001,"15min":2.4049999999999998},"min":{"1min":0.33300000000000002,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":14.974,"5min":14.974,"15min":21.198},"last":10.215999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":6.3200000000000003,"15min":2.726},"min":{"1min":0.28499999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":7.375,"5min":185.89400000000001,"15min":185.89400000000001},"last":9.2989999999999995},{"interface":"front","average":{"1min":3.2120000000000002,"5min":6.6360000000000001,"15min":2.8530000000000002},"min":{"1min":0.27000000000000002,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":13.138999999999999,"5min":186.72900000000001,"15min":186.72900000000001},"last":9.3759999999999994}]},{"osd":4,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":1.599,"15min":1.627},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.9850000000000003,"5min":7.9660000000000002,"15min":15.835000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.7829999999999999,"15min":2.0590000000000002},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.25700000000000001},"max":{"1min":5.1310000000000002,"5min":8.4890000000000008,"15min":14.792999999999999},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":2.1699999999999999,"15min":2.036},"min":{"1min":0.33400000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.0780000000000003,"5min":62.984999999999999,"15min":62.984999999999999},"last":0.371},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.8479999999999999,"15min":2.5920000000000001},"min":{"1min":0.35199999999999998,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":3.8610000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":2.347}]},{"osd":6,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":2.3420000000000001,"15min":2.3759999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.4710000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":2.3090000000000002},{"interface":"front","average":{"1min":1.669,"5min":1.7949999999999999,"15min":1.9430000000000001},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.7439999999999998,"5min":10.606,"15min":15.893000000000001},"last":0.314}]},{"osd":7,"last update":"Mon Oct 9 06:24:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.516,"15min":2.3279999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28899999999999998},"max":{"1min":7.5010000000000003,"5min":62.262,"15min":62.262},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.3340000000000001,"5min":2.9740000000000002,"15min":2.645},"min":{"1min":0.377,"5min":0.377,"15min":0.314},"max":{"1min":14.167,"5min":62.229999999999997,"15min":62.229999999999997},"last":2.371}]}]},{"osd":2,"up_from":128,"seq":549755814017,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908868,"kb_used_data":528124,"kb_used_omap":7,"kb_used_meta":380536,"kb_avail":92836284,"statfs":{"total":95995035648,"available":95064354816,"internally_reserved":0,"allocated":540798976,"data_stored":539555612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":389668945},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":10.217000000000001,"5min":4.6269999999999998,"15min":3.968},"min":{"1min":0.22800000000000001,"5min":0.157,"15min":0.157},"max":{"1min":150.32599999999999,"5min":150.32599999999999,"15min":163.40100000000001},"last":3.9489999999999998},{"interface":"front","average":{"1min":3.1629999999999998,"5min":3.4260000000000002,"15min":3.8820000000000001},"min":{"1min":0.20899999999999999,"5min":0.182,"15min":0.182},"max":{"1min":21.472000000000001,"5min":110.66500000000001,"15min":163.363},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.8579999999999997,"15min":3.6680000000000001},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":7.9710000000000001,"5min":205.148,"15min":222.50299999999999},"last":4.0640000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":4.8280000000000003,"15min":3.984},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":11.811,"5min":205.267,"15min":222.19399999999999},"last":0.66200000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.2559999999999998,"5min":2.629,"15min":1.6759999999999999},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":16.331,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.9950000000000001,"5min":2.8159999999999998,"15min":1.6970000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":16.364000000000001,"5min":69.906999999999996,"15min":81.061999999999998},"last":7.2439999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.952,"15min":2.2240000000000002},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":52.188000000000002,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.94499999999999995},{"interface":"front","average":{"1min":3.6179999999999999,"5min":3.282,"15min":2.4910000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":52.167999999999999,"5min":90.950000000000003,"15min":90.950000000000003},"last":2.8159999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.472,"15min":1.534},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.309},"max":{"1min":5.508,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":3.859},{"interface":"front","average":{"1min":1.256,"5min":1.7549999999999999,"15min":2.0990000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.9349999999999996,"5min":10.548,"15min":10.548},"last":2.8540000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":3.7839999999999998,"15min":2.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":55.392000000000003,"5min":138.291,"15min":138.291},"last":3.2570000000000001},{"interface":"front","average":{"1min":4.8150000000000004,"5min":3.4910000000000001,"15min":2.206},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":55.146999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":3.9980000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9269999999999996,"5min":2.2709999999999999,"15min":1.863},"min":{"1min":0.40699999999999997,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":55.851999999999997,"5min":55.851999999999997,"15min":55.851999999999997},"last":3.9740000000000002},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.1890000000000001,"15min":2.395},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":55.731000000000002,"5min":55.731000000000002,"15min":55.731000000000002},"last":3.899}]}]},{"osd":3,"up_from":149,"seq":639950127215,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":720684,"kb_used_data":342436,"kb_used_omap":2,"kb_used_meta":378237,"kb_avail":93024468,"statfs":{"total":95995035648,"available":95257055232,"internally_reserved":0,"allocated":350654464,"data_stored":349492563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":387314741},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":8.4339999999999993,"5min":8.0909999999999993,"15min":3.198},"min":{"1min":0.308,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":105.252,"5min":187.852,"15min":187.852},"last":4.8460000000000001},{"interface":"front","average":{"1min":8.1020000000000003,"5min":8.7400000000000002,"15min":3.7010000000000001},"min":{"1min":0.245,"5min":0.20100000000000001,"15min":0.191},"max":{"1min":105.331,"5min":187.923,"15min":187.923},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3650000000000002,"5min":6.1289999999999996,"15min":2.8679999999999999},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.223},"max":{"1min":22.045999999999999,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":3.77,"5min":3.7010000000000001,"15min":1.847},"min":{"1min":0.26600000000000001,"5min":0.216,"15min":0.216},"max":{"1min":31.053000000000001,"5min":67.028000000000006,"15min":67.028000000000006},"last":4.8710000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":5.5049999999999999,"15min":2.6110000000000002},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.891999999999999,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.8470000000000004,"15min":2.008},"min":{"1min":0.23400000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":12.202,"5min":187.79499999999999,"15min":187.79499999999999},"last":4.758}]},{"osd":4,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.6639999999999999,"15min":1.05},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":4.2590000000000003,"5min":22.585999999999999,"15min":22.585999999999999},"last":4.7999999999999998},{"interface":"front","average":{"1min":1.8779999999999999,"5min":3.5790000000000002,"15min":1.992},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1370000000000005,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.7749999999999999,"15min":2.1320000000000001},"min":{"1min":0.379,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":8.218,"5min":187.833,"15min":187.833},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.995,"5min":3.4350000000000001,"15min":2.0619999999999998},"min":{"1min":0.33800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.7249999999999996,"5min":187.952,"15min":187.952},"last":3.8319999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":3.762,"15min":2.1030000000000002},"min":{"1min":0.34899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1809999999999992,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.439},{"interface":"front","average":{"1min":1.2989999999999999,"5min":3.3460000000000001,"15min":1.998},"min":{"1min":0.36499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.7030000000000003,"5min":187.81800000000001,"15min":187.81800000000001},"last":3.8719999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.734,"15min":1.7889999999999999},"min":{"1min":0.37,"5min":0.35599999999999998,"15min":0.34100000000000003},"max":{"1min":4.2960000000000003,"5min":192.20400000000001,"15min":192.20400000000001},"last":4.8250000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":3.754,"15min":2.0920000000000001},"min":{"1min":0.38800000000000001,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":8.2509999999999994,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.46400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019251,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1754500,"kb_used_data":863420,"kb_used_omap":0,"kb_used_meta":890880,"kb_avail":91990652,"statfs":{"total":95995035648,"available":94198427648,"internally_reserved":0,"allocated":884142080,"data_stored":883371676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912261120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":3.8069999999999999,"15min":2.3140000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.18,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.498},{"interface":"front","average":{"1min":5.444,"5min":4.2999999999999998,"15min":2.706},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":48.287999999999997,"5min":76.988,"15min":76.988},"last":0.55200000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.9890000000000001,"15min":2.0699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.8069999999999999,"5min":37.383000000000003,"15min":91.849999999999994},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.1869999999999998,"15min":2.177},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.5869999999999997,"5min":37.436999999999998,"15min":91.885000000000005},"last":2.4750000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.9240000000000004,"5min":2.8490000000000002,"15min":2.093},"min":{"1min":0.41599999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":44.807000000000002,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":5.2370000000000001,"5min":3.0139999999999998,"15min":2.278},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":44.697000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":9.0250000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":3.6480000000000001,"15min":2.1459999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.262,"5min":57.826999999999998,"15min":91.697000000000003},"last":7.8360000000000003},{"interface":"front","average":{"1min":1.7669999999999999,"5min":3.6680000000000001,"15min":2.081},"min":{"1min":0.35899999999999999,"5min":0.33400000000000002,"15min":0.28699999999999998},"max":{"1min":6.3250000000000002,"5min":57.853000000000002,"15min":94.356999999999999},"last":2.5379999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1619999999999999,"15min":1.107},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":4.2270000000000003,"5min":10.673,"15min":10.673},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.21,"5min":1.014,"15min":1.23},"min":{"1min":0.29599999999999999,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":4.6189999999999998,"5min":9.6379999999999999,"15min":12.552},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.77400000000000002,"15min":0.83499999999999996},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":1.8740000000000001,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.212,"15min":1.244},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5419999999999998,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.956,"15min":1.49},"min":{"1min":0.34499999999999997,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":5.4480000000000004,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.857,"15min":1.4770000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0640000000000001,"5min":62.573,"15min":62.573},"last":0.60299999999999998}]}]},{"osd":5,"up_from":0,"seq":163208757616,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":642715648,"data_stored":642711576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":389742592,"data_stored":389742592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532676608,"data_stored":532676608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":343212032,"data_stored":343212032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":859705344,"data_stored":859701272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":454365184,"data_stored":454361112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":309329920,"data_stored":309329920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:25:54.294 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:54.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:54.399 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:54.488 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:54 smithi107 ceph-mon[212750]: pgmap v895: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 682 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 16/32796 objects degraded (0.049%); 559/32796 objects misplaced (1.704%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:54.488 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:54 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:25:54.489 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:54 smithi107 ceph-mon[220921]: pgmap v895: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 682 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 16/32796 objects degraded (0.049%); 559/32796 objects misplaced (1.704%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:54.489 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:54 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:25:54.489 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:54.584 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:54.634 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:54.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:54.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:54.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:54 smithi150 ceph-mon[194178]: pgmap v895: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 682 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 16/32796 objects degraded (0.049%); 559/32796 objects misplaced (1.704%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:54.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:54 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:25:54.746 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:25:54.799 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:25:54.799 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:25:54.821 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:55.582 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:55 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/31992 objects degraded (0.050%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:55.583 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:55 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/31992 objects degraded (0.050%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:55.606 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:25:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:55.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:25:55.714 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:55.758 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:55 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/31992 objects degraded (0.050%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:25:55.762 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:55.825 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:55.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:55.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:55.931 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:25:55.977 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:56.192 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:25:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:56 smithi107 ceph-mon[212750]: pgmap v896: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 665 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 16/31992 objects degraded (0.050%); 559/31992 objects misplaced (1.747%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:56.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:56 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:25:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:56 smithi107 ceph-mon[220921]: pgmap v896: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 665 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 16/31992 objects degraded (0.050%); 559/31992 objects misplaced (1.747%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:56 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:25:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:56 smithi150 ceph-mon[194178]: pgmap v896: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 665 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 801 op/s; 16/31992 objects degraded (0.050%); 559/31992 objects misplaced (1.747%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:25:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:56 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:25:57.106 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:25:57.303 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:25:57.443 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:25:57.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:57.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:25:57.558 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:57.589 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:57.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:57.649 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:57.749 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:57.779 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:57.856 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:57.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:57.966 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:25:58.003 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:58.073 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:25:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:58.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:25:58.185 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:25:58.253 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:58.335 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:58.397 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:25:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:58.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:25:58.463 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:25:58.495 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:25:58.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:58 smithi107 ceph-mon[220921]: pgmap v897: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 553 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/26580 objects degraded (0.060%); 214/26580 objects misplaced (0.805%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:58.508 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4127334952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:58.510 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:58 smithi107 ceph-mon[212750]: pgmap v897: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 553 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/26580 objects degraded (0.060%); 214/26580 objects misplaced (0.805%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:58.510 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4127334952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:58.510 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:25:58.535 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:25:58.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:58.581 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:25:58.606 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:25:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:58.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:25:58.630 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:25:58.678 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:25:58.716 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:58.724 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:25:58.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:58 smithi150 ceph-mon[194178]: pgmap v897: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 553 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/26580 objects degraded (0.060%); 214/26580 objects misplaced (0.805%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:25:58.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4127334952' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:25:58.749 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:58.766 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:25:58.770 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.780 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:25:58.811 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:58.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:58.917 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:25:58.999 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:59.390 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:25:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:59.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:25:59.503 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:25:59.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:59.624 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:25:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:59.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:25:59.735 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.760 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:25:59 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:25:59.763 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:25:59.821 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:25:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:25:59.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:25:59.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:25:59 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:25:59.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:25:59 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:25:59.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:25:59.977 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:00.044 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:00.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:00.152 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:00.218 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:00.421 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:00.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:00.470 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:00 smithi107 ceph-mon[220921]: pgmap v898: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 414 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 16/19947 objects degraded (0.080%); 214/19947 objects misplaced (1.073%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:26:00.470 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:00 smithi107 ceph-mon[220921]: 10.12 scrub starts 2023-10-09T06:26:00.527 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:00.608 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:00 smithi150 ceph-mon[194178]: pgmap v898: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 414 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 16/19947 objects degraded (0.080%); 214/19947 objects misplaced (1.073%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:26:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:00 smithi150 ceph-mon[194178]: 10.12 scrub starts 2023-10-09T06:26:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:00 smithi107 ceph-mon[212750]: pgmap v898: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 414 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 16/19947 objects degraded (0.080%); 214/19947 objects misplaced (1.073%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:26:00.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:00 smithi107 ceph-mon[212750]: 10.12 scrub starts 2023-10-09T06:26:01.253 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:01.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:01.366 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:01.404 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:01.478 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:01.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:01.584 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:01.627 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:01 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/17370 objects degraded (0.092%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:01.669 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:01.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:01.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:01.775 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:01.856 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:01.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:01 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/17370 objects degraded (0.092%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:01.862 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:01 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/17370 objects degraded (0.092%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:02.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:02.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:02.121 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:02] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T06:26:02.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:02.217 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:02.286 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:02.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:02.394 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:02.430 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:02.497 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:02.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:02.603 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:02.637 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:02.696 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:02.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:02.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[212750]: pgmap v899: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 361 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 16/17370 objects degraded (0.092%); 16/17370 objects misplaced (0.092%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:02.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:02.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:02.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[220921]: pgmap v899: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 361 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 16/17370 objects degraded (0.092%); 16/17370 objects misplaced (0.092%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:02.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:02.765 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:02.804 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:02.886 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:02 smithi150 ceph-mon[194178]: pgmap v899: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 361 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 16/17370 objects degraded (0.092%); 16/17370 objects misplaced (0.092%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:03.408 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:03.410 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:26:03.604 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:03.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:03.650 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:26:03.692 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4260157212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.692 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.692 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4260157212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.692 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.718 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:03.771 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:03.842 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:03.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4260157212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:03.955 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:03.995 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:04.046 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:04.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:04.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:04.153 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:04.179 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:04.589 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:04.597 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:26:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:04.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:04.658 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":900,"stamp":"2023-10-09T06:26:03.134000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":183441336,"num_objects":2817,"num_object_clones":0,"num_object_copies":8451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":16,"num_objects_unfound":0,"num_objects_dirty":2817,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186814,"num_write_kb":5460638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12003,"num_bytes_recovered":690235482,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":202254,"ondisk_log_size":202254,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6042688,"kb_used_data":1392328,"kb_used_omap":20,"kb_used_meta":4650155,"kb_avail":650173376,"statfs":{"total":671965249536,"available":665777537024,"internally_reserved":0,"allocated":1425743872,"data_stored":1418250635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4761759374},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-724697088,"num_objects":-11058,"num_object_clones":0,"num_object_copies":-33174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-543,"num_objects_unfound":0,"num_objects_dirty":-11058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11058,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":339,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.827451"},"pg_stats":[{"pgid":"10.13","version":"291'9003","reported_seq":17517,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.853686+0000","last_change":"2023-10-09T06:24:52.774608+0000","last_active":"2023-10-09T06:26:01.853686+0000","last_peered":"2023-10-09T06:26:01.853686+0000","last_clean":"2023-10-09T06:26:01.853686+0000","last_became_active":"2023-10-09T06:24:52.762870+0000","last_became_peered":"2023-10-09T06:24:52.762870+0000","last_unstale":"2023-10-09T06:26:01.853686+0000","last_undegraded":"2023-10-09T06:26:01.853686+0000","last_fullsized":"2023-10-09T06:26:01.853686+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":9003,"log_dups_size":0,"ondisk_log_size":9003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7405568,"num_objects":113,"num_object_clones":0,"num_object_copies":339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6423,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2810,"num_bytes_recovered":184156160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":270,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206277+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:25:35.206277+0000","last_peered":"2023-10-09T06:25:35.206277+0000","last_clean":"2023-10-09T06:25:35.206277+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:25:35.206277+0000","last_undegraded":"2023-10-09T06:25:35.206277+0000","last_fullsized":"2023-10-09T06:25:35.206277+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207069+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:25:35.207069+0000","last_peered":"2023-10-09T06:25:35.207069+0000","last_clean":"2023-10-09T06:25:35.207069+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:25:35.207069+0000","last_undegraded":"2023-10-09T06:25:35.207069+0000","last_fullsized":"2023-10-09T06:25:35.207069+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"291'8947","reported_seq":14530,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.817365+0000","last_change":"2023-10-09T06:25:08.543126+0000","last_active":"2023-10-09T06:26:02.817365+0000","last_peered":"2023-10-09T06:26:02.817365+0000","last_clean":"2023-10-09T06:26:02.817365+0000","last_became_active":"2023-10-09T06:23:58.145794+0000","last_became_peered":"2023-10-09T06:23:58.145794+0000","last_unstale":"2023-10-09T06:26:02.817365+0000","last_undegraded":"2023-10-09T06:26:02.817365+0000","last_fullsized":"2023-10-09T06:26:02.817365+0000","mapping_epoch":269,"log_start":"244'1973","ondisk_log_start":"244'1973","created":254,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6974,"log_dups_size":0,"ondisk_log_size":6974,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6619136,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6389,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2723,"num_bytes_recovered":178454528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":263,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.199831+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:25:35.199831+0000","last_peered":"2023-10-09T06:25:35.199831+0000","last_clean":"2023-10-09T06:25:35.199831+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:25:35.199831+0000","last_undegraded":"2023-10-09T06:25:35.199831+0000","last_fullsized":"2023-10-09T06:25:35.199831+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":262,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.210926+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:25:35.210926+0000","last_peered":"2023-10-09T06:25:35.210926+0000","last_clean":"2023-10-09T06:25:35.210926+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:25:35.210926+0000","last_undegraded":"2023-10-09T06:25:35.210926+0000","last_fullsized":"2023-10-09T06:25:35.210926+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"291'9035","reported_seq":14732,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.390482+0000","last_change":"2023-10-09T06:25:27.722935+0000","last_active":"2023-10-09T06:26:00.390482+0000","last_peered":"2023-10-09T06:26:00.390482+0000","last_clean":"2023-10-09T06:26:00.390482+0000","last_became_active":"2023-10-09T06:25:27.703733+0000","last_became_peered":"2023-10-09T06:25:27.703733+0000","last_unstale":"2023-10-09T06:26:00.390482+0000","last_undegraded":"2023-10-09T06:26:00.390482+0000","last_fullsized":"2023-10-09T06:26:00.390482+0000","mapping_epoch":284,"log_start":"242'791","ondisk_log_start":"242'791","created":254,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8244,"log_dups_size":0,"ondisk_log_size":8244,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8323072,"num_objects":127,"num_object_clones":0,"num_object_copies":381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6417,"num_write_kb":209408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2805,"num_bytes_recovered":93388800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":246,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204713+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:25:35.204713+0000","last_peered":"2023-10-09T06:25:35.204713+0000","last_clean":"2023-10-09T06:25:35.204713+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:25:35.204713+0000","last_undegraded":"2023-10-09T06:25:35.204713+0000","last_fullsized":"2023-10-09T06:25:35.204713+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":270,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203666+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:25:35.203666+0000","last_peered":"2023-10-09T06:25:35.203666+0000","last_clean":"2023-10-09T06:25:35.203666+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:25:35.203666+0000","last_undegraded":"2023-10-09T06:25:35.203666+0000","last_fullsized":"2023-10-09T06:25:35.203666+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.211304+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:25:35.211304+0000","last_peered":"2023-10-09T06:25:35.211304+0000","last_clean":"2023-10-09T06:25:35.211304+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:25:35.211304+0000","last_undegraded":"2023-10-09T06:25:35.211304+0000","last_fullsized":"2023-10-09T06:25:35.211304+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204663+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:25:35.204663+0000","last_peered":"2023-10-09T06:25:35.204663+0000","last_clean":"2023-10-09T06:25:35.204663+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:25:35.204663+0000","last_undegraded":"2023-10-09T06:25:35.204663+0000","last_fullsized":"2023-10-09T06:25:35.204663+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.208186+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:25:35.208186+0000","last_peered":"2023-10-09T06:25:35.208186+0000","last_clean":"2023-10-09T06:25:35.208186+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:25:35.208186+0000","last_undegraded":"2023-10-09T06:25:35.208186+0000","last_fullsized":"2023-10-09T06:25:35.208186+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204897+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:25:35.204897+0000","last_peered":"2023-10-09T06:25:35.204897+0000","last_clean":"2023-10-09T06:25:35.204897+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:25:35.204897+0000","last_undegraded":"2023-10-09T06:25:35.204897+0000","last_fullsized":"2023-10-09T06:25:35.204897+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":140,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207872+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:25:35.207872+0000","last_peered":"2023-10-09T06:25:35.207872+0000","last_clean":"2023-10-09T06:25:35.207872+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:25:35.207872+0000","last_undegraded":"2023-10-09T06:25:35.207872+0000","last_fullsized":"2023-10-09T06:25:35.207872+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.202999+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:25:35.202999+0000","last_peered":"2023-10-09T06:25:35.202999+0000","last_clean":"2023-10-09T06:25:35.202999+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:25:35.202999+0000","last_undegraded":"2023-10-09T06:25:35.202999+0000","last_fullsized":"2023-10-09T06:25:35.202999+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":297,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206715+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:25:35.206715+0000","last_peered":"2023-10-09T06:25:35.206715+0000","last_clean":"2023-10-09T06:25:35.206715+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:25:35.206715+0000","last_undegraded":"2023-10-09T06:25:35.206715+0000","last_fullsized":"2023-10-09T06:25:35.206715+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":261,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206615+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:25:35.206615+0000","last_peered":"2023-10-09T06:25:35.206615+0000","last_clean":"2023-10-09T06:25:35.206615+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:25:35.206615+0000","last_undegraded":"2023-10-09T06:25:35.206615+0000","last_fullsized":"2023-10-09T06:25:35.206615+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":140,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207557+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:25:35.207557+0000","last_peered":"2023-10-09T06:25:35.207557+0000","last_clean":"2023-10-09T06:25:35.207557+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:25:35.207557+0000","last_undegraded":"2023-10-09T06:25:35.207557+0000","last_fullsized":"2023-10-09T06:25:35.207557+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":246,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206382+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:25:35.206382+0000","last_peered":"2023-10-09T06:25:35.206382+0000","last_clean":"2023-10-09T06:25:35.206382+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:25:35.206382+0000","last_undegraded":"2023-10-09T06:25:35.206382+0000","last_fullsized":"2023-10-09T06:25:35.206382+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":261,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204432+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:25:35.204432+0000","last_peered":"2023-10-09T06:25:35.204432+0000","last_clean":"2023-10-09T06:25:35.204432+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:25:35.204432+0000","last_undegraded":"2023-10-09T06:25:35.204432+0000","last_fullsized":"2023-10-09T06:25:35.204432+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207782+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:25:35.207782+0000","last_peered":"2023-10-09T06:25:35.207782+0000","last_clean":"2023-10-09T06:25:35.207782+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:25:35.207782+0000","last_undegraded":"2023-10-09T06:25:35.207782+0000","last_fullsized":"2023-10-09T06:25:35.207782+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"291'8866","reported_seq":8938,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.361551+0000","last_change":"2023-10-09T06:23:26.837327+0000","last_active":"2023-10-09T06:26:00.361551+0000","last_peered":"2023-10-09T06:26:00.361551+0000","last_clean":"2023-10-09T06:26:00.361551+0000","last_became_active":"2023-10-09T06:23:26.835211+0000","last_became_peered":"2023-10-09T06:23:26.835211+0000","last_unstale":"2023-10-09T06:26:00.361551+0000","last_undegraded":"2023-10-09T06:26:00.361551+0000","last_fullsized":"2023-10-09T06:26:00.361551+0000","mapping_epoch":254,"log_start":"242'585","ondisk_log_start":"242'585","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8281,"log_dups_size":0,"ondisk_log_size":8281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8617984,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6314,"num_write_kb":206240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":134,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207575+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:25:35.207575+0000","last_peered":"2023-10-09T06:25:35.207575+0000","last_clean":"2023-10-09T06:25:35.207575+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:25:35.207575+0000","last_undegraded":"2023-10-09T06:25:35.207575+0000","last_fullsized":"2023-10-09T06:25:35.207575+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":261,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.211071+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:25:35.211071+0000","last_peered":"2023-10-09T06:25:35.211071+0000","last_clean":"2023-10-09T06:25:35.211071+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:25:35.211071+0000","last_undegraded":"2023-10-09T06:25:35.211071+0000","last_fullsized":"2023-10-09T06:25:35.211071+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"291'8959","reported_seq":9031,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.700576+0000","last_change":"2023-10-09T06:23:26.839514+0000","last_active":"2023-10-09T06:26:01.700576+0000","last_peered":"2023-10-09T06:26:01.700576+0000","last_clean":"2023-10-09T06:26:01.700576+0000","last_became_active":"2023-10-09T06:23:26.837320+0000","last_became_peered":"2023-10-09T06:23:26.837320+0000","last_unstale":"2023-10-09T06:26:01.700576+0000","last_undegraded":"2023-10-09T06:26:01.700576+0000","last_fullsized":"2023-10-09T06:26:01.700576+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8959,"log_dups_size":0,"ondisk_log_size":8959,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7438336,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":208224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":279,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.202864+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:25:35.202864+0000","last_peered":"2023-10-09T06:25:35.202864+0000","last_clean":"2023-10-09T06:25:35.202864+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:25:35.202864+0000","last_undegraded":"2023-10-09T06:25:35.202864+0000","last_fullsized":"2023-10-09T06:25:35.202864+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":246,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207382+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:25:35.207382+0000","last_peered":"2023-10-09T06:25:35.207382+0000","last_clean":"2023-10-09T06:25:35.207382+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:25:35.207382+0000","last_undegraded":"2023-10-09T06:25:35.207382+0000","last_fullsized":"2023-10-09T06:25:35.207382+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"291'8984","reported_seq":9125,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.328841+0000","last_change":"2023-10-09T06:23:31.394134+0000","last_active":"2023-10-09T06:26:00.328841+0000","last_peered":"2023-10-09T06:26:00.328841+0000","last_clean":"2023-10-09T06:26:00.328841+0000","last_became_active":"2023-10-09T06:23:26.818462+0000","last_became_peered":"2023-10-09T06:23:26.818462+0000","last_unstale":"2023-10-09T06:26:00.328841+0000","last_undegraded":"2023-10-09T06:26:00.328841+0000","last_fullsized":"2023-10-09T06:26:00.328841+0000","mapping_epoch":238,"log_start":"243'935","ondisk_log_start":"243'935","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5269","last_scrub_stamp":"2023-10-09T06:23:31.393974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:31.393974+0000","objects_scrubbed":2761,"log_size":8049,"log_dups_size":0,"ondisk_log_size":8049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:10.395411+0000","scrub_duration":0.69991382499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8060928,"num_objects":123,"num_object_clones":0,"num_object_copies":369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3722,"num_write_kb":34688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":152,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207989+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:25:35.207989+0000","last_peered":"2023-10-09T06:25:35.207989+0000","last_clean":"2023-10-09T06:25:35.207989+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:25:35.207989+0000","last_undegraded":"2023-10-09T06:25:35.207989+0000","last_fullsized":"2023-10-09T06:25:35.207989+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":201,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207298+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:25:35.207298+0000","last_peered":"2023-10-09T06:25:35.207298+0000","last_clean":"2023-10-09T06:25:35.207298+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:25:35.207298+0000","last_undegraded":"2023-10-09T06:25:35.207298+0000","last_fullsized":"2023-10-09T06:25:35.207298+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"291'10744","reported_seq":10932,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.920674+0000","last_change":"2023-10-09T06:23:37.300870+0000","last_active":"2023-10-09T06:26:02.920674+0000","last_peered":"2023-10-09T06:26:02.920674+0000","last_clean":"2023-10-09T06:26:02.920674+0000","last_became_active":"2023-10-09T06:23:26.814403+0000","last_became_peered":"2023-10-09T06:23:26.814403+0000","last_unstale":"2023-10-09T06:26:02.920674+0000","last_undegraded":"2023-10-09T06:26:02.920674+0000","last_fullsized":"2023-10-09T06:26:02.920674+0000","mapping_epoch":238,"log_start":"249'4373","ondisk_log_start":"249'4373","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"260'7161","last_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_deep_scrub":"260'7161","last_deep_scrub_stamp":"2023-10-09T06:23:37.300804+0000","last_clean_scrub_stamp":"2023-10-09T06:23:37.300804+0000","objects_scrubbed":2668,"log_size":6371,"log_dups_size":0,"ondisk_log_size":6371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":8,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:02.877562+0000","scrub_duration":7.3906703120000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3642,"num_write_kb":32896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":138,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.208107+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:25:35.208107+0000","last_peered":"2023-10-09T06:25:35.208107+0000","last_clean":"2023-10-09T06:25:35.208107+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:25:35.208107+0000","last_undegraded":"2023-10-09T06:25:35.208107+0000","last_fullsized":"2023-10-09T06:25:35.208107+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":268,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206630+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:25:35.206630+0000","last_peered":"2023-10-09T06:25:35.206630+0000","last_clean":"2023-10-09T06:25:35.206630+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:25:35.206630+0000","last_undegraded":"2023-10-09T06:25:35.206630+0000","last_fullsized":"2023-10-09T06:25:35.206630+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"291'9122","reported_seq":9195,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.303738+0000","last_change":"2023-10-09T06:23:26.802751+0000","last_active":"2023-10-09T06:26:00.303738+0000","last_peered":"2023-10-09T06:26:00.303738+0000","last_clean":"2023-10-09T06:26:00.303738+0000","last_became_active":"2023-10-09T06:23:26.800034+0000","last_became_peered":"2023-10-09T06:23:26.800034+0000","last_unstale":"2023-10-09T06:26:00.303738+0000","last_undegraded":"2023-10-09T06:26:00.303738+0000","last_fullsized":"2023-10-09T06:26:00.303738+0000","mapping_epoch":238,"log_start":"243'1205","ondisk_log_start":"243'1205","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":7917,"log_dups_size":0,"ondisk_log_size":7917,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1933312,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6519,"num_write_kb":209568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":152,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207720+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:25:35.207720+0000","last_peered":"2023-10-09T06:25:35.207720+0000","last_clean":"2023-10-09T06:25:35.207720+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:25:35.207720+0000","last_undegraded":"2023-10-09T06:25:35.207720+0000","last_fullsized":"2023-10-09T06:25:35.207720+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":270,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203253+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:25:35.203253+0000","last_peered":"2023-10-09T06:25:35.203253+0000","last_clean":"2023-10-09T06:25:35.203253+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:25:35.203253+0000","last_undegraded":"2023-10-09T06:25:35.203253+0000","last_fullsized":"2023-10-09T06:25:35.203253+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"291'8941","reported_seq":9014,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.417416+0000","last_change":"2023-10-09T06:23:26.796849+0000","last_active":"2023-10-09T06:26:01.417416+0000","last_peered":"2023-10-09T06:26:01.417416+0000","last_clean":"2023-10-09T06:26:01.417416+0000","last_became_active":"2023-10-09T06:23:26.794430+0000","last_became_peered":"2023-10-09T06:23:26.794430+0000","last_unstale":"2023-10-09T06:26:01.417416+0000","last_undegraded":"2023-10-09T06:26:01.417416+0000","last_fullsized":"2023-10-09T06:26:01.417416+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8941,"log_dups_size":0,"ondisk_log_size":8941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":983040,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6399,"num_write_kb":205248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206252+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:25:35.206252+0000","last_peered":"2023-10-09T06:25:35.206252+0000","last_clean":"2023-10-09T06:25:35.206252+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:25:35.206252+0000","last_undegraded":"2023-10-09T06:25:35.206252+0000","last_fullsized":"2023-10-09T06:25:35.206252+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":243,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203605+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:25:35.203605+0000","last_peered":"2023-10-09T06:25:35.203605+0000","last_clean":"2023-10-09T06:25:35.203605+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:25:35.203605+0000","last_undegraded":"2023-10-09T06:25:35.203605+0000","last_fullsized":"2023-10-09T06:25:35.203605+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"291'8997","reported_seq":9141,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.801859+0000","last_change":"2023-10-09T06:23:32.332291+0000","last_active":"2023-10-09T06:26:01.801859+0000","last_peered":"2023-10-09T06:26:01.801859+0000","last_clean":"2023-10-09T06:26:01.801859+0000","last_became_active":"2023-10-09T06:23:26.843612+0000","last_became_peered":"2023-10-09T06:23:26.843612+0000","last_unstale":"2023-10-09T06:26:01.801859+0000","last_undegraded":"2023-10-09T06:26:01.801859+0000","last_fullsized":"2023-10-09T06:26:01.801859+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5297","last_scrub_stamp":"2023-10-09T06:23:32.331879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:32.331879+0000","objects_scrubbed":2748,"log_size":8997,"log_dups_size":0,"ondisk_log_size":8997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:09.057470+0000","scrub_duration":0.54202623999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5177344,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3703,"num_write_kb":33088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":583,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207310+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:25:35.207310+0000","last_peered":"2023-10-09T06:25:35.207310+0000","last_clean":"2023-10-09T06:25:35.207310+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:25:35.207310+0000","last_undegraded":"2023-10-09T06:25:35.207310+0000","last_fullsized":"2023-10-09T06:25:35.207310+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":255,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:43.818634+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:25:43.818634+0000","last_peered":"2023-10-09T06:25:43.818634+0000","last_clean":"2023-10-09T06:25:43.818634+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:25:43.818634+0000","last_undegraded":"2023-10-09T06:25:43.818634+0000","last_fullsized":"2023-10-09T06:25:43.818634+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":251,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206790+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:25:35.206790+0000","last_peered":"2023-10-09T06:25:35.206790+0000","last_clean":"2023-10-09T06:25:35.206790+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:25:35.206790+0000","last_undegraded":"2023-10-09T06:25:35.206790+0000","last_fullsized":"2023-10-09T06:25:35.206790+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"291'9179","reported_seq":9252,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.536517+0000","last_change":"2023-10-09T06:23:37.908859+0000","last_active":"2023-10-09T06:26:01.536517+0000","last_peered":"2023-10-09T06:26:01.536517+0000","last_clean":"2023-10-09T06:26:01.536517+0000","last_became_active":"2023-10-09T06:23:26.803856+0000","last_became_peered":"2023-10-09T06:23:26.803856+0000","last_unstale":"2023-10-09T06:26:01.536517+0000","last_undegraded":"2023-10-09T06:26:01.536517+0000","last_fullsized":"2023-10-09T06:26:01.536517+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":9179,"log_dups_size":0,"ondisk_log_size":9179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6599,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":574,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.199344+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:25:35.199344+0000","last_peered":"2023-10-09T06:25:35.199344+0000","last_clean":"2023-10-09T06:25:35.199344+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:25:35.199344+0000","last_undegraded":"2023-10-09T06:25:35.199344+0000","last_fullsized":"2023-10-09T06:25:35.199344+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203100+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:25:35.203100+0000","last_peered":"2023-10-09T06:25:35.203100+0000","last_clean":"2023-10-09T06:25:35.203100+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:25:35.203100+0000","last_undegraded":"2023-10-09T06:25:35.203100+0000","last_fullsized":"2023-10-09T06:25:35.203100+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":268,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206869+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:25:35.206869+0000","last_peered":"2023-10-09T06:25:35.206869+0000","last_clean":"2023-10-09T06:25:35.206869+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:25:35.206869+0000","last_undegraded":"2023-10-09T06:25:35.206869+0000","last_fullsized":"2023-10-09T06:25:35.206869+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"291'9018","reported_seq":9204,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.842868+0000","last_change":"2023-10-09T06:23:27.601610+0000","last_active":"2023-10-09T06:26:02.842868+0000","last_peered":"2023-10-09T06:26:02.842868+0000","last_clean":"2023-10-09T06:26:02.842868+0000","last_became_active":"2023-10-09T06:23:26.818025+0000","last_became_peered":"2023-10-09T06:23:26.818025+0000","last_unstale":"2023-10-09T06:26:02.842868+0000","last_undegraded":"2023-10-09T06:26:02.842868+0000","last_fullsized":"2023-10-09T06:26:02.842868+0000","mapping_epoch":238,"log_start":"243'981","ondisk_log_start":"243'981","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5147","last_scrub_stamp":"2023-10-09T06:23:27.601513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:27.601513+0000","objects_scrubbed":2616,"log_size":8037,"log_dups_size":0,"ondisk_log_size":8037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:51.587896+0000","scrub_duration":0.62301826800000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2818048,"num_objects":43,"num_object_clones":0,"num_object_copies":129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3883,"num_write_kb":41920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206353+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:25:35.206353+0000","last_peered":"2023-10-09T06:25:35.206353+0000","last_clean":"2023-10-09T06:25:35.206353+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:25:35.206353+0000","last_undegraded":"2023-10-09T06:25:35.206353+0000","last_fullsized":"2023-10-09T06:25:35.206353+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":251,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207400+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:25:35.207400+0000","last_peered":"2023-10-09T06:25:35.207400+0000","last_clean":"2023-10-09T06:25:35.207400+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:25:35.207400+0000","last_undegraded":"2023-10-09T06:25:35.207400+0000","last_fullsized":"2023-10-09T06:25:35.207400+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"291'9210","reported_seq":9403,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.956755+0000","last_change":"2023-10-09T06:23:33.751853+0000","last_active":"2023-10-09T06:26:02.956755+0000","last_peered":"2023-10-09T06:26:02.956755+0000","last_clean":"2023-10-09T06:26:02.956755+0000","last_became_active":"2023-10-09T06:23:26.806964+0000","last_became_peered":"2023-10-09T06:23:26.806964+0000","last_unstale":"2023-10-09T06:26:02.956755+0000","last_undegraded":"2023-10-09T06:26:02.956755+0000","last_fullsized":"2023-10-09T06:26:02.956755+0000","mapping_epoch":238,"log_start":"245'2425","ondisk_log_start":"245'2425","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5434","last_scrub_stamp":"2023-10-09T06:23:33.751788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:33.751788+0000","objects_scrubbed":2820,"log_size":6785,"log_dups_size":0,"ondisk_log_size":6785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:29.179833+0000","scrub_duration":0.67656161599999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2949120,"num_objects":45,"num_object_clones":0,"num_object_copies":135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3785,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":570,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.199929+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:25:35.199929+0000","last_peered":"2023-10-09T06:25:35.199929+0000","last_clean":"2023-10-09T06:25:35.199929+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:25:35.199929+0000","last_undegraded":"2023-10-09T06:25:35.199929+0000","last_fullsized":"2023-10-09T06:25:35.199929+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":248,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204601+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:25:35.204601+0000","last_peered":"2023-10-09T06:25:35.204601+0000","last_clean":"2023-10-09T06:25:35.204601+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:25:35.204601+0000","last_undegraded":"2023-10-09T06:25:35.204601+0000","last_fullsized":"2023-10-09T06:25:35.204601+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":222,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.199781+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:25:35.199781+0000","last_peered":"2023-10-09T06:25:35.199781+0000","last_clean":"2023-10-09T06:25:35.199781+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:25:35.199781+0000","last_undegraded":"2023-10-09T06:25:35.199781+0000","last_fullsized":"2023-10-09T06:25:35.199781+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"291'12339","reported_seq":12400,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:58.136905+0000","last_change":"2023-10-09T06:22:15.196013+0000","last_active":"2023-10-09T06:25:58.136905+0000","last_peered":"2023-10-09T06:25:58.136905+0000","last_clean":"2023-10-09T06:25:58.136905+0000","last_became_active":"2023-10-09T06:22:15.195711+0000","last_became_peered":"2023-10-09T06:22:15.195711+0000","last_unstale":"2023-10-09T06:25:58.136905+0000","last_undegraded":"2023-10-09T06:25:58.136905+0000","last_fullsized":"2023-10-09T06:25:58.136905+0000","mapping_epoch":238,"log_start":"265'5600","ondisk_log_start":"265'5600","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:00.826763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23658496,"num_objects":361,"num_object_clones":0,"num_object_copies":1083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12339,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":513,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203755+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:25:35.203755+0000","last_peered":"2023-10-09T06:25:35.203755+0000","last_clean":"2023-10-09T06:25:35.203755+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:25:35.203755+0000","last_undegraded":"2023-10-09T06:25:35.203755+0000","last_fullsized":"2023-10-09T06:25:35.203755+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.201835+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:25:35.201835+0000","last_peered":"2023-10-09T06:25:35.201835+0000","last_clean":"2023-10-09T06:25:35.201835+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:25:35.201835+0000","last_undegraded":"2023-10-09T06:25:35.201835+0000","last_fullsized":"2023-10-09T06:25:35.201835+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":200,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:51.330885+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:25:51.330885+0000","last_peered":"2023-10-09T06:25:51.330885+0000","last_clean":"2023-10-09T06:25:51.330885+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:25:51.330885+0000","last_undegraded":"2023-10-09T06:25:51.330885+0000","last_fullsized":"2023-10-09T06:25:51.330885+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":217,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.202506+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:25:35.202506+0000","last_peered":"2023-10-09T06:25:35.202506+0000","last_clean":"2023-10-09T06:25:35.202506+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:25:35.202506+0000","last_undegraded":"2023-10-09T06:25:35.202506+0000","last_fullsized":"2023-10-09T06:25:35.202506+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"291'13033","reported_seq":13094,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.885604+0000","last_change":"2023-10-09T06:22:15.194332+0000","last_active":"2023-10-09T06:26:02.885604+0000","last_peered":"2023-10-09T06:26:02.885604+0000","last_clean":"2023-10-09T06:26:02.885604+0000","last_became_active":"2023-10-09T06:22:15.193967+0000","last_became_peered":"2023-10-09T06:22:15.193967+0000","last_unstale":"2023-10-09T06:26:02.885604+0000","last_undegraded":"2023-10-09T06:26:02.885604+0000","last_fullsized":"2023-10-09T06:26:02.885604+0000","mapping_epoch":238,"log_start":"271'7000","ondisk_log_start":"271'7000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:57.093196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6750208,"num_objects":103,"num_object_clones":0,"num_object_copies":309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13033,"num_write_kb":420352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":576,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204341+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:25:35.204341+0000","last_peered":"2023-10-09T06:25:35.204341+0000","last_clean":"2023-10-09T06:25:35.204341+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:25:35.204341+0000","last_undegraded":"2023-10-09T06:25:35.204341+0000","last_fullsized":"2023-10-09T06:25:35.204341+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":148,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:51.336964+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:25:51.336964+0000","last_peered":"2023-10-09T06:25:51.336964+0000","last_clean":"2023-10-09T06:25:51.336964+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:25:51.336964+0000","last_undegraded":"2023-10-09T06:25:51.336964+0000","last_fullsized":"2023-10-09T06:25:51.336964+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":252,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206512+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:25:35.206512+0000","last_peered":"2023-10-09T06:25:35.206512+0000","last_clean":"2023-10-09T06:25:35.206512+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:25:35.206512+0000","last_undegraded":"2023-10-09T06:25:35.206512+0000","last_fullsized":"2023-10-09T06:25:35.206512+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.201796+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:25:35.201796+0000","last_peered":"2023-10-09T06:25:35.201796+0000","last_clean":"2023-10-09T06:25:35.201796+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:25:35.201796+0000","last_undegraded":"2023-10-09T06:25:35.201796+0000","last_fullsized":"2023-10-09T06:25:35.201796+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"291'9124","reported_seq":9261,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.373587+0000","last_change":"2023-10-09T06:23:28.321984+0000","last_active":"2023-10-09T06:26:01.373587+0000","last_peered":"2023-10-09T06:26:01.373587+0000","last_clean":"2023-10-09T06:26:01.373587+0000","last_became_active":"2023-10-09T06:23:26.803777+0000","last_became_peered":"2023-10-09T06:23:26.803777+0000","last_unstale":"2023-10-09T06:26:01.373587+0000","last_undegraded":"2023-10-09T06:26:01.373587+0000","last_fullsized":"2023-10-09T06:26:01.373587+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"255'5183","last_scrub_stamp":"2023-10-09T06:23:28.321788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:28.321788+0000","objects_scrubbed":2637,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:06.754123+0000","scrub_duration":0.61668156900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5308416,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3952,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":583,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207786+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:25:35.207786+0000","last_peered":"2023-10-09T06:25:35.207786+0000","last_clean":"2023-10-09T06:25:35.207786+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:25:35.207786+0000","last_undegraded":"2023-10-09T06:25:35.207786+0000","last_fullsized":"2023-10-09T06:25:35.207786+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":257,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.202320+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:25:35.202320+0000","last_peered":"2023-10-09T06:25:35.202320+0000","last_clean":"2023-10-09T06:25:35.202320+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:25:35.202320+0000","last_undegraded":"2023-10-09T06:25:35.202320+0000","last_fullsized":"2023-10-09T06:25:35.202320+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":264,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.202179+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:25:35.202179+0000","last_peered":"2023-10-09T06:25:35.202179+0000","last_clean":"2023-10-09T06:25:35.202179+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:25:35.202179+0000","last_undegraded":"2023-10-09T06:25:35.202179+0000","last_fullsized":"2023-10-09T06:25:35.202179+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":260,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:51.336788+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:25:51.336788+0000","last_peered":"2023-10-09T06:25:51.336788+0000","last_clean":"2023-10-09T06:25:51.336788+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:25:51.336788+0000","last_undegraded":"2023-10-09T06:25:51.336788+0000","last_fullsized":"2023-10-09T06:25:51.336788+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"291'12918","reported_seq":12979,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.927873+0000","last_change":"2023-10-09T06:22:15.194383+0000","last_active":"2023-10-09T06:26:02.927873+0000","last_peered":"2023-10-09T06:26:02.927873+0000","last_clean":"2023-10-09T06:26:02.927873+0000","last_became_active":"2023-10-09T06:22:15.194163+0000","last_became_peered":"2023-10-09T06:22:15.194163+0000","last_unstale":"2023-10-09T06:26:02.927873+0000","last_undegraded":"2023-10-09T06:26:02.927873+0000","last_fullsized":"2023-10-09T06:26:02.927873+0000","mapping_epoch":238,"log_start":"271'6900","ondisk_log_start":"271'6900","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6018,"log_dups_size":0,"ondisk_log_size":6018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:01.053961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6553600,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12918,"num_write_kb":416576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":224,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:51.324533+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:25:51.324533+0000","last_peered":"2023-10-09T06:25:51.324533+0000","last_clean":"2023-10-09T06:25:51.324533+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:25:51.324533+0000","last_undegraded":"2023-10-09T06:25:51.324533+0000","last_fullsized":"2023-10-09T06:25:51.324533+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":564,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207449+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:25:35.207449+0000","last_peered":"2023-10-09T06:25:35.207449+0000","last_clean":"2023-10-09T06:25:35.207449+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:25:35.207449+0000","last_undegraded":"2023-10-09T06:25:35.207449+0000","last_fullsized":"2023-10-09T06:25:35.207449+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.205354+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:25:35.205354+0000","last_peered":"2023-10-09T06:25:35.205354+0000","last_clean":"2023-10-09T06:25:35.205354+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:25:35.205354+0000","last_undegraded":"2023-10-09T06:25:35.205354+0000","last_fullsized":"2023-10-09T06:25:35.205354+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":253,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204060+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:25:35.204060+0000","last_peered":"2023-10-09T06:25:35.204060+0000","last_clean":"2023-10-09T06:25:35.204060+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:25:35.204060+0000","last_undegraded":"2023-10-09T06:25:35.204060+0000","last_fullsized":"2023-10-09T06:25:35.204060+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"291'8838","reported_seq":8974,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.301551+0000","last_change":"2023-10-09T06:23:29.155920+0000","last_active":"2023-10-09T06:26:00.301551+0000","last_peered":"2023-10-09T06:26:00.301551+0000","last_clean":"2023-10-09T06:26:00.301551+0000","last_became_active":"2023-10-09T06:23:26.812564+0000","last_became_peered":"2023-10-09T06:23:26.812564+0000","last_unstale":"2023-10-09T06:26:00.301551+0000","last_undegraded":"2023-10-09T06:26:00.301551+0000","last_fullsized":"2023-10-09T06:26:00.301551+0000","mapping_epoch":238,"log_start":"242'611","ondisk_log_start":"242'611","created":238,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"256'5187","last_scrub_stamp":"2023-10-09T06:23:29.155723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:23:29.155723+0000","objects_scrubbed":2646,"log_size":8227,"log_dups_size":0,"ondisk_log_size":8227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:15.787062+0000","scrub_duration":0.55572164899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3660,"num_write_kb":36608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":574,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.200359+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:25:35.200359+0000","last_peered":"2023-10-09T06:25:35.200359+0000","last_clean":"2023-10-09T06:25:35.200359+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:25:35.200359+0000","last_undegraded":"2023-10-09T06:25:35.200359+0000","last_fullsized":"2023-10-09T06:25:35.200359+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":134,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207518+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:25:35.207518+0000","last_peered":"2023-10-09T06:25:35.207518+0000","last_clean":"2023-10-09T06:25:35.207518+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:25:35.207518+0000","last_undegraded":"2023-10-09T06:25:35.207518+0000","last_fullsized":"2023-10-09T06:25:35.207518+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204292+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:25:35.204292+0000","last_peered":"2023-10-09T06:25:35.204292+0000","last_clean":"2023-10-09T06:25:35.204292+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:25:35.204292+0000","last_undegraded":"2023-10-09T06:25:35.204292+0000","last_fullsized":"2023-10-09T06:25:35.204292+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":267,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.353736+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:25:35.353736+0000","last_peered":"2023-10-09T06:25:35.353736+0000","last_clean":"2023-10-09T06:25:35.353736+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:25:35.353736+0000","last_undegraded":"2023-10-09T06:25:35.353736+0000","last_fullsized":"2023-10-09T06:25:35.353736+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"291'13092","reported_seq":13154,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.865507+0000","last_change":"2023-10-09T06:22:15.196668+0000","last_active":"2023-10-09T06:26:02.865507+0000","last_peered":"2023-10-09T06:26:02.865507+0000","last_clean":"2023-10-09T06:26:02.865507+0000","last_became_active":"2023-10-09T06:22:15.196378+0000","last_became_peered":"2023-10-09T06:22:15.196378+0000","last_unstale":"2023-10-09T06:26:02.865507+0000","last_undegraded":"2023-10-09T06:26:02.865507+0000","last_fullsized":"2023-10-09T06:26:02.865507+0000","mapping_epoch":238,"log_start":"271'7000","ondisk_log_start":"271'7000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:23.469656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929880,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13092,"num_write_kb":422785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":513,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.203573+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:25:35.203573+0000","last_peered":"2023-10-09T06:25:35.203573+0000","last_clean":"2023-10-09T06:25:35.203573+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:25:35.203573+0000","last_undegraded":"2023-10-09T06:25:35.203573+0000","last_fullsized":"2023-10-09T06:25:35.203573+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207832+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:25:35.207832+0000","last_peered":"2023-10-09T06:25:35.207832+0000","last_clean":"2023-10-09T06:25:35.207832+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:25:35.207832+0000","last_undegraded":"2023-10-09T06:25:35.207832+0000","last_fullsized":"2023-10-09T06:25:35.207832+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":273,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.201753+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:25:35.201753+0000","last_peered":"2023-10-09T06:25:35.201753+0000","last_clean":"2023-10-09T06:25:35.201753+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:25:35.201753+0000","last_undegraded":"2023-10-09T06:25:35.201753+0000","last_fullsized":"2023-10-09T06:25:35.201753+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204025+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:25:35.204025+0000","last_peered":"2023-10-09T06:25:35.204025+0000","last_clean":"2023-10-09T06:25:35.204025+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:25:35.204025+0000","last_undegraded":"2023-10-09T06:25:35.204025+0000","last_fullsized":"2023-10-09T06:25:35.204025+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"291'12748","reported_seq":12809,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.027793+0000","last_change":"2023-10-09T06:22:15.195392+0000","last_active":"2023-10-09T06:26:03.027793+0000","last_peered":"2023-10-09T06:26:03.027793+0000","last_clean":"2023-10-09T06:26:03.027793+0000","last_became_active":"2023-10-09T06:22:15.195084+0000","last_became_peered":"2023-10-09T06:22:15.195084+0000","last_unstale":"2023-10-09T06:26:03.027793+0000","last_undegraded":"2023-10-09T06:26:03.027793+0000","last_fullsized":"2023-10-09T06:26:03.027793+0000","mapping_epoch":238,"log_start":"273'7300","ondisk_log_start":"273'7300","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":5448,"log_dups_size":0,"ondisk_log_size":5448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:31.182451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7077888,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12748,"num_write_kb":411392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":574,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206649+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:25:35.206649+0000","last_peered":"2023-10-09T06:25:35.206649+0000","last_clean":"2023-10-09T06:25:35.206649+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:25:35.206649+0000","last_undegraded":"2023-10-09T06:25:35.206649+0000","last_fullsized":"2023-10-09T06:25:35.206649+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":281,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.207426+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:25:35.207426+0000","last_peered":"2023-10-09T06:25:35.207426+0000","last_clean":"2023-10-09T06:25:35.207426+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:25:35.207426+0000","last_undegraded":"2023-10-09T06:25:35.207426+0000","last_fullsized":"2023-10-09T06:25:35.207426+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":250,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:25:51.339662+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:25:51.339662+0000","last_peered":"2023-10-09T06:25:51.339662+0000","last_clean":"2023-10-09T06:25:51.339662+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:25:51.339662+0000","last_undegraded":"2023-10-09T06:25:51.339662+0000","last_fullsized":"2023-10-09T06:25:51.339662+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":270,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204548+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:25:35.204548+0000","last_peered":"2023-10-09T06:25:35.204548+0000","last_clean":"2023-10-09T06:25:35.204548+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:25:35.204548+0000","last_undegraded":"2023-10-09T06:25:35.204548+0000","last_fullsized":"2023-10-09T06:25:35.204548+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"291'13030","reported_seq":13091,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.875306+0000","last_change":"2023-10-09T06:22:15.412864+0000","last_active":"2023-10-09T06:26:02.875306+0000","last_peered":"2023-10-09T06:26:02.875306+0000","last_clean":"2023-10-09T06:26:02.875306+0000","last_became_active":"2023-10-09T06:22:15.412501+0000","last_became_peered":"2023-10-09T06:22:15.412501+0000","last_unstale":"2023-10-09T06:26:02.875306+0000","last_undegraded":"2023-10-09T06:26:02.875306+0000","last_fullsized":"2023-10-09T06:26:02.875306+0000","mapping_epoch":238,"log_start":"271'7000","ondisk_log_start":"271'7000","created":238,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":6030,"log_dups_size":0,"ondisk_log_size":6030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:20.090165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7077888,"num_objects":108,"num_object_clones":0,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13030,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":595,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206531+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:25:35.206531+0000","last_peered":"2023-10-09T06:25:35.206531+0000","last_clean":"2023-10-09T06:25:35.206531+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:25:35.206531+0000","last_undegraded":"2023-10-09T06:25:35.206531+0000","last_fullsized":"2023-10-09T06:25:35.206531+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.204020+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:25:35.204020+0000","last_peered":"2023-10-09T06:25:35.204020+0000","last_clean":"2023-10-09T06:25:35.204020+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:25:35.204020+0000","last_undegraded":"2023-10-09T06:25:35.204020+0000","last_fullsized":"2023-10-09T06:25:35.204020+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":248,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.353714+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:25:35.353714+0000","last_peered":"2023-10-09T06:25:35.353714+0000","last_clean":"2023-10-09T06:25:35.353714+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:25:35.353714+0000","last_undegraded":"2023-10-09T06:25:35.353714+0000","last_fullsized":"2023-10-09T06:25:35.353714+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206563+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:25:35.206563+0000","last_peered":"2023-10-09T06:25:35.206563+0000","last_clean":"2023-10-09T06:25:35.206563+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:25:35.206563+0000","last_undegraded":"2023-10-09T06:25:35.206563+0000","last_fullsized":"2023-10-09T06:25:35.206563+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"291'9899","reported_seq":21019,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.874983+0000","last_change":"2023-10-09T06:25:59.241505+0000","last_active":"2023-10-09T06:26:02.874983+0000","last_peered":"2023-10-09T06:26:02.874983+0000","last_clean":"2023-10-09T06:26:02.874983+0000","last_became_active":"2023-10-09T06:25:34.201087+0000","last_became_peered":"2023-10-09T06:25:34.201087+0000","last_unstale":"2023-10-09T06:26:02.874983+0000","last_undegraded":"2023-10-09T06:26:02.874983+0000","last_fullsized":"2023-10-09T06:26:02.874983+0000","mapping_epoch":288,"log_start":"246'2623","ondisk_log_start":"246'2623","created":254,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":1349,"log_size":7276,"log_dups_size":0,"ondisk_log_size":7276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3964928,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6395,"num_write_kb":206560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2786,"num_bytes_recovered":182583296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":250,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:45.365292+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:25:45.365292+0000","last_peered":"2023-10-09T06:25:45.365292+0000","last_clean":"2023-10-09T06:25:45.365292+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:25:45.365292+0000","last_undegraded":"2023-10-09T06:25:45.365292+0000","last_fullsized":"2023-10-09T06:25:45.365292+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":273,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.200657+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:25:35.200657+0000","last_peered":"2023-10-09T06:25:35.200657+0000","last_clean":"2023-10-09T06:25:35.200657+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:25:35.200657+0000","last_undegraded":"2023-10-09T06:25:35.200657+0000","last_fullsized":"2023-10-09T06:25:35.200657+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"291'8881","reported_seq":8954,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:01.517762+0000","last_change":"2023-10-09T06:23:26.833266+0000","last_active":"2023-10-09T06:26:01.517762+0000","last_peered":"2023-10-09T06:26:01.517762+0000","last_clean":"2023-10-09T06:26:01.517762+0000","last_became_active":"2023-10-09T06:23:26.831359+0000","last_became_peered":"2023-10-09T06:23:26.831359+0000","last_unstale":"2023-10-09T06:26:01.517762+0000","last_undegraded":"2023-10-09T06:26:01.517762+0000","last_fullsized":"2023-10-09T06:26:01.517762+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8881,"log_dups_size":0,"ondisk_log_size":8881,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11993088,"num_objects":183,"num_object_clones":0,"num_object_copies":549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":208704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":96,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206641+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:25:35.206641+0000","last_peered":"2023-10-09T06:25:35.206641+0000","last_clean":"2023-10-09T06:25:35.206641+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:25:35.206641+0000","last_undegraded":"2023-10-09T06:25:35.206641+0000","last_fullsized":"2023-10-09T06:25:35.206641+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":278,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206447+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:25:35.206447+0000","last_peered":"2023-10-09T06:25:35.206447+0000","last_clean":"2023-10-09T06:25:35.206447+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:25:35.206447+0000","last_undegraded":"2023-10-09T06:25:35.206447+0000","last_fullsized":"2023-10-09T06:25:35.206447+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"291'8990","reported_seq":11624,"reported_epoch":291,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:26:00.371821+0000","last_change":"2023-10-09T06:25:33.463651+0000","last_active":"2023-10-09T06:26:00.371821+0000","last_peered":"2023-10-09T06:26:00.371821+0000","last_clean":"2023-10-09T06:25:31.596130+0000","last_became_active":"2023-10-09T06:25:33.383675+0000","last_became_peered":"2023-10-09T06:25:33.383675+0000","last_unstale":"2023-10-09T06:26:00.371821+0000","last_undegraded":"2023-10-09T06:25:33.331302+0000","last_fullsized":"2023-10-09T06:25:33.192605+0000","mapping_epoch":287,"log_start":"242'586","ondisk_log_start":"242'586","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8404,"log_dups_size":0,"ondisk_log_size":8404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13729792,"num_objects":210,"num_object_clones":0,"num_object_copies":630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":16,"num_objects_unfound":0,"num_objects_dirty":210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6388,"num_write_kb":211104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":851,"num_bytes_recovered":51314688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[0,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,4","objects":194}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":96,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206901+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:25:35.206901+0000","last_peered":"2023-10-09T06:25:35.206901+0000","last_clean":"2023-10-09T06:25:35.206901+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:25:35.206901+0000","last_undegraded":"2023-10-09T06:25:35.206901+0000","last_fullsized":"2023-10-09T06:25:35.206901+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":268,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206369+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:25:35.206369+0000","last_peered":"2023-10-09T06:25:35.206369+0000","last_clean":"2023-10-09T06:25:35.206369+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:25:35.206369+0000","last_undegraded":"2023-10-09T06:25:35.206369+0000","last_fullsized":"2023-10-09T06:25:35.206369+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"291'8885","reported_seq":8971,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:02.858645+0000","last_change":"2023-10-09T06:25:58.265576+0000","last_active":"2023-10-09T06:26:02.858645+0000","last_peered":"2023-10-09T06:26:02.858645+0000","last_clean":"2023-10-09T06:26:02.858645+0000","last_became_active":"2023-10-09T06:23:26.830850+0000","last_became_peered":"2023-10-09T06:23:26.830850+0000","last_unstale":"2023-10-09T06:26:02.858645+0000","last_undegraded":"2023-10-09T06:26:02.858645+0000","last_fullsized":"2023-10-09T06:26:02.858645+0000","mapping_epoch":254,"log_start":"242'817","ondisk_log_start":"242'817","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8068,"log_dups_size":0,"ondisk_log_size":8068,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1441792,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6356,"num_write_kb":204096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":249,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.205760+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:25:35.205760+0000","last_peered":"2023-10-09T06:25:35.205760+0000","last_clean":"2023-10-09T06:25:35.205760+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:25:35.205760+0000","last_undegraded":"2023-10-09T06:25:35.205760+0000","last_fullsized":"2023-10-09T06:25:35.205760+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":263,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.205143+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:25:35.205143+0000","last_peered":"2023-10-09T06:25:35.205143+0000","last_clean":"2023-10-09T06:25:35.205143+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:25:35.205143+0000","last_undegraded":"2023-10-09T06:25:35.205143+0000","last_fullsized":"2023-10-09T06:25:35.205143+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"291'8872","reported_seq":8945,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-10-09T06:26:00.358988+0000","last_change":"2023-10-09T06:23:26.850127+0000","last_active":"2023-10-09T06:26:00.358988+0000","last_peered":"2023-10-09T06:26:00.358988+0000","last_clean":"2023-10-09T06:26:00.358988+0000","last_became_active":"2023-10-09T06:23:26.846263+0000","last_became_peered":"2023-10-09T06:23:26.846263+0000","last_unstale":"2023-10-09T06:26:00.358988+0000","last_undegraded":"2023-10-09T06:26:00.358988+0000","last_fullsized":"2023-10-09T06:26:00.358988+0000","mapping_epoch":254,"log_start":"242'521","ondisk_log_start":"242'521","created":254,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:22:14.163573+0000","last_clean_scrub_stamp":"2023-10-09T06:22:14.163573+0000","objects_scrubbed":0,"log_size":8351,"log_dups_size":0,"ondisk_log_size":8351,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10846208,"num_objects":166,"num_object_clones":0,"num_object_copies":498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6314,"num_write_kb":207328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":263,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.211173+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:25:35.211173+0000","last_peered":"2023-10-09T06:25:35.211173+0000","last_clean":"2023-10-09T06:25:35.211173+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:25:35.211173+0000","last_undegraded":"2023-10-09T06:25:35.211173+0000","last_fullsized":"2023-10-09T06:25:35.211173+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":251,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T06:25:35.206133+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:25:35.206133+0000","last_peered":"2023-10-09T06:25:35.206133+0000","last_clean":"2023-10-09T06:25:35.206133+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:25:35.206133+0000","last_undegraded":"2023-10-09T06:25:35.206133+0000","last_fullsized":"2023-10-09T06:25:35.206133+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":182845464,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":16,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186753,"num_write_kb":5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11975,"num_bytes_recovered":689897472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1313222656,"data_stored":1313210440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200428,"ondisk_log_size":200428,"up":78,"acting":77,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234344,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032020,"kb_used_data":135884,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":92713132,"statfs":{"total":95995035648,"available":94938247168,"internally_reserved":0,"allocated":139145216,"data_stored":138366921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2149999999999999,"5min":8.8960000000000008,"15min":3.8580000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":89.027000000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":4.2919999999999998},{"interface":"front","average":{"1min":7.0759999999999996,"5min":9.2319999999999993,"15min":3.8980000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.218},"max":{"1min":89.135999999999996,"5min":219.399,"15min":219.399},"last":4.5640000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:25:20 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":3.1869999999999998,"15min":2.6659999999999999},"min":{"1min":0.253,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.242,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.52},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.4289999999999998,"15min":2.4630000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.7629999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":5.8419999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:25:32 2023","interfaces":[{"interface":"back","average":{"1min":12.579000000000001,"5min":12.257999999999999,"15min":5.2720000000000002},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":92.293000000000006,"5min":215.34800000000001,"15min":215.34800000000001},"last":1.347},{"interface":"front","average":{"1min":12.222,"5min":12.436,"15min":5.4109999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.727999999999994,"5min":215.869,"15min":215.869},"last":5.8760000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.9889999999999999,"15min":1.542},"min":{"1min":0.40999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.2149999999999999,"5min":50.966999999999999,"15min":50.966999999999999},"last":2.3370000000000002},{"interface":"front","average":{"1min":2.012,"5min":3.1389999999999998,"15min":1.599},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.4550000000000001,"5min":51.133000000000003,"15min":51.133000000000003},"last":2.3690000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":5.6710000000000003,"5min":2.161,"15min":1.673},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":94.105999999999995,"5min":94.105999999999995,"15min":94.105999999999995},"last":4.3620000000000001},{"interface":"front","average":{"1min":5.5339999999999998,"5min":2.218,"15min":1.4379999999999999},"min":{"1min":0.31900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":93.713999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":1.8640000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.429,"15min":1.714},"min":{"1min":0.25600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.9220000000000002,"5min":8.9649999999999999,"15min":50.304000000000002},"last":1.6619999999999999},{"interface":"front","average":{"1min":1.341,"5min":1.3140000000000001,"15min":1.429},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":5.7130000000000001,"5min":7.452,"15min":48.774999999999999},"last":4.3280000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.494,"15min":1.359},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7889999999999997,"5min":8.4890000000000008,"15min":11.878},"last":6.7169999999999996},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.464,"15min":1.256},"min":{"1min":0.38600000000000001,"5min":0.36799999999999999,"15min":0.33000000000000002},"max":{"1min":5.2690000000000001,"5min":12.141999999999999,"15min":12.141999999999999},"last":0.33200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495982,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975756,"kb_used_data":76740,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92769396,"statfs":{"total":95995035648,"available":94995861504,"internally_reserved":0,"allocated":78581760,"data_stored":77812045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":3.7690000000000001,"15min":3.0790000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":4.2560000000000002,"5min":110.926,"15min":110.926},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.2869999999999999,"5min":3.7480000000000002,"15min":3.2669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2859999999999996,"5min":110.911,"15min":110.911},"last":0.50800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:25:19 2023","interfaces":[{"interface":"back","average":{"1min":6.5010000000000003,"5min":2.8690000000000002,"15min":3.5310000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":66.796999999999997,"5min":66.796999999999997,"15min":114.202},"last":0.40699999999999997},{"interface":"front","average":{"1min":6.5460000000000003,"5min":2.5630000000000002,"15min":3.3530000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":66.704999999999998,"5min":66.704999999999998,"15min":114.179},"last":0.43099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":4.0890000000000004,"15min":2.774},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":52.609999999999999,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.95999999999999996},{"interface":"front","average":{"1min":5.1059999999999999,"5min":3.9119999999999999,"15min":2.6669999999999998},"min":{"1min":0.29899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.575000000000003,"5min":75.120000000000005,"15min":75.120000000000005},"last":1.002}]},{"osd":3,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.25,"15min":2.02},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":6.9290000000000003,"5min":110.358,"15min":110.358},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.804,"5min":3.27,"15min":2.069},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.9359999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.77100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.1299999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.1890000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.03,"5min":1.474,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":4.2119999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.456,"15min":1.3640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.2460000000000004,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.488,"15min":1.407},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.8920000000000003,"5min":24.401,"15min":24.401},"last":0.71099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.5139999999999998,"15min":1.5129999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":8.5879999999999992,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":2.4969999999999999,"15min":1.712},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":3.5390000000000001,"5min":100.995,"15min":100.995},"last":0.752}]}]},{"osd":1,"up_from":110,"seq":472446402718,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633324,"kb_used_data":164260,"kb_used_omap":4,"kb_used_meta":469051,"kb_avail":93111828,"statfs":{"total":95995035648,"available":95346511872,"internally_reserved":0,"allocated":168202240,"data_stored":166931379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":480309134},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5919999999999996,"5min":6.9119999999999999,"15min":4.032},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":102.471,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":7.7779999999999996,"5min":7.9870000000000001,"15min":4.5599999999999996},"min":{"1min":0.20399999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":102.054,"5min":145.982,"15min":149.02699999999999},"last":2.6230000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":5.6630000000000003,"15min":3.1059999999999999},"min":{"1min":0.26000000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":8.4030000000000005,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.3530000000000002,"5min":6.7800000000000002,"15min":3.863},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.219},"max":{"1min":9.8859999999999992,"5min":140.29900000000001,"15min":140.29900000000001},"last":8.157}]},{"osd":3,"last update":"Mon Oct 9 06:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":3.9089999999999998,"15min":2.327},"min":{"1min":0.30599999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.585,"5min":77.331999999999994,"15min":153.18199999999999},"last":1.863},{"interface":"front","average":{"1min":0.56299999999999994,"5min":2.5379999999999998,"15min":1.8340000000000001},"min":{"1min":0.31,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.754,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":5.266,"15min":3.7469999999999999},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.1760000000000002,"5min":99.391000000000005,"15min":199.43799999999999},"last":1.8089999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":5.1180000000000003,"15min":3.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.3170000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":1.667}]},{"osd":5,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.5859999999999999,"15min":2.5499999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7169999999999996,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.503},{"interface":"front","average":{"1min":1.9890000000000001,"5min":3.5169999999999999,"15min":2.911},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":6.7789999999999999,"5min":77.388000000000005,"15min":77.388000000000005},"last":1.6930000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":4.1269999999999998,"15min":2.948},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":6.6799999999999997,"5min":77.292000000000002,"15min":77.292000000000002},"last":1.631},{"interface":"front","average":{"1min":2.3559999999999999,"5min":3.8130000000000002,"15min":2.7429999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9290000000000003,"5min":77.378,"15min":77.378},"last":1.7729999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5289999999999999,"15min":2.706},"min":{"1min":0.36099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.2069999999999999,"5min":77.456999999999994,"15min":77.456999999999994},"last":1.839},{"interface":"front","average":{"1min":1.105,"5min":2.9929999999999999,"15min":2.2040000000000002},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6900000000000004,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.63600000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024151,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1040096,"kb_used_data":348056,"kb_used_omap":5,"kb_used_meta":692026,"kb_avail":92705056,"statfs":{"total":95995035648,"available":94929977344,"internally_reserved":0,"allocated":356409344,"data_stored":354918215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":708635258},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.3769999999999998,"15min":2.8170000000000002},"min":{"1min":0.193,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.5369999999999999,"5min":185.17400000000001,"15min":185.17400000000001},"last":2.1600000000000001},{"interface":"front","average":{"1min":2.6280000000000001,"5min":4.5570000000000004,"15min":3.351},"min":{"1min":0.254,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":6.6319999999999997,"5min":186.691,"15min":186.691},"last":0.34799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:25:36 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":2.4620000000000002,"15min":2.3010000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.17799999999999999},"max":{"1min":10.255000000000001,"5min":13.211,"15min":17.588999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.6970000000000001,"15min":2.4049999999999998},"min":{"1min":0.33300000000000002,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":14.974,"5min":14.974,"15min":21.198},"last":7.25}]},{"osd":3,"last update":"Mon Oct 9 06:25:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2010000000000001,"5min":3.3999999999999999,"15min":2.9140000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2989999999999995,"5min":52.683,"15min":185.89400000000001},"last":2.6389999999999998},{"interface":"front","average":{"1min":2.9950000000000001,"5min":3.4289999999999998,"15min":3.0259999999999998},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":9.3759999999999994,"5min":52.941000000000003,"15min":186.72900000000001},"last":2.3860000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5860000000000001,"15min":1.516},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.024,"5min":7.9660000000000002,"15min":13.244},"last":5.7229999999999999},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.833,"15min":1.9410000000000001},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":4.9770000000000003,"5min":8.4890000000000008,"15min":12.752000000000001},"last":1.468}]},{"osd":5,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.0990000000000002,"15min":1.9239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8450000000000002,"5min":62.984999999999999,"15min":62.984999999999999},"last":5.7939999999999996},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.7090000000000001,"15min":2.504},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":5.8650000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":2.806}]},{"osd":6,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.282,"15min":2.319},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.1360000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":7.2080000000000002},{"interface":"front","average":{"1min":1.498,"5min":1.6299999999999999,"15min":1.8400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.819,"5min":7.9980000000000002,"15min":13.315},"last":5.7649999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.5099999999999998,"15min":2.1920000000000002},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":4.7279999999999998,"5min":62.262,"15min":62.262},"last":1.27},{"interface":"front","average":{"1min":2.202,"5min":2.9910000000000001,"15min":2.5619999999999998},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":7.1059999999999999,"5min":62.229999999999997,"15min":62.229999999999997},"last":2.8359999999999999}]}]},{"osd":2,"up_from":128,"seq":549755814020,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514064,"kb_used_data":105608,"kb_used_omap":7,"kb_used_meta":408440,"kb_avail":93231088,"statfs":{"total":95995035648,"available":95468634112,"internally_reserved":0,"allocated":108142592,"data_stored":106895460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":418242641},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":10.217000000000001,"5min":4.6269999999999998,"15min":3.968},"min":{"1min":0.22800000000000001,"5min":0.157,"15min":0.157},"max":{"1min":150.32599999999999,"5min":150.32599999999999,"15min":163.40100000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":3.1629999999999998,"5min":3.4260000000000002,"15min":3.8820000000000001},"min":{"1min":0.20899999999999999,"5min":0.182,"15min":0.182},"max":{"1min":21.472000000000001,"5min":110.66500000000001,"15min":163.363},"last":1.0149999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.8579999999999997,"15min":3.6680000000000001},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":7.9710000000000001,"5min":205.148,"15min":222.50299999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":4.8280000000000003,"15min":3.984},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":11.811,"5min":205.267,"15min":222.19399999999999},"last":7.9029999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":2.8780000000000001,"15min":1.8140000000000001},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":13.664999999999999,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":3.0150000000000001,"5min":2.9369999999999998,"15min":1.8700000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.728,"5min":69.906999999999996,"15min":81.061999999999998},"last":1.9379999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.952,"15min":2.2240000000000002},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":52.188000000000002,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.53100000000000003},{"interface":"front","average":{"1min":3.6179999999999999,"5min":3.282,"15min":2.4910000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":52.167999999999999,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.59199999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.472,"15min":1.534},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.309},"max":{"1min":5.508,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":2.2010000000000001},{"interface":"front","average":{"1min":1.256,"5min":1.7549999999999999,"15min":2.0990000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.9349999999999996,"5min":10.548,"15min":10.548},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":3.7839999999999998,"15min":2.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":55.392000000000003,"5min":138.291,"15min":138.291},"last":0.77000000000000002},{"interface":"front","average":{"1min":4.8150000000000004,"5min":3.4910000000000001,"15min":2.206},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":55.146999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":1.123}]},{"osd":7,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9269999999999996,"5min":2.2709999999999999,"15min":1.863},"min":{"1min":0.40699999999999997,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":55.851999999999997,"5min":55.851999999999997,"15min":55.851999999999997},"last":1.151},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.1890000000000001,"15min":2.395},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":55.731000000000002,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.495}]}]},{"osd":3,"up_from":149,"seq":639950127217,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":583864,"kb_used_data":197424,"kb_used_omap":2,"kb_used_meta":386429,"kb_avail":93161288,"statfs":{"total":95995035648,"available":95397158912,"internally_reserved":0,"allocated":202162176,"data_stored":200996507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":395703349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":7.8659999999999997,"15min":3.262},"min":{"1min":0.19800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":10.367000000000001,"5min":187.852,"15min":187.852},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.23,"5min":8.4719999999999995,"15min":3.7200000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":4.6539999999999999,"5min":187.923,"15min":187.923},"last":0.222}]},{"osd":1,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":6.0439999999999996,"15min":2.9500000000000002},"min":{"1min":0.255,"5min":0.246,"15min":0.223},"max":{"1min":11.367000000000001,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.255},{"interface":"front","average":{"1min":1.978,"5min":3.673,"15min":1.9379999999999999},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.8810000000000002,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.48099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":5.2350000000000003,"15min":2.649},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.57,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":2.3210000000000002,"5min":4.8769999999999998,"15min":2.1349999999999998},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5250000000000004,"5min":187.79499999999999,"15min":187.79499999999999},"last":0.52100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.544,"15min":1.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":6.7830000000000004,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.2389999999999999,"15min":1.964},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.9649999999999999,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":3.2530000000000001,"15min":2.1080000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":3.0059999999999998,"5min":187.833,"15min":187.833},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.0649999999999999,"15min":2.036},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":187.952,"15min":187.952},"last":0.371}]},{"osd":6,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.6080000000000001,"15min":2.1560000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6280000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1890000000000001,"15min":1.9910000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8719999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.34699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":3.5169999999999999,"15min":1.8320000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":6.8239999999999998,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.242,"5min":3.4580000000000002,"15min":2.093},"min":{"1min":0.41899999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":5.431,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.61899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019254,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263564,"kb_used_data":364356,"kb_used_omap":0,"kb_used_meta":899072,"kb_avail":92481588,"statfs":{"total":95995035648,"available":94701146112,"internally_reserved":0,"allocated":373100544,"data_stored":372330108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920649728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":3.5310000000000001,"15min":2.2469999999999999},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.7480000000000002,"5min":76.944999999999993,"15min":76.944999999999993},"last":1.409},{"interface":"front","average":{"1min":1.1279999999999999,"5min":3.9089999999999998,"15min":2.6549999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":4.3799999999999999,"5min":76.988,"15min":76.988},"last":3.597}]},{"osd":1,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.9890000000000001,"15min":2.0699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.8069999999999999,"5min":37.383000000000003,"15min":91.849999999999994},"last":0.441},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.1869999999999998,"15min":2.177},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.5869999999999997,"5min":37.436999999999998,"15min":91.885000000000005},"last":0.52700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.9240000000000004,"5min":2.8490000000000002,"15min":2.093},"min":{"1min":0.41599999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":44.807000000000002,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":5.2370000000000001,"5min":3.0139999999999998,"15min":2.278},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":44.697000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":3.488}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.7029999999999998,"15min":2.2879999999999998},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.8360000000000003,"5min":45.591000000000001,"15min":91.697000000000003},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.341,"5min":2.3140000000000001,"15min":2.129},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":3.5230000000000001,"5min":45.548999999999999,"15min":94.356999999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1619999999999999,"15min":1.107},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":4.2270000000000003,"5min":10.673,"15min":10.673},"last":6.2670000000000003},{"interface":"front","average":{"1min":1.21,"5min":1.014,"15min":1.23},"min":{"1min":0.29599999999999999,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":4.6189999999999998,"5min":9.6379999999999999,"15min":12.552},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.83399999999999996,"15min":0.84899999999999998},"min":{"1min":0.29399999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":2.1179999999999999,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1830000000000001,"15min":1.286},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.859,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.69899999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.956,"15min":1.49},"min":{"1min":0.34499999999999997,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":5.4480000000000004,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.857,"15min":1.4770000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0640000000000001,"5min":62.573,"15min":62.573},"last":0.55800000000000005}]}]},{"osd":5,"up_from":0,"seq":163208757618,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":330240000,"data_stored":330235928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":159186944,"data_stored":159186944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":103546880,"data_stored":103546880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":193724416,"data_stored":193724416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":340856832,"data_stored":340852760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":57610240,"data_stored":57606168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":128057344,"data_stored":128057344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:26:04.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:26:04.702 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:04.729 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:04.771 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:04.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:04.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:04.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: pgmap v900: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 175 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 16/8451 objects degraded (0.189%); 16/8451 objects misplaced (0.189%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: 10.17 scrub starts 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T06:26:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: pgmap v900: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 175 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 16/8451 objects degraded (0.189%); 16/8451 objects misplaced (0.189%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: 10.17 scrub starts 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: failed to encode map e292 with expected crc 2023-10-09T06:26:04.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:04 smithi107 ceph-mon[220921]: failed to encode map e292 with expected crc 2023-10-09T06:26:04.903 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: pgmap v900: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 175 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 16/8451 objects degraded (0.189%); 16/8451 objects misplaced (0.189%); 1.4 MiB/s, 28 objects/s recovering 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: 10.17 scrub starts 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: failed to encode map e292 with expected crc 2023-10-09T06:26:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:04 smithi150 ceph-mon[194178]: failed to encode map e292 with expected crc 2023-10-09T06:26:05.235 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:05.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:05.344 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:05.376 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:05.452 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:05.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:05.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:05.547 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 104089 objects 2023-10-09T06:26:05.548 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :129.787 2023-10-09T06:26:05.557 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:05.583 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:05.626 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:05.631 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:05 smithi150 ceph-mon[194178]: failed to encode map e292 with expected crc 2023-10-09T06:26:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:05.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:05.659 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:05.676 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:05.733 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:05.761 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:05 smithi107 ceph-mon[212750]: failed to encode map e292 with expected crc 2023-10-09T06:26:05.761 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:26:05.761 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:26:05.761 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:26:05.761 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:26:05.761 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:05 smithi107 ceph-mon[220921]: failed to encode map e292 with expected crc 2023-10-09T06:26:05.764 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:06.226 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:06.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:06.330 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:06.358 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:06.620 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:06 smithi107 ceph-mon[212750]: pgmap v902: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 159 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/7677 objects degraded (0.208%); 16/7677 objects misplaced (0.208%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:06.621 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:06 smithi107 ceph-mon[220921]: pgmap v902: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 159 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/7677 objects degraded (0.208%); 16/7677 objects misplaced (0.208%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:06 smithi150 ceph-mon[194178]: pgmap v902: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 159 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 810 op/s; 16/7677 objects degraded (0.208%); 16/7677 objects misplaced (0.208%); 1.7 MiB/s, 33 objects/s recovering 2023-10-09T06:26:07.071 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:07.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:07.175 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:07.207 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:07.256 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:07.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:07.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:07.349 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:26:07.361 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:07.380 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:26:07.423 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:07.535 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:07.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:07.628 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/7677 objects degraded (0.208%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:07.629 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/7677 objects degraded (0.208%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:07.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:07.670 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:07.716 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:07.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:07.821 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:07.848 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:07.891 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:07.894 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/7677 objects degraded (0.208%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:07.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:07.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:08.024 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:08.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:08 smithi107 ceph-mon[212750]: pgmap v903: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 83 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 16/4017 objects degraded (0.398%); 16/4017 objects misplaced (0.398%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:08.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:08 smithi107 ceph-mon[220921]: pgmap v903: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 83 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 16/4017 objects degraded (0.398%); 16/4017 objects misplaced (0.398%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:08 smithi150 ceph-mon[194178]: pgmap v903: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 83 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 16/4017 objects degraded (0.398%); 16/4017 objects misplaced (0.398%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:10.085 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:10.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:10.187 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:10.215 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:10.520 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:10.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:10.626 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:10.658 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:10.706 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:10.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:10.813 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:10.816 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:10.850 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:10 smithi107 ceph-mon[212750]: pgmap v904: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 25 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16/1257 objects degraded (1.273%); 16/1257 objects misplaced (1.273%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:10.850 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:10 smithi107 ceph-mon[220921]: pgmap v904: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 25 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16/1257 objects degraded (1.273%); 16/1257 objects misplaced (1.273%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:10.854 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:10.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:10 smithi150 ceph-mon[194178]: pgmap v904: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 25 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 16/1257 objects degraded (1.273%); 16/1257 objects misplaced (1.273%); 633 KiB/s, 32 objects/s recovering 2023-10-09T06:26:11.128 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:11.395 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T06:26:11.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T06:26:11.551 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:26:11.589 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:26:11.612 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:26:11.638 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:26:11.664 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:26:11.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:26:11.714 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:26:11.736 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:26:11.737 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.738 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:26:11.758 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1747915756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:11.759 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1747915756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:11.806 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:11.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:11.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1747915756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:11.912 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:11.917 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:11.917 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1184,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":292,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":7,"osd_in_since":1696832504,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":108},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":392,"data_bytes":24549280,"bytes_used":5454356480,"bytes_avail":666510893056,"bytes_total":671965249536,"degraded_objects":16,"degraded_total":1176,"degraded_ratio":0.013605442176870748,"misplaced_objects":16,"misplaced_total":1176,"misplaced_ratio":0.013605442176870748,"recovering_objects_per_sec":89,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":897,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":563},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T06:22:27.038508+0000","services":{}},"progress_events":{"be02b243-5ad1-4e62-ae2b-ff1425dec846":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 1s)","progress":0.99082571268081665,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:26:11.943 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:12] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[212750]: pgmap v905: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/183491858' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[220921]: pgmap v905: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/183491858' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:12 smithi150 ceph-mon[194178]: pgmap v905: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/183491858' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:26:14.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:14 smithi107 ceph-mon[212750]: pgmap v906: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:14.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:14 smithi107 ceph-mon[220921]: pgmap v906: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:14 smithi150 ceph-mon[194178]: pgmap v906: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 89 objects/s recovering 2023-10-09T06:26:16.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[212750]: pgmap v907: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 77 objects/s recovering 2023-10-09T06:26:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[220921]: pgmap v907: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 77 objects/s recovering 2023-10-09T06:26:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:16 smithi150 ceph-mon[194178]: pgmap v907: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 16/1176 objects degraded (1.361%); 16/1176 objects misplaced (1.361%); 0 B/s, 77 objects/s recovering 2023-10-09T06:26:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:17.065 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:26:17.066 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:26:17.130 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:17.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:17.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:17.208 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:17.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:17.269 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:17.319 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:17.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:17.426 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:17.457 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:17.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[212750]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[212750]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[220921]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[220921]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[220921]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[220921]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:17 smithi107 ceph-mon[220921]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.734 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:17.734 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:17.743 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:17.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:17.847 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:17.879 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:17.926 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:17.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:17 smithi150 ceph-mon[194178]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:17 smithi150 ceph-mon[194178]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T06:26:17.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:17 smithi150 ceph-mon[194178]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:17 smithi150 ceph-mon[194178]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:17 smithi150 ceph-mon[194178]: failed to encode map e293 with expected crc 2023-10-09T06:26:17.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:17.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:18.033 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:18.060 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:18.102 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:18.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:18.207 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:18.236 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:18.402 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:26:18.669 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:26:18.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: pgmap v909: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:26:18.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:26:18.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded) 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3419989506' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:18.690 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: pgmap v909: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded) 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: failed to encode map e294 with expected crc 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3419989506' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:18.691 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:18 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:18.931 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:18.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:19.034 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: pgmap v909: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 0 B/s, 0 objects/s recovering 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1176 objects degraded (1.361%), 1 pg degraded) 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T06:26:19.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: failed to encode map e294 with expected crc 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3419989506' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:19.064 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:18 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:19.066 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:19.116 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:19.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:19.221 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:19.236 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:26:19.248 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:19.295 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:19.422 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:19.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:19.530 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:19.559 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:19.580 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:19.669 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:19.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[212750]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.732 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:19 smithi107 ceph-mon[220921]: failed to encode map e295 with expected crc 2023-10-09T06:26:19.773 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:19.805 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:19.853 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:19.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:19.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:19.958 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:19.989 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: failed to encode map e295 with expected crc 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: failed to encode map e295 with expected crc 2023-10-09T06:26:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:19 smithi150 ceph-mon[194178]: failed to encode map e295 with expected crc 2023-10-09T06:26:20.236 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:20.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:20.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:20.368 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:20.684 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_5' removed 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[212750]: pgmap v912: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3896871446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[220921]: pgmap v912: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3896871446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:20.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:20 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:21.000 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:21.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:21.122 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:21.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:20 smithi150 ceph-mon[194178]: pgmap v912: 109 pgs: 1 creating+peering, 108 active+clean; 23 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3896871446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:20 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:26:21.156 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:21.301 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:21.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:21.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:21.449 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:21.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:26:21.491 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:21.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:21.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:21.598 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:21.624 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[212750]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:26:21.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T06:26:21.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:21 smithi107 ceph-mon[220921]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: failed to encode map e296 with expected crc 2023-10-09T06:26:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:21 smithi150 ceph-mon[194178]: failed to encode map e296 with expected crc 2023-10-09T06:26:22.125 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:22.140 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:22.141 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:22] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.43.0" 2023-10-09T06:26:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:22.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:22.157 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:26:22.158 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:26:22.158 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-10-09T06:26:22.158 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-10-09T06:26:22.247 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:22.276 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:22.322 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:22.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:22.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:22.402 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:22.405 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:26:22.428 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:22.454 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:22.457 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:26:22.479 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:26:22.482 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:22.496 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:22.501 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:26:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:22.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:22.525 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:26:22.547 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:26:22.567 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:26:22.586 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:26:22.604 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.607 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.608 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:26:22.634 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:22.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[212750]: pgmap v914: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:22.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:26:22.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2040852450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:22.841 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[220921]: pgmap v914: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:22.841 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:26:22.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2040852450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:22 smithi150 ceph-mon[194178]: pgmap v914: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:22 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:26:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2040852450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:23.277 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":914,"stamp":"2023-10-09T06:26:21.138880+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":338010,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5217036,"kb_used_data":544660,"kb_used_omap":20,"kb_used_meta":4672299,"kb_avail":650999028,"statfs":{"total":671965249536,"available":666623004672,"internally_reserved":0,"allocated":557731840,"data_stored":550220293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4784434830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-23953408,"num_objects":-368,"num_object_clones":0,"num_object_copies":-1104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-16,"num_objects_misplaced":-16,"num_objects_unfound":0,"num_objects_dirty":-368,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-189415,"num_write_kb":-5460609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-14013,"num_bytes_recovered":-689897472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.492039"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":273,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650227+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:26:16.650227+0000","last_peered":"2023-10-09T06:26:16.650227+0000","last_clean":"2023-10-09T06:26:16.650227+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:26:16.650227+0000","last_undegraded":"2023-10-09T06:26:16.650227+0000","last_fullsized":"2023-10-09T06:26:16.650227+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654935+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:26:16.654935+0000","last_peered":"2023-10-09T06:26:16.654935+0000","last_clean":"2023-10-09T06:26:16.654935+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:26:16.654935+0000","last_undegraded":"2023-10-09T06:26:16.654935+0000","last_fullsized":"2023-10-09T06:26:16.654935+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":265,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.642683+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:26:03.642683+0000","last_peered":"2023-10-09T06:26:03.642683+0000","last_clean":"2023-10-09T06:26:03.642683+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:26:03.642683+0000","last_undegraded":"2023-10-09T06:26:03.642683+0000","last_fullsized":"2023-10-09T06:26:03.642683+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653455+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:26:17.653455+0000","last_peered":"2023-10-09T06:26:17.653455+0000","last_clean":"2023-10-09T06:26:17.653455+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:26:17.653455+0000","last_undegraded":"2023-10-09T06:26:17.653455+0000","last_fullsized":"2023-10-09T06:26:17.653455+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":249,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654448+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:26:16.654448+0000","last_peered":"2023-10-09T06:26:16.654448+0000","last_clean":"2023-10-09T06:26:16.654448+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:26:16.654448+0000","last_undegraded":"2023-10-09T06:26:16.654448+0000","last_fullsized":"2023-10-09T06:26:16.654448+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.638016+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:26:03.638016+0000","last_peered":"2023-10-09T06:26:03.638016+0000","last_clean":"2023-10-09T06:26:03.638016+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:26:03.638016+0000","last_undegraded":"2023-10-09T06:26:03.638016+0000","last_fullsized":"2023-10-09T06:26:03.638016+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":256,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653625+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:26:17.653625+0000","last_peered":"2023-10-09T06:26:17.653625+0000","last_clean":"2023-10-09T06:26:17.653625+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:26:17.653625+0000","last_undegraded":"2023-10-09T06:26:17.653625+0000","last_fullsized":"2023-10-09T06:26:17.653625+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":232,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654359+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:26:16.654359+0000","last_peered":"2023-10-09T06:26:16.654359+0000","last_clean":"2023-10-09T06:26:16.654359+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:26:16.654359+0000","last_undegraded":"2023-10-09T06:26:16.654359+0000","last_fullsized":"2023-10-09T06:26:16.654359+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661519+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:26:16.661519+0000","last_peered":"2023-10-09T06:26:16.661519+0000","last_clean":"2023-10-09T06:26:16.661519+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:26:16.661519+0000","last_undegraded":"2023-10-09T06:26:16.661519+0000","last_fullsized":"2023-10-09T06:26:16.661519+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":230,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654349+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:26:16.654349+0000","last_peered":"2023-10-09T06:26:16.654349+0000","last_clean":"2023-10-09T06:26:16.654349+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:26:16.654349+0000","last_undegraded":"2023-10-09T06:26:16.654349+0000","last_fullsized":"2023-10-09T06:26:16.654349+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":143,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661834+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:26:16.661834+0000","last_peered":"2023-10-09T06:26:16.661834+0000","last_clean":"2023-10-09T06:26:16.661834+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:26:16.661834+0000","last_undegraded":"2023-10-09T06:26:16.661834+0000","last_fullsized":"2023-10-09T06:26:16.661834+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":224,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.643146+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:26:03.643146+0000","last_peered":"2023-10-09T06:26:03.643146+0000","last_clean":"2023-10-09T06:26:03.643146+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:26:03.643146+0000","last_undegraded":"2023-10-09T06:26:03.643146+0000","last_fullsized":"2023-10-09T06:26:03.643146+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":300,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650635+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:26:16.650635+0000","last_peered":"2023-10-09T06:26:16.650635+0000","last_clean":"2023-10-09T06:26:16.650635+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:26:16.650635+0000","last_undegraded":"2023-10-09T06:26:16.650635+0000","last_fullsized":"2023-10-09T06:26:16.650635+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":264,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650623+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:26:16.650623+0000","last_peered":"2023-10-09T06:26:16.650623+0000","last_clean":"2023-10-09T06:26:16.650623+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:26:16.650623+0000","last_undegraded":"2023-10-09T06:26:16.650623+0000","last_fullsized":"2023-10-09T06:26:16.650623+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":143,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661195+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:26:16.661195+0000","last_peered":"2023-10-09T06:26:16.661195+0000","last_clean":"2023-10-09T06:26:16.661195+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:26:16.661195+0000","last_undegraded":"2023-10-09T06:26:16.661195+0000","last_fullsized":"2023-10-09T06:26:16.661195+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":250,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654952+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:26:16.654952+0000","last_peered":"2023-10-09T06:26:16.654952+0000","last_clean":"2023-10-09T06:26:16.654952+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:26:16.654952+0000","last_undegraded":"2023-10-09T06:26:16.654952+0000","last_fullsized":"2023-10-09T06:26:16.654952+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.652092+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:26:17.652092+0000","last_peered":"2023-10-09T06:26:17.652092+0000","last_clean":"2023-10-09T06:26:17.652092+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:26:17.652092+0000","last_undegraded":"2023-10-09T06:26:17.652092+0000","last_fullsized":"2023-10-09T06:26:17.652092+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":221,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661680+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:26:16.661680+0000","last_peered":"2023-10-09T06:26:16.661680+0000","last_clean":"2023-10-09T06:26:16.661680+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:26:16.661680+0000","last_undegraded":"2023-10-09T06:26:16.661680+0000","last_fullsized":"2023-10-09T06:26:16.661680+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.662892+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:26:16.662892+0000","last_peered":"2023-10-09T06:26:16.662892+0000","last_clean":"2023-10-09T06:26:16.662892+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:26:16.662892+0000","last_undegraded":"2023-10-09T06:26:16.662892+0000","last_fullsized":"2023-10-09T06:26:16.662892+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":265,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653597+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:26:17.653597+0000","last_peered":"2023-10-09T06:26:17.653597+0000","last_clean":"2023-10-09T06:26:17.653597+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:26:17.653597+0000","last_undegraded":"2023-10-09T06:26:17.653597+0000","last_fullsized":"2023-10-09T06:26:17.653597+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":281,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.637581+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:26:03.637581+0000","last_peered":"2023-10-09T06:26:03.637581+0000","last_clean":"2023-10-09T06:26:03.637581+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:26:03.637581+0000","last_undegraded":"2023-10-09T06:26:03.637581+0000","last_fullsized":"2023-10-09T06:26:03.637581+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":249,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654803+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:26:16.654803+0000","last_peered":"2023-10-09T06:26:16.654803+0000","last_clean":"2023-10-09T06:26:16.654803+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:26:16.654803+0000","last_undegraded":"2023-10-09T06:26:16.654803+0000","last_fullsized":"2023-10-09T06:26:16.654803+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":155,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661977+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:26:16.661977+0000","last_peered":"2023-10-09T06:26:16.661977+0000","last_clean":"2023-10-09T06:26:16.661977+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:26:16.661977+0000","last_undegraded":"2023-10-09T06:26:16.661977+0000","last_fullsized":"2023-10-09T06:26:16.661977+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":204,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.660886+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:26:16.660886+0000","last_peered":"2023-10-09T06:26:16.660886+0000","last_clean":"2023-10-09T06:26:16.660886+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:26:16.660886+0000","last_undegraded":"2023-10-09T06:26:16.660886+0000","last_fullsized":"2023-10-09T06:26:16.660886+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":141,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661465+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:26:16.661465+0000","last_peered":"2023-10-09T06:26:16.661465+0000","last_clean":"2023-10-09T06:26:16.661465+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:26:16.661465+0000","last_undegraded":"2023-10-09T06:26:16.661465+0000","last_fullsized":"2023-10-09T06:26:16.661465+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":271,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650476+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:26:16.650476+0000","last_peered":"2023-10-09T06:26:16.650476+0000","last_clean":"2023-10-09T06:26:16.650476+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:26:16.650476+0000","last_undegraded":"2023-10-09T06:26:16.650476+0000","last_fullsized":"2023-10-09T06:26:16.650476+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":155,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661632+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:26:16.661632+0000","last_peered":"2023-10-09T06:26:16.661632+0000","last_clean":"2023-10-09T06:26:16.661632+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:26:16.661632+0000","last_undegraded":"2023-10-09T06:26:16.661632+0000","last_fullsized":"2023-10-09T06:26:16.661632+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":272,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.637755+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:26:03.637755+0000","last_peered":"2023-10-09T06:26:03.637755+0000","last_clean":"2023-10-09T06:26:03.637755+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:26:03.637755+0000","last_undegraded":"2023-10-09T06:26:03.637755+0000","last_fullsized":"2023-10-09T06:26:03.637755+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.683665+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:26:17.683665+0000","last_peered":"2023-10-09T06:26:17.683665+0000","last_clean":"2023-10-09T06:26:17.683665+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:26:17.683665+0000","last_undegraded":"2023-10-09T06:26:17.683665+0000","last_fullsized":"2023-10-09T06:26:17.683665+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":248,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.652369+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:26:17.652369+0000","last_peered":"2023-10-09T06:26:17.652369+0000","last_clean":"2023-10-09T06:26:17.652369+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:26:17.652369+0000","last_undegraded":"2023-10-09T06:26:17.652369+0000","last_fullsized":"2023-10-09T06:26:17.652369+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":586,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.662720+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:26:16.662720+0000","last_peered":"2023-10-09T06:26:16.662720+0000","last_clean":"2023-10-09T06:26:16.662720+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:26:16.662720+0000","last_undegraded":"2023-10-09T06:26:16.662720+0000","last_fullsized":"2023-10-09T06:26:16.662720+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":258,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.655005+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:26:16.655005+0000","last_peered":"2023-10-09T06:26:16.655005+0000","last_clean":"2023-10-09T06:26:16.655005+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:26:16.655005+0000","last_undegraded":"2023-10-09T06:26:16.655005+0000","last_fullsized":"2023-10-09T06:26:16.655005+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":254,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650751+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:26:16.650751+0000","last_peered":"2023-10-09T06:26:16.650751+0000","last_clean":"2023-10-09T06:26:16.650751+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:26:16.650751+0000","last_undegraded":"2023-10-09T06:26:16.650751+0000","last_fullsized":"2023-10-09T06:26:16.650751+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":577,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:06.708822+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:26:06.708822+0000","last_peered":"2023-10-09T06:26:06.708822+0000","last_clean":"2023-10-09T06:26:06.708822+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:26:06.708822+0000","last_undegraded":"2023-10-09T06:26:06.708822+0000","last_fullsized":"2023-10-09T06:26:06.708822+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":253,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.637655+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:26:03.637655+0000","last_peered":"2023-10-09T06:26:03.637655+0000","last_clean":"2023-10-09T06:26:03.637655+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:26:03.637655+0000","last_undegraded":"2023-10-09T06:26:03.637655+0000","last_fullsized":"2023-10-09T06:26:03.637655+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":271,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650979+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:26:16.650979+0000","last_peered":"2023-10-09T06:26:16.650979+0000","last_clean":"2023-10-09T06:26:16.650979+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:26:16.650979+0000","last_undegraded":"2023-10-09T06:26:16.650979+0000","last_fullsized":"2023-10-09T06:26:16.650979+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":275,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650282+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:26:16.650282+0000","last_peered":"2023-10-09T06:26:16.650282+0000","last_clean":"2023-10-09T06:26:16.650282+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:26:16.650282+0000","last_undegraded":"2023-10-09T06:26:16.650282+0000","last_fullsized":"2023-10-09T06:26:16.650282+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":254,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650927+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:26:16.650927+0000","last_peered":"2023-10-09T06:26:16.650927+0000","last_clean":"2023-10-09T06:26:16.650927+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:26:16.650927+0000","last_undegraded":"2023-10-09T06:26:16.650927+0000","last_fullsized":"2023-10-09T06:26:16.650927+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":573,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:06.708880+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:26:06.708880+0000","last_peered":"2023-10-09T06:26:06.708880+0000","last_clean":"2023-10-09T06:26:06.708880+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:26:06.708880+0000","last_undegraded":"2023-10-09T06:26:06.708880+0000","last_fullsized":"2023-10-09T06:26:06.708880+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654341+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:26:16.654341+0000","last_peered":"2023-10-09T06:26:16.654341+0000","last_clean":"2023-10-09T06:26:16.654341+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:26:16.654341+0000","last_undegraded":"2023-10-09T06:26:16.654341+0000","last_fullsized":"2023-10-09T06:26:16.654341+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.645464+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:26:03.645464+0000","last_peered":"2023-10-09T06:26:03.645464+0000","last_clean":"2023-10-09T06:26:03.645464+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:26:03.645464+0000","last_undegraded":"2023-10-09T06:26:03.645464+0000","last_fullsized":"2023-10-09T06:26:03.645464+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":515,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.638033+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:26:03.638033+0000","last_peered":"2023-10-09T06:26:03.638033+0000","last_clean":"2023-10-09T06:26:03.638033+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:26:03.638033+0000","last_undegraded":"2023-10-09T06:26:03.638033+0000","last_fullsized":"2023-10-09T06:26:03.638033+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":276,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.644547+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:26:03.644547+0000","last_peered":"2023-10-09T06:26:03.644547+0000","last_clean":"2023-10-09T06:26:03.644547+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:26:03.644547+0000","last_undegraded":"2023-10-09T06:26:03.644547+0000","last_fullsized":"2023-10-09T06:26:03.644547+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":203,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654805+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:26:16.654805+0000","last_peered":"2023-10-09T06:26:16.654805+0000","last_clean":"2023-10-09T06:26:16.654805+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:26:16.654805+0000","last_undegraded":"2023-10-09T06:26:16.654805+0000","last_fullsized":"2023-10-09T06:26:16.654805+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":219,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.642769+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:26:03.642769+0000","last_peered":"2023-10-09T06:26:03.642769+0000","last_clean":"2023-10-09T06:26:03.642769+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:26:03.642769+0000","last_undegraded":"2023-10-09T06:26:03.642769+0000","last_fullsized":"2023-10-09T06:26:03.642769+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":580,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.650977+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:26:17.650977+0000","last_peered":"2023-10-09T06:26:17.650977+0000","last_clean":"2023-10-09T06:26:17.650977+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:26:17.650977+0000","last_undegraded":"2023-10-09T06:26:17.650977+0000","last_fullsized":"2023-10-09T06:26:17.650977+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":150,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.642895+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:26:03.642895+0000","last_peered":"2023-10-09T06:26:03.642895+0000","last_clean":"2023-10-09T06:26:03.642895+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:26:03.642895+0000","last_undegraded":"2023-10-09T06:26:03.642895+0000","last_fullsized":"2023-10-09T06:26:03.642895+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":255,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650507+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:26:16.650507+0000","last_peered":"2023-10-09T06:26:16.650507+0000","last_clean":"2023-10-09T06:26:16.650507+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:26:16.650507+0000","last_undegraded":"2023-10-09T06:26:16.650507+0000","last_fullsized":"2023-10-09T06:26:16.650507+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.644508+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:26:03.644508+0000","last_peered":"2023-10-09T06:26:03.644508+0000","last_clean":"2023-10-09T06:26:03.644508+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:26:03.644508+0000","last_undegraded":"2023-10-09T06:26:03.644508+0000","last_fullsized":"2023-10-09T06:26:03.644508+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":586,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.662278+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:26:16.662278+0000","last_peered":"2023-10-09T06:26:16.662278+0000","last_clean":"2023-10-09T06:26:16.662278+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:26:16.662278+0000","last_undegraded":"2023-10-09T06:26:16.662278+0000","last_fullsized":"2023-10-09T06:26:16.662278+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":259,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.630689+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:26:03.630689+0000","last_peered":"2023-10-09T06:26:03.630689+0000","last_clean":"2023-10-09T06:26:03.630689+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:26:03.630689+0000","last_undegraded":"2023-10-09T06:26:03.630689+0000","last_fullsized":"2023-10-09T06:26:03.630689+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":266,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.643604+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:26:03.643604+0000","last_peered":"2023-10-09T06:26:03.643604+0000","last_clean":"2023-10-09T06:26:03.643604+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:26:03.643604+0000","last_undegraded":"2023-10-09T06:26:03.643604+0000","last_fullsized":"2023-10-09T06:26:03.643604+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":262,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.645546+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:26:03.645546+0000","last_peered":"2023-10-09T06:26:03.645546+0000","last_clean":"2023-10-09T06:26:03.645546+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:26:03.645546+0000","last_undegraded":"2023-10-09T06:26:03.645546+0000","last_fullsized":"2023-10-09T06:26:03.645546+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":226,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.626697+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:26:03.626697+0000","last_peered":"2023-10-09T06:26:03.626697+0000","last_clean":"2023-10-09T06:26:03.626697+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:26:03.626697+0000","last_undegraded":"2023-10-09T06:26:03.626697+0000","last_fullsized":"2023-10-09T06:26:03.626697+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":567,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661047+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:26:16.661047+0000","last_peered":"2023-10-09T06:26:16.661047+0000","last_clean":"2023-10-09T06:26:16.661047+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:26:16.661047+0000","last_undegraded":"2023-10-09T06:26:16.661047+0000","last_fullsized":"2023-10-09T06:26:16.661047+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":221,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654495+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:26:16.654495+0000","last_peered":"2023-10-09T06:26:16.654495+0000","last_clean":"2023-10-09T06:26:16.654495+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:26:16.654495+0000","last_undegraded":"2023-10-09T06:26:16.654495+0000","last_fullsized":"2023-10-09T06:26:16.654495+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":255,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.638041+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:26:03.638041+0000","last_peered":"2023-10-09T06:26:03.638041+0000","last_clean":"2023-10-09T06:26:03.638041+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:26:03.638041+0000","last_undegraded":"2023-10-09T06:26:03.638041+0000","last_fullsized":"2023-10-09T06:26:03.638041+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":576,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.643949+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:26:03.643949+0000","last_peered":"2023-10-09T06:26:03.643949+0000","last_clean":"2023-10-09T06:26:03.643949+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:26:03.643949+0000","last_undegraded":"2023-10-09T06:26:03.643949+0000","last_fullsized":"2023-10-09T06:26:03.643949+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":137,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.660770+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:26:16.660770+0000","last_peered":"2023-10-09T06:26:16.660770+0000","last_clean":"2023-10-09T06:26:16.660770+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:26:16.660770+0000","last_undegraded":"2023-10-09T06:26:16.660770+0000","last_fullsized":"2023-10-09T06:26:16.660770+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":248,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.650892+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:26:17.650892+0000","last_peered":"2023-10-09T06:26:17.650892+0000","last_clean":"2023-10-09T06:26:17.650892+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:26:17.650892+0000","last_undegraded":"2023-10-09T06:26:17.650892+0000","last_fullsized":"2023-10-09T06:26:17.650892+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653525+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:26:17.653525+0000","last_peered":"2023-10-09T06:26:17.653525+0000","last_clean":"2023-10-09T06:26:17.653525+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:26:17.653525+0000","last_undegraded":"2023-10-09T06:26:17.653525+0000","last_fullsized":"2023-10-09T06:26:17.653525+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":515,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.637886+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:26:03.637886+0000","last_peered":"2023-10-09T06:26:03.637886+0000","last_clean":"2023-10-09T06:26:03.637886+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:26:03.637886+0000","last_undegraded":"2023-10-09T06:26:03.637886+0000","last_fullsized":"2023-10-09T06:26:03.637886+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":137,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.661724+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:26:16.661724+0000","last_peered":"2023-10-09T06:26:16.661724+0000","last_clean":"2023-10-09T06:26:16.661724+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:26:16.661724+0000","last_undegraded":"2023-10-09T06:26:16.661724+0000","last_fullsized":"2023-10-09T06:26:16.661724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":275,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.644457+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:26:03.644457+0000","last_peered":"2023-10-09T06:26:03.644457+0000","last_clean":"2023-10-09T06:26:03.644457+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:26:03.644457+0000","last_undegraded":"2023-10-09T06:26:03.644457+0000","last_fullsized":"2023-10-09T06:26:03.644457+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.652526+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:26:17.652526+0000","last_peered":"2023-10-09T06:26:17.652526+0000","last_clean":"2023-10-09T06:26:17.652526+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:26:17.652526+0000","last_undegraded":"2023-10-09T06:26:17.652526+0000","last_fullsized":"2023-10-09T06:26:17.652526+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":577,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650713+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:26:16.650713+0000","last_peered":"2023-10-09T06:26:16.650713+0000","last_clean":"2023-10-09T06:26:16.650713+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:26:16.650713+0000","last_undegraded":"2023-10-09T06:26:16.650713+0000","last_fullsized":"2023-10-09T06:26:16.650713+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":284,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654881+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:26:16.654881+0000","last_peered":"2023-10-09T06:26:16.654881+0000","last_clean":"2023-10-09T06:26:16.654881+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:26:16.654881+0000","last_undegraded":"2023-10-09T06:26:16.654881+0000","last_fullsized":"2023-10-09T06:26:16.654881+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":252,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.645878+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:26:03.645878+0000","last_peered":"2023-10-09T06:26:03.645878+0000","last_clean":"2023-10-09T06:26:03.645878+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:26:03.645878+0000","last_undegraded":"2023-10-09T06:26:03.645878+0000","last_fullsized":"2023-10-09T06:26:03.645878+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":273,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654272+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:26:16.654272+0000","last_peered":"2023-10-09T06:26:16.654272+0000","last_clean":"2023-10-09T06:26:16.654272+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:26:16.654272+0000","last_undegraded":"2023-10-09T06:26:16.654272+0000","last_fullsized":"2023-10-09T06:26:16.654272+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":598,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.655004+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:26:16.655004+0000","last_peered":"2023-10-09T06:26:16.655004+0000","last_clean":"2023-10-09T06:26:16.655004+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:26:16.655004+0000","last_undegraded":"2023-10-09T06:26:16.655004+0000","last_fullsized":"2023-10-09T06:26:16.655004+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":277,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.638189+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:26:03.638189+0000","last_peered":"2023-10-09T06:26:03.638189+0000","last_clean":"2023-10-09T06:26:03.638189+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:26:03.638189+0000","last_undegraded":"2023-10-09T06:26:03.638189+0000","last_fullsized":"2023-10-09T06:26:03.638189+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":252,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653376+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:26:17.653376+0000","last_peered":"2023-10-09T06:26:17.653376+0000","last_clean":"2023-10-09T06:26:17.653376+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:26:17.653376+0000","last_undegraded":"2023-10-09T06:26:17.653376+0000","last_fullsized":"2023-10-09T06:26:17.653376+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650604+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:26:16.650604+0000","last_peered":"2023-10-09T06:26:16.650604+0000","last_clean":"2023-10-09T06:26:16.650604+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:26:16.650604+0000","last_undegraded":"2023-10-09T06:26:16.650604+0000","last_fullsized":"2023-10-09T06:26:16.650604+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":254,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.652472+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:26:17.652472+0000","last_peered":"2023-10-09T06:26:17.652472+0000","last_clean":"2023-10-09T06:26:17.652472+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:26:17.652472+0000","last_undegraded":"2023-10-09T06:26:17.652472+0000","last_fullsized":"2023-10-09T06:26:17.652472+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":275,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-10-09T06:26:03.630693+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:26:03.630693+0000","last_peered":"2023-10-09T06:26:03.630693+0000","last_clean":"2023-10-09T06:26:03.630693+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:26:03.630693+0000","last_undegraded":"2023-10-09T06:26:03.630693+0000","last_fullsized":"2023-10-09T06:26:03.630693+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":99,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650503+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:26:16.650503+0000","last_peered":"2023-10-09T06:26:16.650503+0000","last_clean":"2023-10-09T06:26:16.650503+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:26:16.650503+0000","last_undegraded":"2023-10-09T06:26:16.650503+0000","last_fullsized":"2023-10-09T06:26:16.650503+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":281,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650416+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:26:16.650416+0000","last_peered":"2023-10-09T06:26:16.650416+0000","last_clean":"2023-10-09T06:26:16.650416+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:26:16.650416+0000","last_undegraded":"2023-10-09T06:26:16.650416+0000","last_fullsized":"2023-10-09T06:26:16.650416+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":99,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.651006+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:26:16.651006+0000","last_peered":"2023-10-09T06:26:16.651006+0000","last_clean":"2023-10-09T06:26:16.651006+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:26:16.651006+0000","last_undegraded":"2023-10-09T06:26:16.651006+0000","last_fullsized":"2023-10-09T06:26:16.651006+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":271,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650501+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:26:16.650501+0000","last_peered":"2023-10-09T06:26:16.650501+0000","last_clean":"2023-10-09T06:26:16.650501+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:26:16.650501+0000","last_undegraded":"2023-10-09T06:26:16.650501+0000","last_fullsized":"2023-10-09T06:26:16.650501+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":252,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654678+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:26:16.654678+0000","last_peered":"2023-10-09T06:26:16.654678+0000","last_clean":"2023-10-09T06:26:16.654678+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:26:16.654678+0000","last_undegraded":"2023-10-09T06:26:16.654678+0000","last_fullsized":"2023-10-09T06:26:16.654678+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.654489+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:26:16.654489+0000","last_peered":"2023-10-09T06:26:16.654489+0000","last_clean":"2023-10-09T06:26:16.654489+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:26:16.654489+0000","last_undegraded":"2023-10-09T06:26:16.654489+0000","last_fullsized":"2023-10-09T06:26:16.654489+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":267,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-10-09T06:26:17.653493+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:26:17.653493+0000","last_peered":"2023-10-09T06:26:17.653493+0000","last_clean":"2023-10-09T06:26:17.653493+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:26:17.653493+0000","last_undegraded":"2023-10-09T06:26:17.653493+0000","last_fullsized":"2023-10-09T06:26:17.653493+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":254,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-10-09T06:26:16.650081+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:26:16.650081+0000","last_peered":"2023-10-09T06:26:16.650081+0000","last_clean":"2023-10-09T06:26:16.650081+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:26:16.650081+0000","last_undegraded":"2023-10-09T06:26:16.650081+0000","last_fullsized":"2023-10-09T06:26:16.650081+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234348,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907492,"kb_used_data":3164,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":92837660,"statfs":{"total":95995035648,"available":95065763840,"internally_reserved":0,"allocated":3239936,"data_stored":2461548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2149999999999999,"5min":8.8960000000000008,"15min":3.8580000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":89.027000000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":7.0759999999999996,"5min":9.2319999999999993,"15min":3.8980000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.218},"max":{"1min":89.135999999999996,"5min":219.399,"15min":219.399},"last":0.66800000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:25:20 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":3.1869999999999998,"15min":2.6659999999999999},"min":{"1min":0.253,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.242,"5min":141.44300000000001,"15min":141.44300000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.1739999999999999,"5min":3.4289999999999998,"15min":2.4630000000000001},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.7629999999999999,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.48399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:25:32 2023","interfaces":[{"interface":"back","average":{"1min":12.579000000000001,"5min":12.257999999999999,"15min":5.2720000000000002},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":92.293000000000006,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":12.222,"5min":12.436,"15min":5.4109999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.727999999999994,"5min":215.869,"15min":215.869},"last":0.65200000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.359,"15min":1.627},"min":{"1min":0.35599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4710000000000001,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.6739999999999999,"5min":2.4409999999999998,"15min":1.669},"min":{"1min":0.40500000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6760000000000002,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":2.0870000000000002,"15min":1.694},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.3620000000000001,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2,"15min":1.419},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9729999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.76700000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.169,"15min":1.7230000000000001},"min":{"1min":0.33900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.952,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.151,"15min":1.45},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":4.3280000000000003,"5min":5.7130000000000001,"15min":48.774999999999999},"last":0.79600000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.458,"15min":1.3580000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7169999999999996,"5min":8.4890000000000008,"15min":11.878},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.2310000000000001,"15min":1.216},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33000000000000002},"max":{"1min":3.0539999999999998,"5min":9.4900000000000002,"15min":12.141999999999999},"last":0.751}]}]},{"osd":6,"up_from":46,"seq":197568495985,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906520,"kb_used_data":3408,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":92838632,"statfs":{"total":95995035648,"available":95066759168,"internally_reserved":0,"allocated":3489792,"data_stored":2724056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":3.7690000000000001,"15min":3.0790000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":4.2560000000000002,"5min":110.926,"15min":110.926},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.2869999999999999,"5min":3.7480000000000002,"15min":3.2669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2859999999999996,"5min":110.911,"15min":110.911},"last":0.58199999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:25:19 2023","interfaces":[{"interface":"back","average":{"1min":6.5010000000000003,"5min":2.8690000000000002,"15min":3.5310000000000001},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":66.796999999999997,"5min":66.796999999999997,"15min":114.202},"last":0.68100000000000005},{"interface":"front","average":{"1min":6.5460000000000003,"5min":2.5630000000000002,"15min":3.3530000000000002},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":66.704999999999998,"5min":66.704999999999998,"15min":114.179},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":4.0890000000000004,"15min":2.774},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":52.609999999999999,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.71999999999999997},{"interface":"front","average":{"1min":5.1059999999999999,"5min":3.9119999999999999,"15min":2.6669999999999998},"min":{"1min":0.29899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.575000000000003,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.25,"15min":2.02},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":6.9290000000000003,"5min":110.358,"15min":110.358},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.804,"5min":3.27,"15min":2.069},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.9359999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.66100000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.1299999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.1890000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":0.751},{"interface":"front","average":{"1min":1.03,"5min":1.474,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":4.2119999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.63600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.456,"15min":1.3640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.2460000000000004,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.488,"15min":1.407},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.8920000000000003,"5min":24.401,"15min":24.401},"last":0.73499999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:05 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":2.3639999999999999,"15min":1.536},"min":{"1min":0.35399999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":4.7610000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.115,"5min":2.343,"15min":1.732},"min":{"1min":0.371,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":4.7510000000000003,"5min":100.995,"15min":100.995},"last":0.76000000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402721,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":584436,"kb_used_data":93228,"kb_used_omap":4,"kb_used_meta":491195,"kb_avail":93160716,"statfs":{"total":95995035648,"available":95396573184,"internally_reserved":0,"allocated":95465472,"data_stored":94194579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":502984590},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5919999999999996,"5min":6.9119999999999999,"15min":4.032},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":102.471,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.247},{"interface":"front","average":{"1min":7.7779999999999996,"5min":7.9870000000000001,"15min":4.5599999999999996},"min":{"1min":0.20399999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":102.054,"5min":145.982,"15min":149.02699999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":5.6630000000000003,"15min":3.1059999999999999},"min":{"1min":0.26000000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":8.4030000000000005,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":3.3530000000000002,"5min":6.7800000000000002,"15min":3.863},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.219},"max":{"1min":9.8859999999999992,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.30399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:26:06 2023","interfaces":[{"interface":"back","average":{"1min":4.2439999999999998,"5min":4.0350000000000001,"15min":2.585},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":16.047999999999998,"5min":77.331999999999994,"15min":153.18199999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.6549999999999998,"15min":1.9319999999999999},"min":{"1min":0.23799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.21,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":5.266,"15min":3.7469999999999999},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.1760000000000002,"5min":99.391000000000005,"15min":199.43799999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":5.1180000000000003,"15min":3.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.3170000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":0.433}]},{"osd":5,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.5859999999999999,"15min":2.5499999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7169999999999996,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":3.5169999999999999,"15min":2.911},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":6.7789999999999999,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.38900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":4.1269999999999998,"15min":2.948},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":6.6799999999999997,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.3559999999999999,"5min":3.8130000000000002,"15min":2.7429999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9290000000000003,"5min":77.378,"15min":77.378},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5289999999999999,"15min":2.706},"min":{"1min":0.36099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.2069999999999999,"5min":77.456999999999994,"15min":77.456999999999994},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.105,"5min":2.9929999999999999,"15min":2.2040000000000002},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6900000000000004,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.57699999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024155,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":858188,"kb_used_data":174340,"kb_used_omap":5,"kb_used_meta":683834,"kb_avail":92886964,"statfs":{"total":95995035648,"available":95116251136,"internally_reserved":0,"allocated":178524160,"data_stored":177021981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":700246650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.3769999999999998,"15min":2.8170000000000002},"min":{"1min":0.193,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":7.5369999999999999,"5min":185.17400000000001,"15min":185.17400000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.6280000000000001,"5min":4.5570000000000004,"15min":3.351},"min":{"1min":0.254,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":6.6319999999999997,"5min":186.691,"15min":186.691},"last":0.251}]},{"osd":2,"last update":"Mon Oct 9 06:25:36 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":2.4620000000000002,"15min":2.3010000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.17799999999999999},"max":{"1min":10.255000000000001,"5min":13.211,"15min":17.588999999999999},"last":0.19900000000000001},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.6970000000000001,"15min":2.4049999999999998},"min":{"1min":0.33300000000000002,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":14.974,"5min":14.974,"15min":21.198},"last":0.30599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:25:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2010000000000001,"5min":3.3999999999999999,"15min":2.9140000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2989999999999995,"5min":52.683,"15min":185.89400000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.9950000000000001,"5min":3.4289999999999998,"15min":3.0259999999999998},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":9.3759999999999994,"5min":52.941000000000003,"15min":186.72900000000001},"last":0.27900000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5860000000000001,"15min":1.516},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.024,"5min":7.9660000000000002,"15min":13.244},"last":0.746},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.833,"15min":1.9410000000000001},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":4.9770000000000003,"5min":8.4890000000000008,"15min":12.752000000000001},"last":0.63700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.0990000000000002,"15min":1.9239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8450000000000002,"5min":62.984999999999999,"15min":62.984999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.7090000000000001,"15min":2.504},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":5.8650000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.52000000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.282,"15min":2.319},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.1360000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.498,"5min":1.6299999999999999,"15min":1.8400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.819,"5min":7.9980000000000002,"15min":13.315},"last":0.78700000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.5099999999999998,"15min":2.1920000000000002},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":4.7279999999999998,"5min":62.262,"15min":62.262},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.202,"5min":2.9910000000000001,"15min":2.5619999999999998},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":7.1059999999999999,"5min":62.229999999999997,"15min":62.229999999999997},"last":0.71699999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814023,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416156,"kb_used_data":3604,"kb_used_omap":7,"kb_used_meta":412536,"kb_avail":93328996,"statfs":{"total":95995035648,"available":95568891904,"internally_reserved":0,"allocated":3690496,"data_stored":2439596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":422436945},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":10.217000000000001,"5min":4.6269999999999998,"15min":3.968},"min":{"1min":0.22800000000000001,"5min":0.157,"15min":0.157},"max":{"1min":150.32599999999999,"5min":150.32599999999999,"15min":163.40100000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.1629999999999998,"5min":3.4260000000000002,"15min":3.8820000000000001},"min":{"1min":0.20899999999999999,"5min":0.182,"15min":0.182},"max":{"1min":21.472000000000001,"5min":110.66500000000001,"15min":163.363},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.8579999999999997,"15min":3.6680000000000001},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":7.9710000000000001,"5min":205.148,"15min":222.50299999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.2090000000000001,"5min":4.8280000000000003,"15min":3.984},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":11.811,"5min":205.267,"15min":222.19399999999999},"last":0.20599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":2.8780000000000001,"15min":1.8140000000000001},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":13.664999999999999,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.0150000000000001,"5min":2.9369999999999998,"15min":1.8700000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.728,"5min":69.906999999999996,"15min":81.061999999999998},"last":0.25700000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.952,"15min":2.2240000000000002},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":52.188000000000002,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.6179999999999999,"5min":3.282,"15min":2.4910000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":52.167999999999999,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.472,"15min":1.534},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.309},"max":{"1min":5.508,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.256,"5min":1.7549999999999999,"15min":2.0990000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.9349999999999996,"5min":10.548,"15min":10.548},"last":0.60399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":3.7839999999999998,"15min":2.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":55.392000000000003,"5min":138.291,"15min":138.291},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.8150000000000004,"5min":3.4910000000000001,"15min":2.206},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":55.146999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9269999999999996,"5min":2.2709999999999999,"15min":1.863},"min":{"1min":0.40699999999999997,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":55.851999999999997,"5min":55.851999999999997,"15min":55.851999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.1890000000000001,"15min":2.395},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":55.731000000000002,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.58599999999999997}]}]},{"osd":3,"up_from":149,"seq":639950127221,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":458524,"kb_used_data":67988,"kb_used_omap":2,"kb_used_meta":390525,"kb_avail":93286628,"statfs":{"total":95995035648,"available":95525507072,"internally_reserved":0,"allocated":69619712,"data_stored":68442718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":399897653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":7.8659999999999997,"15min":3.262},"min":{"1min":0.19800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":10.367000000000001,"5min":187.852,"15min":187.852},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.23,"5min":8.4719999999999995,"15min":3.7200000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":4.6539999999999999,"5min":187.923,"15min":187.923},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":6.0439999999999996,"15min":2.9500000000000002},"min":{"1min":0.255,"5min":0.246,"15min":0.223},"max":{"1min":11.367000000000001,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.978,"5min":3.673,"15min":1.9379999999999999},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.8810000000000002,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.312}]},{"osd":2,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":5.2350000000000003,"15min":2.649},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.57,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.3210000000000002,"5min":4.8769999999999998,"15min":2.1349999999999998},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5250000000000004,"5min":187.79499999999999,"15min":187.79499999999999},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.544,"15min":1.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":6.7830000000000004,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.2389999999999999,"15min":1.964},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.9649999999999999,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.56399999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":3.2530000000000001,"15min":2.1080000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":3.0059999999999998,"5min":187.833,"15min":187.833},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.0649999999999999,"15min":2.036},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":187.952,"15min":187.952},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.6080000000000001,"15min":2.1560000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6280000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1890000000000001,"15min":1.9910000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8719999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.504}]},{"osd":7,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":3.5169999999999999,"15min":1.8320000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":6.8239999999999998,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.242,"5min":3.4580000000000002,"15min":2.093},"min":{"1min":0.41899999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":5.431,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.53900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019257,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085720,"kb_used_data":198928,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":92659432,"statfs":{"total":95995035648,"available":94883258368,"internally_reserved":0,"allocated":203702272,"data_stored":202935815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":3.5310000000000001,"15min":2.2469999999999999},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.7480000000000002,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.1279999999999999,"5min":3.9089999999999998,"15min":2.6549999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":4.3799999999999999,"5min":76.988,"15min":76.988},"last":0.442}]},{"osd":1,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.9890000000000001,"15min":2.0699999999999998},"min":{"1min":0.25700000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.8069999999999999,"5min":37.383000000000003,"15min":91.849999999999994},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.1869999999999998,"15min":2.177},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":5.5869999999999997,"5min":37.436999999999998,"15min":91.885000000000005},"last":0.73699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.9240000000000004,"5min":2.8490000000000002,"15min":2.093},"min":{"1min":0.41599999999999998,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":44.807000000000002,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":5.2370000000000001,"5min":3.0139999999999998,"15min":2.278},"min":{"1min":0.29699999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":44.697000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.7029999999999998,"15min":2.2879999999999998},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.8360000000000003,"5min":45.591000000000001,"15min":91.697000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.341,"5min":2.3140000000000001,"15min":2.129},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":3.5230000000000001,"5min":45.548999999999999,"15min":94.356999999999999},"last":1.1919999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.1619999999999999,"15min":1.107},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.24299999999999999},"max":{"1min":4.2270000000000003,"5min":10.673,"15min":10.673},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.21,"5min":1.014,"15min":1.23},"min":{"1min":0.29599999999999999,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":4.6189999999999998,"5min":9.6379999999999999,"15min":12.552},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.83399999999999996,"15min":0.84899999999999998},"min":{"1min":0.29399999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":2.1179999999999999,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1830000000000001,"15min":1.286},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.859,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.70599999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.7350000000000001,"15min":1.476},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":2.1880000000000002,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.876,"5min":1.706,"15min":1.488},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":3.8450000000000002,"5min":62.573,"15min":62.573},"last":0.81200000000000006}]}]},{"osd":5,"up_from":0,"seq":163208757622,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:26:23.339 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:23.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:23.442 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:23.473 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:23.522 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:23.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:23.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:23.657 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:23.700 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:23.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:23.806 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:23.830 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:23 smithi150 ceph-mon[194178]: from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:23.833 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:23.840 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:23 smithi107 ceph-mon[212750]: from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:23.840 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:23 smithi107 ceph-mon[220921]: from='client.45935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:23.876 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:23.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:23.981 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:24.007 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:24.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:24.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:24.152 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:24.178 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:24.220 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:24.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:24.235 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:24.327 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:24.356 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:24.723 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:24.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:24.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:24 smithi107 ceph-mon[212750]: pgmap v915: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:24.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:24 smithi107 ceph-mon[220921]: pgmap v915: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:24.827 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:24.856 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:24.900 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:24.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:25.007 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:25.035 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:24 smithi150 ceph-mon[194178]: pgmap v915: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:25.368 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:25.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:25.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:25.500 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:25.600 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:25.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:25.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:25.705 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:25.730 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64723 9.2 9:4bf0c3c2:::benchmark_data_smithi107_300016_object64722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.86158438299998, 2023-10-09T06:26:26.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.362261308, 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64723 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078542+0000", 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078541+0000", 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078567+0000", 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6197e-05 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078577+0000", 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7799999999999995e-06 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078583+0000", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4130000000000003e-06 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079048+0000", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046495299999999999 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079196+0000", 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014766599999999999 2023-10-09T06:26:26.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.079363+0000", 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016698200000000001 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.084453+0000", 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050901360000000003 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440670+0000", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.356217269 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440709+0000", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9119999999999998e-05 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440762+0000", 2023-10-09T06:26:26.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2203000000000001e-05 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440804+0000", 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1916999999999999e-05 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64724 9.7 e163/160 9:e68efaaf:::benchmark_data_smithi107_300016_object64723:head v 163'4118, mlcod=0'0)", 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:26:26.116 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.86148559899999, 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35938271500000002, 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078641+0000", 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078682+0000", 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0822999999999999e-05 2023-10-09T06:26:26.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078689+0000", 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0609999999999998e-06 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078694+0000", 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0590000000000002e-06 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081003+0000", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023091069999999999 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081031+0000", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8022e-05 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437591+0000", 2023-10-09T06:26:26.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35656025299999999 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438024+0000", 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043272699999999999 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64725 9.f e163/152 9:f1989478:::benchmark_data_smithi107_300016_object64724:head v 163'3999, mlcod=163'3997)", 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:26:26.121 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.86141992199998, 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35934786400000002, 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078707+0000", 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967296 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078736+0000", 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9133e-05 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078741+0000", 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5060000000000003e-06 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.078745+0000", 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1790000000000001e-06 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083200+0000", 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044543509999999996 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.083218+0000", 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8397999999999999e-05 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438028+0000", 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35480992500000003 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438055+0000", 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6554e-05 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64727 9.8 9:1ddec4f7:::benchmark_data_smithi107_300016_object64726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.85853713500001, 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35651537999999999, 2023-10-09T06:26:26.126 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64727 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081590+0000", 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081588+0000", 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081613+0000", 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5406e-05 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081626+0000", 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2520999999999999e-05 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081634+0000", 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6500000000000002e-06 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081691+0000", 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6190999999999998e-05 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081807+0000", 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011626299999999999 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:26:26.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081946+0000", 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013932899999999999 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087000+0000", 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050537489999999997 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437968+0000", 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35096775800000002 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438021+0000", 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3038999999999998e-05 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438065+0000", 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4045999999999998e-05 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438105+0000", 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0014e-05 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64729 9.3 9:cba02ed8:::benchmark_data_smithi107_300016_object64728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.85847494400002, 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37691757999999997, 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.133 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64729 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081651+0000", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081673+0000", 2023-10-09T06:26:26.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2178999999999999e-05 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081681+0000", 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9219999999999997e-06 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081688+0000", 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2469999999999997e-06 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085025+0000", 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033377429999999998 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085147+0000", 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000122089 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085280+0000", 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013249099999999999 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444158+0000", 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35887852100000001 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444204+0000", 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5355999999999997e-05 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456528+0000", 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01232391 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456591+0000", 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.458569+0000", 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019784849999999999 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.139 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64730 9.f e163/152 9:f3222a22:::benchmark_data_smithi107_300016_object64729:head v 163'4000, mlcod=163'3997)", 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.85839858000003, 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35634943299999999, 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081728+0000", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081727+0000", 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:26:26.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081887+0000", 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016013400000000001 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081898+0000", 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0875e-05 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081916+0000", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7195000000000001e-05 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085279+0000", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033639469999999999 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085297+0000", 2023-10-09T06:26:26.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7669e-05 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438058+0000", 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35276105099999999 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438077+0000", 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9317000000000002e-05 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.144 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64733 9.2 9:4941674d:::benchmark_data_smithi107_300016_object64732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.85339060899997, 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.360635498, 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64733 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086736+0000", 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086735+0000", 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086763+0000", 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7846000000000001e-05 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086772+0000", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9039999999999998e-06 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086778+0000", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2759999999999997e-06 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.086950+0000", 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172048 2023-10-09T06:26:26.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087082+0000", 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013214800000000001 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.087233+0000", 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000151213 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437967+0000", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35073413799999997 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.440845+0000", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028772379999999998 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447284+0000", 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064393209999999996 2023-10-09T06:26:26.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447333+0000", 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9299999999999999e-05 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447372+0000", 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8277e-05 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.151 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64729 9.3 e163/160 ondisk, result = 0)", 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.851302858, 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.355337442, 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:26:26.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088824+0000", 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088823+0000", 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088829+0000", 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5189999999999998e-06 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088840+0000", 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0951000000000001e-05 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.088848+0000", 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.391e-06 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444142+0000", 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35529475900000002 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444151+0000", 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:26:26.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444161+0000", 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0239e-05 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.45275.0:64727 9.8 e163/152 ondisk, result = 0)", 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.85018151499997, 2023-10-09T06:26:26.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.348172755, 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089945+0000", 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089944+0000", 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.089951+0000", 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3820000000000001e-06 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437572+0000", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34762106599999998 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437921+0000", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034914000000000001 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438007+0000", 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6438000000000001e-05 2023-10-09T06:26:26.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438015+0000", 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4359999999999996e-06 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438118+0000", 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103149 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.160 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64737 9.2 9:42e1c82a:::benchmark_data_smithi107_300016_object64736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.849771369, 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.359336867, 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.161 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64737 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090355+0000", 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090354+0000", 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090375+0000", 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1294e-05 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437567+0000", 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34719175099999999 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437894+0000", 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032692099999999998 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437999+0000", 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010511299999999999 2023-10-09T06:26:26.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438153+0000", 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015426700000000001 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438341+0000", 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018829099999999999 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447252+0000", 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0089106849999999998 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.447436+0000", 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018348799999999999 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449593+0000", 2023-10-09T06:26:26.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002157413 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449650+0000", 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7450000000000001e-05 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449692+0000", 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1686e-05 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.167 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:64736 9.a e163/160 9:553011b2:::benchmark_data_smithi107_300016_object64735:head v 163'4090, mlcod=0'0)", 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 457.84940632299998, 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35900552499999999, 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090720+0000", 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090719+0000", 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:26:26.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.090765+0000", 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5793e-05 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437575+0000", 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34680993900000001 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437924+0000", 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034926999999999999 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.437970+0000", 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5179999999999998e-05 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.438005+0000", 2023-10-09T06:26:26.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5361000000000002e-05 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449697+0000", 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011691906 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.449726+0000", 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8932999999999999e-05 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.172 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:112415 9.3 9:c7576938:::benchmark_data_smithi107_300016_object14843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.343694513, 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29174138700000002, 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.173 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 112415 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596432+0000", 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596434+0000", 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7590000000000002e-06 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596440+0000", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3129999999999998e-06 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.596445+0000", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0579999999999997e-06 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601164+0000", 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0047186659999999998 2023-10-09T06:26:26.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601299+0000", 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013497699999999999 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.601421+0000", 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012268199999999999 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.607663+0000", 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0062417699999999998 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.608492+0000", 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082930000000000005 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887363+0000", 2023-10-09T06:26:26.178 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27887095200000001 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.887422+0000", 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.889e-05 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:19:57.888174+0000", 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075141200000000002 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.179 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141939 9.7 e201/188 9:ee97ad70:::benchmark_data_smithi107_300016_object44367:head v 201'9015, mlcod=201'9011)", 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 345.009235639, 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29274838399999997, 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.180 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930891+0000", 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930890+0000", 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930897+0000", 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3210000000000001e-06 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930907+0000", 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0074e-05 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.182 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.930913+0000", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3160000000000002e-06 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935291+0000", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043775979999999999 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.935312+0000", 2023-10-09T06:26:26.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1523e-05 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223606+0000", 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28829408499999998 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.223639+0000", 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3176999999999998e-05 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.184 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141944 9.f e201/152 9:f293b650:::benchmark_data_smithi107_300016_object44372:head v 201'8823, mlcod=201'8821)", 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 345.00074253600002, 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30728261400000001, 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939384+0000", 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939383+0000", 2023-10-09T06:26:26.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939389+0000", 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1379999999999998e-06 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939397+0000", 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2600000000000005e-06 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939404+0000", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8419999999999999e-06 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939455+0000", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1220000000000001e-05 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.939471+0000", 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5988e-05 2023-10-09T06:26:26.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246587+0000", 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.307116161 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246667+0000", 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9617999999999996e-05 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.189 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141947 9.2 9:429ff582:::benchmark_data_smithi107_300016_object44375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 344.99972084400002, 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29032069100000002, 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.190 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141947 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940406+0000", 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940405+0000", 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940410+0000", 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8690000000000001e-06 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940422+0000", 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1296e-05 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940429+0000", 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1130000000000003e-06 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.940473+0000", 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4391999999999998e-05 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219069+0000", 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27859536699999998 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219175+0000", 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106272 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.226985+0000", 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0078100460000000002 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230576+0000", 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035914990000000002 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230642+0000", 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5840000000000007e-05 2023-10-09T06:26:26.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230689+0000", 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6653999999999998e-05 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230726+0000", 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7589000000000002e-05 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:141951 9.1 9:831efef4:::benchmark_data_smithi107_300016_object44379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 344.99593589699998, 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.286240618, 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 141951 2023-10-09T06:26:26.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944191+0000", 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944189+0000", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944195+0000", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8109999999999999e-06 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219289+0000", 2023-10-09T06:26:26.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.275094218 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219299+0000", 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7510000000000003e-06 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219377+0000", 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7982e-05 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219481+0000", 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103836 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219580+0000", 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9399000000000002e-05 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227278+0000", 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0076973400000000004 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.227425+0000", 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014719900000000001 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230360+0000", 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029345669999999999 2023-10-09T06:26:26.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230398+0000", 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8917e-05 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.230431+0000", 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2916000000000001e-05 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.203 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45275.0:141948 9.f e201/152 9:f4f21468:::benchmark_data_smithi107_300016_object44376:head v 201'8824, mlcod=201'8821)", 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 344.99563263099998, 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30220260900000001, 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.204 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944494+0000", 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944493+0000", 2023-10-09T06:26:26.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:40.944500+0000", 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4910000000000002e-06 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218928+0000", 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27442793999999998 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218942+0000", 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.412e-05 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.218983+0000", 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0599999999999998e-05 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.219005+0000", 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2347000000000001e-05 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246673+0000", 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027668321999999999 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:20:41.246697+0000", 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3408999999999999e-05 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.208 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189419 9.8 9:1506b9c6:::benchmark_data_smithi107_300016_object91847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.40251205099997, 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99043692000000005, 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.209 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189419 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537613+0000", 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537620+0000", 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2089999999999999e-06 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537637+0000", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7238e-05 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537645+0000", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8110000000000003e-06 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537696+0000", 2023-10-09T06:26:26.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0652000000000002e-05 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537708+0000", 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2262999999999999e-05 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508194+0000", 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97048550899999997 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508310+0000", 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116286 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508418+0000", 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010815999999999999 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.214 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.518815+0000", 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010397269000000001 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524555+0000", 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057400180000000004 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.527965+0000", 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034098969999999998 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528013+0000", 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8173000000000001e-05 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528052+0000", 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8163e-05 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.216 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189440 9.8 9:186cfebb:::benchmark_data_smithi107_300016_object91868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.35776846099998, 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94578946799999997, 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189440 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.217 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.219 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582357+0000", 2023-10-09T06:26:26.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.219 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582364+0000", 2023-10-09T06:26:26.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5750000000000003e-06 2023-10-09T06:26:26.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582377+0000", 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3012e-05 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582385+0000", 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0059999999999992e-06 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582445+0000", 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0266000000000002e-05 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582457+0000", 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1567e-05 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511852+0000", 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.92939540200000004 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511956+0000", 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103732 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.512066+0000", 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109555 2023-10-09T06:26:26.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.520344+0000", 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0082784669999999994 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524583+0000", 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004239306 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528084+0000", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035007110000000001 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528117+0000", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3117999999999997e-05 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528148+0000", 2023-10-09T06:26:26.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0361e-05 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189461 9.8 9:171409ee:::benchmark_data_smithi107_300016_object91889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.31365197700001, 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.90177728999999995, 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:26:26.227 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189461 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626473+0000", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626479+0000", 2023-10-09T06:26:26.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8989999999999998e-06 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626490+0000", 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0424999999999999e-05 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626495+0000", 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2089999999999996e-06 2023-10-09T06:26:26.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626547+0000", 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1792000000000002e-05 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626557+0000", 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0407e-05 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515348+0000", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.88879119399999995 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515455+0000", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106475 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515561+0000", 2023-10-09T06:26:26.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010639199999999999 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524611+0000", 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090500259999999992 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.526227+0000", 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016156779999999999 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528177+0000", 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001950212 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528210+0000", 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2511000000000001e-05 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:26:26.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528252+0000", 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2354000000000001e-05 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:26:26.235 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:26:26.316 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:26:26.540 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:26.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:26.644 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:26.676 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:26.728 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:26.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:26.780 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:26:26.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:26 smithi107 ceph-mon[212750]: pgmap v916: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:26.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:26 smithi107 ceph-mon[220921]: pgmap v916: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:26.835 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:26.865 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:26.900 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:26:26.925 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:26 smithi150 ceph-mon[194178]: pgmap v916: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:26.968 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:26.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:26.985 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:27.076 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:27.103 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:27.149 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:27.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:27.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:27.252 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:27.255 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:27.286 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:27.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:27.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:27.439 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:27.466 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:27.604 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:27.615 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:27.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:27.719 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:27.747 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:27.774 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_6' created 2023-10-09T06:26:27.774 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:26:27.803 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:27.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:27.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:27.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2545046750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:27.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2545046750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:27.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:27.907 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:27.935 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:27.982 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:27.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:28.086 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:28.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2545046750' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:28.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:26:28.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:28.113 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:28.156 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:28.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:28.260 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:28.288 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:28.834 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:28.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:28.894 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-10-09T06:26:28.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:28.964 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: pgmap v917: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T06:26:29.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:28 smithi150 ceph-mon[194178]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.066 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: pgmap v917: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:29.066 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T06:26:29.066 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:29.066 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.066 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[212750]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: pgmap v917: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:26:29.067 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3338610495' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T06:26:29.068 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T06:26:29.068 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:28 smithi107 ceph-mon[220921]: failed to encode map e297 with expected crc 2023-10-09T06:26:29.225 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:26:29.226 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:26:29.226 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:26:29.226 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:26:29.666 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:29.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:29.769 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:29.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T06:26:29.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[212750]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T06:26:29.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T06:26:29.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T06:26:29.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[212750]: failed to encode map e298 with expected crc 2023-10-09T06:26:29.797 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:29.855 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:26:29.942 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:29.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:29.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:30.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[220921]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[220921]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T06:26:30.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[220921]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[220921]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:29 smithi107 ceph-mon[220921]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.046 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:30.081 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:29 smithi150 ceph-mon[194178]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:29 smithi150 ceph-mon[194178]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T06:26:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:29 smithi150 ceph-mon[194178]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:29 smithi150 ceph-mon[194178]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:29 smithi150 ceph-mon[194178]: failed to encode map e298 with expected crc 2023-10-09T06:26:30.200 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:30.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:30.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:30.370 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:30.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:30.470 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:30.515 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:30.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:30.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:30.527 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:26:30.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:26:30.564 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:26:30.574 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:30.576 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:26:30.584 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:26:30.604 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:26:30.619 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:30.623 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:26:30.643 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:26:30.649 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:30.669 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:26:30.671 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.671 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.671 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.671 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.672 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.672 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.672 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.672 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: pgmap v920: 99 pgs: 6 creating+peering, 6 unknown, 87 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/942544322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: pgmap v920: 99 pgs: 6 creating+peering, 6 unknown, 87 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T06:26:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: failed to encode map e299 with expected crc 2023-10-09T06:26:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/942544322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: pgmap v920: 99 pgs: 6 creating+peering, 6 unknown, 87 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: failed to encode map e299 with expected crc 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: failed to encode map e299 with expected crc 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: failed to encode map e299 with expected crc 2023-10-09T06:26:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/942544322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:31.251 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:31.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:31.356 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:31.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:31.429 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:31.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:31.533 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:31.561 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:31.615 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":921,"stamp":"2023-10-09T06:26:29.768317+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":338010,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":256,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5161800,"kb_used_data":431184,"kb_used_omap":20,"kb_used_meta":4730539,"kb_avail":651054264,"statfs":{"total":671965249536,"available":666679566336,"internally_reserved":0,"allocated":441532416,"data_stored":433946495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4844072590},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.629092"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":276,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.677468+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:26:20.677468+0000","last_peered":"2023-10-09T06:26:20.677468+0000","last_clean":"2023-10-09T06:26:20.677468+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:26:20.677468+0000","last_undegraded":"2023-10-09T06:26:20.677468+0000","last_fullsized":"2023-10-09T06:26:20.677468+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":235,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773882+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:26:27.773882+0000","last_peered":"2023-10-09T06:26:27.773882+0000","last_clean":"2023-10-09T06:26:27.773882+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:26:27.773882+0000","last_undegraded":"2023-10-09T06:26:27.773882+0000","last_fullsized":"2023-10-09T06:26:27.773882+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":268,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958371+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:26:18.958371+0000","last_peered":"2023-10-09T06:26:18.958371+0000","last_clean":"2023-10-09T06:26:18.958371+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:26:18.958371+0000","last_undegraded":"2023-10-09T06:26:18.958371+0000","last_fullsized":"2023-10-09T06:26:18.958371+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665031+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:26:18.665031+0000","last_peered":"2023-10-09T06:26:18.665031+0000","last_clean":"2023-10-09T06:26:18.665031+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:26:18.665031+0000","last_undegraded":"2023-10-09T06:26:18.665031+0000","last_fullsized":"2023-10-09T06:26:18.665031+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":254,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772748+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:26:27.772748+0000","last_peered":"2023-10-09T06:26:27.772748+0000","last_clean":"2023-10-09T06:26:27.772748+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:26:27.772748+0000","last_undegraded":"2023-10-09T06:26:27.772748+0000","last_fullsized":"2023-10-09T06:26:27.772748+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":276,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.024440+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:26:21.024440+0000","last_peered":"2023-10-09T06:26:21.024440+0000","last_clean":"2023-10-09T06:26:21.024440+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:26:21.024440+0000","last_undegraded":"2023-10-09T06:26:21.024440+0000","last_fullsized":"2023-10-09T06:26:21.024440+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":257,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665963+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:26:18.665963+0000","last_peered":"2023-10-09T06:26:18.665963+0000","last_clean":"2023-10-09T06:26:18.665963+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:26:18.665963+0000","last_undegraded":"2023-10-09T06:26:18.665963+0000","last_fullsized":"2023-10-09T06:26:18.665963+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":236,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772648+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:26:27.772648+0000","last_peered":"2023-10-09T06:26:27.772648+0000","last_clean":"2023-10-09T06:26:27.772648+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:26:27.772648+0000","last_undegraded":"2023-10-09T06:26:27.772648+0000","last_fullsized":"2023-10-09T06:26:27.772648+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.011049+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:26:21.011049+0000","last_peered":"2023-10-09T06:26:21.011049+0000","last_clean":"2023-10-09T06:26:21.011049+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:26:21.011049+0000","last_undegraded":"2023-10-09T06:26:21.011049+0000","last_fullsized":"2023-10-09T06:26:21.011049+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":234,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772725+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:26:27.772725+0000","last_peered":"2023-10-09T06:26:27.772725+0000","last_clean":"2023-10-09T06:26:27.772725+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:26:27.772725+0000","last_undegraded":"2023-10-09T06:26:27.772725+0000","last_fullsized":"2023-10-09T06:26:27.772725+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":146,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010824+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:26:21.010824+0000","last_peered":"2023-10-09T06:26:21.010824+0000","last_clean":"2023-10-09T06:26:21.010824+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:26:21.010824+0000","last_undegraded":"2023-10-09T06:26:21.010824+0000","last_fullsized":"2023-10-09T06:26:21.010824+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958611+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:26:18.958611+0000","last_peered":"2023-10-09T06:26:18.958611+0000","last_clean":"2023-10-09T06:26:18.958611+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:26:18.958611+0000","last_undegraded":"2023-10-09T06:26:18.958611+0000","last_fullsized":"2023-10-09T06:26:18.958611+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":303,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678622+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:26:20.678622+0000","last_peered":"2023-10-09T06:26:20.678622+0000","last_clean":"2023-10-09T06:26:20.678622+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:26:20.678622+0000","last_undegraded":"2023-10-09T06:26:20.678622+0000","last_fullsized":"2023-10-09T06:26:20.678622+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":267,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678949+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:26:20.678949+0000","last_peered":"2023-10-09T06:26:20.678949+0000","last_clean":"2023-10-09T06:26:20.678949+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:26:20.678949+0000","last_undegraded":"2023-10-09T06:26:20.678949+0000","last_fullsized":"2023-10-09T06:26:20.678949+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":146,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010543+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:26:21.010543+0000","last_peered":"2023-10-09T06:26:21.010543+0000","last_clean":"2023-10-09T06:26:21.010543+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:26:21.010543+0000","last_undegraded":"2023-10-09T06:26:21.010543+0000","last_fullsized":"2023-10-09T06:26:21.010543+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773528+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:26:27.773528+0000","last_peered":"2023-10-09T06:26:27.773528+0000","last_clean":"2023-10-09T06:26:27.773528+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:26:27.773528+0000","last_undegraded":"2023-10-09T06:26:27.773528+0000","last_fullsized":"2023-10-09T06:26:27.773528+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664635+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:26:18.664635+0000","last_peered":"2023-10-09T06:26:18.664635+0000","last_clean":"2023-10-09T06:26:18.664635+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:26:18.664635+0000","last_undegraded":"2023-10-09T06:26:18.664635+0000","last_fullsized":"2023-10-09T06:26:18.664635+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010524+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:26:21.010524+0000","last_peered":"2023-10-09T06:26:21.010524+0000","last_clean":"2023-10-09T06:26:21.010524+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:26:21.010524+0000","last_undegraded":"2023-10-09T06:26:21.010524+0000","last_fullsized":"2023-10-09T06:26:21.010524+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":140,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.011501+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:26:21.011501+0000","last_peered":"2023-10-09T06:26:21.011501+0000","last_clean":"2023-10-09T06:26:21.011501+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:26:21.011501+0000","last_undegraded":"2023-10-09T06:26:21.011501+0000","last_fullsized":"2023-10-09T06:26:21.011501+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":266,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665158+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:26:18.665158+0000","last_peered":"2023-10-09T06:26:18.665158+0000","last_clean":"2023-10-09T06:26:18.665158+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:26:18.665158+0000","last_undegraded":"2023-10-09T06:26:18.665158+0000","last_fullsized":"2023-10-09T06:26:18.665158+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":285,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.019584+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:26:21.019584+0000","last_peered":"2023-10-09T06:26:21.019584+0000","last_clean":"2023-10-09T06:26:21.019584+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:26:21.019584+0000","last_undegraded":"2023-10-09T06:26:21.019584+0000","last_fullsized":"2023-10-09T06:26:21.019584+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":254,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.774201+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:26:27.774201+0000","last_peered":"2023-10-09T06:26:27.774201+0000","last_clean":"2023-10-09T06:26:27.774201+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:26:27.774201+0000","last_undegraded":"2023-10-09T06:26:27.774201+0000","last_fullsized":"2023-10-09T06:26:27.774201+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":158,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010976+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:26:21.010976+0000","last_peered":"2023-10-09T06:26:21.010976+0000","last_clean":"2023-10-09T06:26:21.010976+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:26:21.010976+0000","last_undegraded":"2023-10-09T06:26:21.010976+0000","last_fullsized":"2023-10-09T06:26:21.010976+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":207,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010394+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:26:21.010394+0000","last_peered":"2023-10-09T06:26:21.010394+0000","last_clean":"2023-10-09T06:26:21.010394+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:26:21.010394+0000","last_undegraded":"2023-10-09T06:26:21.010394+0000","last_fullsized":"2023-10-09T06:26:21.010394+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":144,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.011541+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:26:21.011541+0000","last_peered":"2023-10-09T06:26:21.011541+0000","last_clean":"2023-10-09T06:26:21.011541+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:26:21.011541+0000","last_undegraded":"2023-10-09T06:26:21.011541+0000","last_fullsized":"2023-10-09T06:26:21.011541+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":274,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.679489+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:26:20.679489+0000","last_peered":"2023-10-09T06:26:20.679489+0000","last_clean":"2023-10-09T06:26:20.679489+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:26:20.679489+0000","last_undegraded":"2023-10-09T06:26:20.679489+0000","last_fullsized":"2023-10-09T06:26:20.679489+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.769960+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:06:28.768036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":158,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010442+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:26:21.010442+0000","last_peered":"2023-10-09T06:26:21.010442+0000","last_clean":"2023-10-09T06:26:21.010442+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:26:21.010442+0000","last_undegraded":"2023-10-09T06:26:21.010442+0000","last_fullsized":"2023-10-09T06:26:21.010442+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":276,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.020949+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:26:21.020949+0000","last_peered":"2023-10-09T06:26:21.020949+0000","last_clean":"2023-10-09T06:26:21.020949+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:26:21.020949+0000","last_undegraded":"2023-10-09T06:26:21.020949+0000","last_fullsized":"2023-10-09T06:26:21.020949+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.769221+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:32.451881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":190,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773402+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:26:27.773402+0000","last_peered":"2023-10-09T06:26:27.773402+0000","last_clean":"2023-10-09T06:26:27.773402+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:26:27.773402+0000","last_undegraded":"2023-10-09T06:26:27.773402+0000","last_fullsized":"2023-10-09T06:26:27.773402+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":249,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664664+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:26:18.664664+0000","last_peered":"2023-10-09T06:26:18.664664+0000","last_clean":"2023-10-09T06:26:18.664664+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:26:18.664664+0000","last_undegraded":"2023-10-09T06:26:18.664664+0000","last_fullsized":"2023-10-09T06:26:18.664664+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.771252+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:00:59.154493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":589,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010984+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:26:21.010984+0000","last_peered":"2023-10-09T06:26:21.010984+0000","last_clean":"2023-10-09T06:26:21.010984+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:26:21.010984+0000","last_undegraded":"2023-10-09T06:26:21.010984+0000","last_fullsized":"2023-10-09T06:26:21.010984+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":262,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773218+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:26:27.773218+0000","last_peered":"2023-10-09T06:26:27.773218+0000","last_clean":"2023-10-09T06:26:27.773218+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:26:27.773218+0000","last_undegraded":"2023-10-09T06:26:27.773218+0000","last_fullsized":"2023-10-09T06:26:27.773218+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":257,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678714+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:26:20.678714+0000","last_peered":"2023-10-09T06:26:20.678714+0000","last_clean":"2023-10-09T06:26:20.678714+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:26:20.678714+0000","last_undegraded":"2023-10-09T06:26:20.678714+0000","last_fullsized":"2023-10-09T06:26:20.678714+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":580,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665842+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:26:18.665842+0000","last_peered":"2023-10-09T06:26:18.665842+0000","last_clean":"2023-10-09T06:26:18.665842+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:26:18.665842+0000","last_undegraded":"2023-10-09T06:26:18.665842+0000","last_fullsized":"2023-10-09T06:26:18.665842+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":257,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.019939+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:26:21.019939+0000","last_peered":"2023-10-09T06:26:21.019939+0000","last_clean":"2023-10-09T06:26:21.019939+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:26:21.019939+0000","last_undegraded":"2023-10-09T06:26:21.019939+0000","last_fullsized":"2023-10-09T06:26:21.019939+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":274,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678107+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:26:20.678107+0000","last_peered":"2023-10-09T06:26:20.678107+0000","last_clean":"2023-10-09T06:26:20.678107+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:26:20.678107+0000","last_undegraded":"2023-10-09T06:26:20.678107+0000","last_fullsized":"2023-10-09T06:26:20.678107+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.771220+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:33:52.573214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":278,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.679245+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:26:20.679245+0000","last_peered":"2023-10-09T06:26:20.679245+0000","last_clean":"2023-10-09T06:26:20.679245+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:26:20.679245+0000","last_undegraded":"2023-10-09T06:26:20.679245+0000","last_fullsized":"2023-10-09T06:26:20.679245+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":257,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.679001+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:26:20.679001+0000","last_peered":"2023-10-09T06:26:20.679001+0000","last_clean":"2023-10-09T06:26:20.679001+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:26:20.679001+0000","last_undegraded":"2023-10-09T06:26:20.679001+0000","last_fullsized":"2023-10-09T06:26:20.679001+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":576,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.666396+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:26:18.666396+0000","last_peered":"2023-10-09T06:26:18.666396+0000","last_clean":"2023-10-09T06:26:18.666396+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:26:18.666396+0000","last_undegraded":"2023-10-09T06:26:18.666396+0000","last_fullsized":"2023-10-09T06:26:18.666396+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":255,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772575+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:26:27.772575+0000","last_peered":"2023-10-09T06:26:27.772575+0000","last_clean":"2023-10-09T06:26:27.772575+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:26:27.772575+0000","last_undegraded":"2023-10-09T06:26:27.772575+0000","last_fullsized":"2023-10-09T06:26:27.772575+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":227,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:19.000674+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:26:19.000674+0000","last_peered":"2023-10-09T06:26:19.000674+0000","last_clean":"2023-10-09T06:26:19.000674+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:26:19.000674+0000","last_undegraded":"2023-10-09T06:26:19.000674+0000","last_fullsized":"2023-10-09T06:26:19.000674+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":519,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.024539+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:26:21.024539+0000","last_peered":"2023-10-09T06:26:21.024539+0000","last_clean":"2023-10-09T06:26:21.024539+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:26:21.024539+0000","last_undegraded":"2023-10-09T06:26:21.024539+0000","last_fullsized":"2023-10-09T06:26:21.024539+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":279,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958221+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:26:18.958221+0000","last_peered":"2023-10-09T06:26:18.958221+0000","last_clean":"2023-10-09T06:26:18.958221+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:26:18.958221+0000","last_undegraded":"2023-10-09T06:26:18.958221+0000","last_fullsized":"2023-10-09T06:26:18.958221+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":207,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773768+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:26:27.773768+0000","last_peered":"2023-10-09T06:26:27.773768+0000","last_clean":"2023-10-09T06:26:27.773768+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:26:27.773768+0000","last_undegraded":"2023-10-09T06:26:27.773768+0000","last_fullsized":"2023-10-09T06:26:27.773768+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":222,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958379+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:26:18.958379+0000","last_peered":"2023-10-09T06:26:18.958379+0000","last_clean":"2023-10-09T06:26:18.958379+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:26:18.958379+0000","last_undegraded":"2023-10-09T06:26:18.958379+0000","last_fullsized":"2023-10-09T06:26:18.958379+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":10,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T06:26:28.777686+0000","last_change":"2023-10-09T06:26:28.773695+0000","last_active":"2023-10-09T06:26:28.777686+0000","last_peered":"2023-10-09T06:26:28.777686+0000","last_clean":"2023-10-09T06:26:28.777686+0000","last_became_active":"2023-10-09T06:26:28.773467+0000","last_became_peered":"2023-10-09T06:26:28.773467+0000","last_unstale":"2023-10-09T06:26:28.777686+0000","last_undegraded":"2023-10-09T06:26:28.777686+0000","last_fullsized":"2023-10-09T06:26:28.777686+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:33.681958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":581,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664463+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:26:18.664463+0000","last_peered":"2023-10-09T06:26:18.664463+0000","last_clean":"2023-10-09T06:26:18.664463+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:26:18.664463+0000","last_undegraded":"2023-10-09T06:26:18.664463+0000","last_fullsized":"2023-10-09T06:26:18.664463+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":153,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958456+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:26:18.958456+0000","last_peered":"2023-10-09T06:26:18.958456+0000","last_clean":"2023-10-09T06:26:18.958456+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:26:18.958456+0000","last_undegraded":"2023-10-09T06:26:18.958456+0000","last_fullsized":"2023-10-09T06:26:18.958456+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":258,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678593+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:26:20.678593+0000","last_peered":"2023-10-09T06:26:20.678593+0000","last_clean":"2023-10-09T06:26:20.678593+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:26:20.678593+0000","last_undegraded":"2023-10-09T06:26:20.678593+0000","last_fullsized":"2023-10-09T06:26:20.678593+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:19.000756+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:26:19.000756+0000","last_peered":"2023-10-09T06:26:19.000756+0000","last_clean":"2023-10-09T06:26:19.000756+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:26:19.000756+0000","last_undegraded":"2023-10-09T06:26:19.000756+0000","last_fullsized":"2023-10-09T06:26:19.000756+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":10,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T06:26:28.772338+0000","last_change":"2023-10-09T06:26:28.772110+0000","last_active":"2023-10-09T06:26:28.772338+0000","last_peered":"2023-10-09T06:26:28.772338+0000","last_clean":"2023-10-09T06:26:28.772338+0000","last_became_active":"2023-10-09T06:26:28.771837+0000","last_became_peered":"2023-10-09T06:26:28.771837+0000","last_unstale":"2023-10-09T06:26:28.772338+0000","last_undegraded":"2023-10-09T06:26:28.772338+0000","last_fullsized":"2023-10-09T06:26:28.772338+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:42.271885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":589,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.009888+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:26:21.009888+0000","last_peered":"2023-10-09T06:26:21.009888+0000","last_clean":"2023-10-09T06:26:21.009888+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:26:21.009888+0000","last_undegraded":"2023-10-09T06:26:21.009888+0000","last_fullsized":"2023-10-09T06:26:21.009888+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":262,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.666003+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:26:18.666003+0000","last_peered":"2023-10-09T06:26:18.666003+0000","last_clean":"2023-10-09T06:26:18.666003+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:26:18.666003+0000","last_undegraded":"2023-10-09T06:26:18.666003+0000","last_fullsized":"2023-10-09T06:26:18.666003+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958497+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:26:18.958497+0000","last_peered":"2023-10-09T06:26:18.958497+0000","last_clean":"2023-10-09T06:26:18.958497+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:26:18.958497+0000","last_undegraded":"2023-10-09T06:26:18.958497+0000","last_fullsized":"2023-10-09T06:26:18.958497+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":265,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:19.000646+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:26:19.000646+0000","last_peered":"2023-10-09T06:26:19.000646+0000","last_clean":"2023-10-09T06:26:19.000646+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:26:19.000646+0000","last_undegraded":"2023-10-09T06:26:19.000646+0000","last_fullsized":"2023-10-09T06:26:19.000646+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":10,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T06:26:28.777570+0000","last_change":"2023-10-09T06:26:28.773784+0000","last_active":"2023-10-09T06:26:28.777570+0000","last_peered":"2023-10-09T06:26:28.777570+0000","last_clean":"2023-10-09T06:26:28.777570+0000","last_became_active":"2023-10-09T06:26:28.773431+0000","last_became_peered":"2023-10-09T06:26:28.773431+0000","last_unstale":"2023-10-09T06:26:28.777570+0000","last_undegraded":"2023-10-09T06:26:28.777570+0000","last_fullsized":"2023-10-09T06:26:28.777570+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:54.972414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":229,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665916+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:26:18.665916+0000","last_peered":"2023-10-09T06:26:18.665916+0000","last_clean":"2023-10-09T06:26:18.665916+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:26:18.665916+0000","last_undegraded":"2023-10-09T06:26:18.665916+0000","last_fullsized":"2023-10-09T06:26:18.665916+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":570,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010483+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:26:21.010483+0000","last_peered":"2023-10-09T06:26:21.010483+0000","last_clean":"2023-10-09T06:26:21.010483+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:26:21.010483+0000","last_undegraded":"2023-10-09T06:26:21.010483+0000","last_fullsized":"2023-10-09T06:26:21.010483+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":225,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772944+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:26:27.772944+0000","last_peered":"2023-10-09T06:26:27.772944+0000","last_clean":"2023-10-09T06:26:27.772944+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:26:27.772944+0000","last_undegraded":"2023-10-09T06:26:27.772944+0000","last_fullsized":"2023-10-09T06:26:27.772944+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.028982+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:26:21.028982+0000","last_peered":"2023-10-09T06:26:21.028982+0000","last_clean":"2023-10-09T06:26:21.028982+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:26:21.028982+0000","last_undegraded":"2023-10-09T06:26:21.028982+0000","last_fullsized":"2023-10-09T06:26:21.028982+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":10,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-10-09T06:26:28.778694+0000","last_change":"2023-10-09T06:26:28.778366+0000","last_active":"2023-10-09T06:26:28.778694+0000","last_peered":"2023-10-09T06:26:28.778694+0000","last_clean":"2023-10-09T06:26:28.778694+0000","last_became_active":"2023-10-09T06:26:28.777926+0000","last_became_peered":"2023-10-09T06:26:28.777926+0000","last_unstale":"2023-10-09T06:26:28.778694+0000","last_undegraded":"2023-10-09T06:26:28.778694+0000","last_fullsized":"2023-10-09T06:26:28.778694+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:41.094025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":579,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.958629+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:26:18.958629+0000","last_peered":"2023-10-09T06:26:18.958629+0000","last_clean":"2023-10-09T06:26:18.958629+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:26:18.958629+0000","last_undegraded":"2023-10-09T06:26:18.958629+0000","last_fullsized":"2023-10-09T06:26:18.958629+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":140,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010241+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:26:21.010241+0000","last_peered":"2023-10-09T06:26:21.010241+0000","last_clean":"2023-10-09T06:26:21.010241+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:26:21.010241+0000","last_undegraded":"2023-10-09T06:26:21.010241+0000","last_fullsized":"2023-10-09T06:26:21.010241+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":249,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664434+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:26:18.664434+0000","last_peered":"2023-10-09T06:26:18.664434+0000","last_clean":"2023-10-09T06:26:18.664434+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:26:18.664434+0000","last_undegraded":"2023-10-09T06:26:18.664434+0000","last_fullsized":"2023-10-09T06:26:18.664434+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665094+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:26:18.665094+0000","last_peered":"2023-10-09T06:26:18.665094+0000","last_clean":"2023-10-09T06:26:18.665094+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:26:18.665094+0000","last_undegraded":"2023-10-09T06:26:18.665094+0000","last_fullsized":"2023-10-09T06:26:18.665094+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.751676+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":519,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.020607+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:26:21.020607+0000","last_peered":"2023-10-09T06:26:21.020607+0000","last_clean":"2023-10-09T06:26:21.020607+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:26:21.020607+0000","last_undegraded":"2023-10-09T06:26:21.020607+0000","last_fullsized":"2023-10-09T06:26:21.020607+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":140,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.010605+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:26:21.010605+0000","last_peered":"2023-10-09T06:26:21.010605+0000","last_clean":"2023-10-09T06:26:21.010605+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:26:21.010605+0000","last_undegraded":"2023-10-09T06:26:21.010605+0000","last_fullsized":"2023-10-09T06:26:21.010605+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":278,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:19.000610+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:26:19.000610+0000","last_peered":"2023-10-09T06:26:19.000610+0000","last_clean":"2023-10-09T06:26:19.000610+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:26:19.000610+0000","last_undegraded":"2023-10-09T06:26:19.000610+0000","last_fullsized":"2023-10-09T06:26:19.000610+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":261,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664808+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:26:18.664808+0000","last_peered":"2023-10-09T06:26:18.664808+0000","last_clean":"2023-10-09T06:26:18.664808+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:26:18.664808+0000","last_undegraded":"2023-10-09T06:26:18.664808+0000","last_fullsized":"2023-10-09T06:26:18.664808+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.769490+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:29.341064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":580,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.677115+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:26:20.677115+0000","last_peered":"2023-10-09T06:26:20.677115+0000","last_clean":"2023-10-09T06:26:20.677115+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:26:20.677115+0000","last_undegraded":"2023-10-09T06:26:20.677115+0000","last_fullsized":"2023-10-09T06:26:20.677115+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":288,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.774319+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:26:27.774319+0000","last_peered":"2023-10-09T06:26:27.774319+0000","last_clean":"2023-10-09T06:26:27.774319+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:26:27.774319+0000","last_undegraded":"2023-10-09T06:26:27.774319+0000","last_fullsized":"2023-10-09T06:26:27.774319+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":255,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:19.000723+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:26:19.000723+0000","last_peered":"2023-10-09T06:26:19.000723+0000","last_clean":"2023-10-09T06:26:19.000723+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:26:19.000723+0000","last_undegraded":"2023-10-09T06:26:19.000723+0000","last_fullsized":"2023-10-09T06:26:19.000723+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":277,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772494+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:26:27.772494+0000","last_peered":"2023-10-09T06:26:27.772494+0000","last_clean":"2023-10-09T06:26:27.772494+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:26:27.772494+0000","last_undegraded":"2023-10-09T06:26:27.772494+0000","last_fullsized":"2023-10-09T06:26:27.772494+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-10-09T06:26:27.751676+0000","last_change":"2023-10-09T06:26:27.767426+0000","last_active":"2023-10-09T06:26:27.751676+0000","last_peered":"2023-10-09T06:26:27.751676+0000","last_clean":"2023-10-09T06:26:27.751676+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:26:27.751676+0000","last_undegraded":"2023-10-09T06:26:27.751676+0000","last_fullsized":"2023-10-09T06:26:27.751676+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:22.215884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,7],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":602,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.773642+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:26:27.773642+0000","last_peered":"2023-10-09T06:26:27.773642+0000","last_clean":"2023-10-09T06:26:27.773642+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:26:27.773642+0000","last_undegraded":"2023-10-09T06:26:27.773642+0000","last_fullsized":"2023-10-09T06:26:27.773642+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":281,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:21.028029+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:26:21.028029+0000","last_peered":"2023-10-09T06:26:21.028029+0000","last_clean":"2023-10-09T06:26:21.028029+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:26:21.028029+0000","last_undegraded":"2023-10-09T06:26:21.028029+0000","last_fullsized":"2023-10-09T06:26:21.028029+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":253,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664964+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:26:18.664964+0000","last_peered":"2023-10-09T06:26:18.664964+0000","last_clean":"2023-10-09T06:26:18.664964+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:26:18.664964+0000","last_undegraded":"2023-10-09T06:26:18.664964+0000","last_fullsized":"2023-10-09T06:26:18.664964+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.679408+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:26:20.679408+0000","last_peered":"2023-10-09T06:26:20.679408+0000","last_clean":"2023-10-09T06:26:20.679408+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:26:20.679408+0000","last_undegraded":"2023-10-09T06:26:20.679408+0000","last_fullsized":"2023-10-09T06:26:20.679408+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":255,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.664765+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:26:18.664765+0000","last_peered":"2023-10-09T06:26:18.664765+0000","last_clean":"2023-10-09T06:26:18.664765+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:26:18.664765+0000","last_undegraded":"2023-10-09T06:26:18.664765+0000","last_fullsized":"2023-10-09T06:26:18.664765+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":278,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.666146+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:26:18.666146+0000","last_peered":"2023-10-09T06:26:18.666146+0000","last_clean":"2023-10-09T06:26:18.666146+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:26:18.666146+0000","last_undegraded":"2023-10-09T06:26:18.666146+0000","last_fullsized":"2023-10-09T06:26:18.666146+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":102,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678314+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:26:20.678314+0000","last_peered":"2023-10-09T06:26:20.678314+0000","last_clean":"2023-10-09T06:26:20.678314+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:26:20.678314+0000","last_undegraded":"2023-10-09T06:26:20.678314+0000","last_fullsized":"2023-10-09T06:26:20.678314+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":284,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678555+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:26:20.678555+0000","last_peered":"2023-10-09T06:26:20.678555+0000","last_clean":"2023-10-09T06:26:20.678555+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:26:20.678555+0000","last_undegraded":"2023-10-09T06:26:20.678555+0000","last_fullsized":"2023-10-09T06:26:20.678555+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":102,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.678465+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:26:20.678465+0000","last_peered":"2023-10-09T06:26:20.678465+0000","last_clean":"2023-10-09T06:26:20.678465+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:26:20.678465+0000","last_undegraded":"2023-10-09T06:26:20.678465+0000","last_fullsized":"2023-10-09T06:26:20.678465+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.677646+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:26:20.677646+0000","last_peered":"2023-10-09T06:26:20.677646+0000","last_clean":"2023-10-09T06:26:20.677646+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:26:20.677646+0000","last_undegraded":"2023-10-09T06:26:20.677646+0000","last_fullsized":"2023-10-09T06:26:20.677646+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":256,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.774105+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:26:27.774105+0000","last_peered":"2023-10-09T06:26:27.774105+0000","last_clean":"2023-10-09T06:26:27.774105+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:26:27.774105+0000","last_undegraded":"2023-10-09T06:26:27.774105+0000","last_fullsized":"2023-10-09T06:26:27.774105+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":271,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T06:26:27.772961+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:26:27.772961+0000","last_peered":"2023-10-09T06:26:27.772961+0000","last_clean":"2023-10-09T06:26:27.772961+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:26:27.772961+0000","last_undegraded":"2023-10-09T06:26:27.772961+0000","last_fullsized":"2023-10-09T06:26:27.772961+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":268,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-10-09T06:26:18.665822+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:26:18.665822+0000","last_peered":"2023-10-09T06:26:18.665822+0000","last_clean":"2023-10-09T06:26:18.665822+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:26:18.665822+0000","last_undegraded":"2023-10-09T06:26:18.665822+0000","last_fullsized":"2023-10-09T06:26:18.665822+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":257,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T06:26:20.677610+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:26:20.677610+0000","last_peered":"2023-10-09T06:26:20.677610+0000","last_clean":"2023-10-09T06:26:20.677610+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:26:20.677610+0000","last_undegraded":"2023-10-09T06:26:20.677610+0000","last_fullsized":"2023-10-09T06:26:20.677610+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":30,"acting":30,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234350,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907528,"kb_used_data":3200,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":92837624,"statfs":{"total":95995035648,"available":95065726976,"internally_reserved":0,"allocated":3276800,"data_stored":2494079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2149999999999999,"5min":8.8960000000000008,"15min":3.8580000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":89.027000000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":7.0759999999999996,"5min":9.2319999999999993,"15min":3.8980000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.218},"max":{"1min":89.135999999999996,"5min":219.399,"15min":219.399},"last":0.52000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":4.1980000000000004,"5min":3.3839999999999999,"15min":2.8479999999999999},"min":{"1min":0.25,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":49.808,"5min":141.44300000000001,"15min":141.44300000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":4.2930000000000001,"5min":3.835,"15min":2.6619999999999999},"min":{"1min":0.33900000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":49.476999999999997,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.56299999999999994}]},{"osd":2,"last update":"Mon Oct 9 06:25:32 2023","interfaces":[{"interface":"back","average":{"1min":12.579000000000001,"5min":12.257999999999999,"15min":5.2720000000000002},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":92.293000000000006,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":12.222,"5min":12.436,"15min":5.4109999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.727999999999994,"5min":215.869,"15min":215.869},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.359,"15min":1.627},"min":{"1min":0.35599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4710000000000001,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.443},{"interface":"front","average":{"1min":1.6739999999999999,"5min":2.4409999999999998,"15min":1.669},"min":{"1min":0.40500000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6760000000000002,"5min":51.133000000000003,"15min":51.133000000000003},"last":2.8149999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":2.0870000000000002,"15min":1.694},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.3620000000000001,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2,"15min":1.419},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9729999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.169,"15min":1.7230000000000001},"min":{"1min":0.33900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.952,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.151,"15min":1.45},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":4.3280000000000003,"5min":5.7130000000000001,"15min":48.774999999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.458,"15min":1.3580000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7169999999999996,"5min":8.4890000000000008,"15min":11.878},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.2310000000000001,"15min":1.216},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33000000000000002},"max":{"1min":3.0539999999999998,"5min":9.4900000000000002,"15min":12.141999999999999},"last":0.48199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495987,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906548,"kb_used_data":3436,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":92838604,"statfs":{"total":95995035648,"available":95066730496,"internally_reserved":0,"allocated":3518464,"data_stored":2748449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":3.7690000000000001,"15min":3.0790000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":4.2560000000000002,"5min":110.926,"15min":110.926},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.2869999999999999,"5min":3.7480000000000002,"15min":3.2669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2859999999999996,"5min":110.911,"15min":110.911},"last":0.60499999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.7669999999999999,"15min":3.456},"min":{"1min":0.308,"5min":0.215,"15min":0.215},"max":{"1min":4.4939999999999998,"5min":66.796999999999997,"15min":114.202},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.244,"5min":2.4889999999999999,"15min":3.2810000000000001},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.7389999999999999,"5min":66.704999999999998,"15min":114.179},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":4.0890000000000004,"15min":2.774},"min":{"1min":0.34799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":52.609999999999999,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.56699999999999995},{"interface":"front","average":{"1min":5.1059999999999999,"5min":3.9119999999999999,"15min":2.6669999999999998},"min":{"1min":0.29899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":52.575000000000003,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.25,"15min":2.02},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":6.9290000000000003,"5min":110.358,"15min":110.358},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.804,"5min":3.27,"15min":2.069},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.9359999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.1299999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.1890000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":0.38},{"interface":"front","average":{"1min":1.03,"5min":1.474,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":4.2119999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.456,"15min":1.3640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.2460000000000004,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.499},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.488,"15min":1.407},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.8920000000000003,"5min":24.401,"15min":24.401},"last":0.36399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:05 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":2.3639999999999999,"15min":1.536},"min":{"1min":0.35399999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":4.7610000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.115,"5min":2.343,"15min":1.732},"min":{"1min":0.371,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":4.7510000000000003,"5min":100.995,"15min":100.995},"last":0.44600000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402723,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565352,"kb_used_data":74144,"kb_used_omap":4,"kb_used_meta":491195,"kb_avail":93179800,"statfs":{"total":95995035648,"available":95416115200,"internally_reserved":0,"allocated":75923456,"data_stored":74633324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":502984590},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5919999999999996,"5min":6.9119999999999999,"15min":4.032},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":102.471,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":7.7779999999999996,"5min":7.9870000000000001,"15min":4.5599999999999996},"min":{"1min":0.20399999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":102.054,"5min":145.982,"15min":149.02699999999999},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:25:34 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":5.6630000000000003,"15min":3.1059999999999999},"min":{"1min":0.26000000000000001,"5min":0.19400000000000001,"15min":0.193},"max":{"1min":8.4030000000000005,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.3530000000000002,"5min":6.7800000000000002,"15min":3.863},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.219},"max":{"1min":9.8859999999999992,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:26:06 2023","interfaces":[{"interface":"back","average":{"1min":4.2439999999999998,"5min":4.0350000000000001,"15min":2.585},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":16.047999999999998,"5min":77.331999999999994,"15min":153.18199999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.6549999999999998,"15min":1.9319999999999999},"min":{"1min":0.23799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.21,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.49199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":5.266,"15min":3.7469999999999999},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.1760000000000002,"5min":99.391000000000005,"15min":199.43799999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.6100000000000001,"5min":5.1180000000000003,"15min":3.6659999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.3170000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":0.72699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.5859999999999999,"15min":2.5499999999999998},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7169999999999996,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.9890000000000001,"5min":3.5169999999999999,"15min":2.911},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":6.7789999999999999,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.60199999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":4.1269999999999998,"15min":2.948},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":6.6799999999999997,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":2.3559999999999999,"5min":3.8130000000000002,"15min":2.7429999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9290000000000003,"5min":77.378,"15min":77.378},"last":0.63800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:25 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":3.5289999999999999,"15min":2.706},"min":{"1min":0.36099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.2069999999999999,"5min":77.456999999999994,"15min":77.456999999999994},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.105,"5min":2.9929999999999999,"15min":2.2040000000000002},"min":{"1min":0.35599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.6900000000000004,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.71099999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024156,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868636,"kb_used_data":166740,"kb_used_omap":5,"kb_used_meta":701882,"kb_avail":92876516,"statfs":{"total":95995035648,"available":95105552384,"internally_reserved":0,"allocated":170741760,"data_stored":169231696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":718727802},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.0139999999999998,"15min":2.8359999999999999},"min":{"1min":0.20699999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":9.1820000000000004,"5min":13.916,"15min":185.17400000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.7599999999999998,"5min":2.3879999999999999,"15min":3.3599999999999999},"min":{"1min":0.251,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":9.3369999999999997,"5min":13.945,"15min":186.691},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:25:36 2023","interfaces":[{"interface":"back","average":{"1min":3.29,"5min":2.4620000000000002,"15min":2.3010000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.17799999999999999},"max":{"1min":10.255000000000001,"5min":13.211,"15min":17.588999999999999},"last":0.309},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.6970000000000001,"15min":2.4049999999999998},"min":{"1min":0.33300000000000002,"5min":0.21199999999999999,"15min":0.20699999999999999},"max":{"1min":14.974,"5min":14.974,"15min":21.198},"last":0.254}]},{"osd":3,"last update":"Mon Oct 9 06:25:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2010000000000001,"5min":3.3999999999999999,"15min":2.9140000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2989999999999995,"5min":52.683,"15min":185.89400000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":2.9950000000000001,"5min":3.4289999999999998,"15min":3.0259999999999998},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":9.3759999999999994,"5min":52.941000000000003,"15min":186.72900000000001},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5860000000000001,"15min":1.516},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.024,"5min":7.9660000000000002,"15min":13.244},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.833,"15min":1.9410000000000001},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":4.9770000000000003,"5min":8.4890000000000008,"15min":12.752000000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.0990000000000002,"15min":1.9239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8450000000000002,"5min":62.984999999999999,"15min":62.984999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.7090000000000001,"15min":2.504},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":5.8650000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.282,"15min":2.319},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.1360000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.498,"5min":1.6299999999999999,"15min":1.8400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.819,"5min":7.9980000000000002,"15min":13.315},"last":0.432}]},{"osd":7,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.5099999999999998,"15min":2.1920000000000002},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":4.7279999999999998,"5min":62.262,"15min":62.262},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.202,"5min":2.9910000000000001,"15min":2.5619999999999998},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":7.1059999999999999,"5min":62.229999999999997,"15min":62.229999999999997},"last":0.47799999999999998}]}]},{"osd":2,"up_from":128,"seq":549755814025,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438352,"kb_used_data":3656,"kb_used_omap":7,"kb_used_meta":434680,"kb_avail":93306800,"statfs":{"total":95995035648,"available":95546163200,"internally_reserved":0,"allocated":3743744,"data_stored":2473605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":445112401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":10.217000000000001,"5min":4.6269999999999998,"15min":3.968},"min":{"1min":0.22800000000000001,"5min":0.157,"15min":0.157},"max":{"1min":150.32599999999999,"5min":150.32599999999999,"15min":163.40100000000001},"last":0.23499999999999999},{"interface":"front","average":{"1min":3.1629999999999998,"5min":3.4260000000000002,"15min":3.8820000000000001},"min":{"1min":0.20899999999999999,"5min":0.182,"15min":0.182},"max":{"1min":21.472000000000001,"5min":110.66500000000001,"15min":163.363},"last":0.35299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.8579999999999997,"15min":3.6680000000000001},"min":{"1min":0.255,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":7.9710000000000001,"5min":205.148,"15min":222.50299999999999},"last":0.314},{"interface":"front","average":{"1min":2.2090000000000001,"5min":4.8280000000000003,"15min":3.984},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20999999999999999},"max":{"1min":11.811,"5min":205.267,"15min":222.19399999999999},"last":0.28899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":2.8780000000000001,"15min":1.8140000000000001},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":13.664999999999999,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":3.0150000000000001,"5min":2.9369999999999998,"15min":1.8700000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.728,"5min":69.906999999999996,"15min":81.061999999999998},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":2.952,"15min":2.2240000000000002},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":52.188000000000002,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.61499999999999999},{"interface":"front","average":{"1min":3.6179999999999999,"5min":3.282,"15min":2.4910000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":52.167999999999999,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.57499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.472,"15min":1.534},"min":{"1min":0.34000000000000002,"5min":0.317,"15min":0.309},"max":{"1min":5.508,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.256,"5min":1.7549999999999999,"15min":2.0990000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":6.9349999999999996,"5min":10.548,"15min":10.548},"last":0.60299999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":3.7839999999999998,"15min":2.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":55.392000000000003,"5min":138.291,"15min":138.291},"last":0.52700000000000002},{"interface":"front","average":{"1min":4.8150000000000004,"5min":3.4910000000000001,"15min":2.206},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":55.146999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 06:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.9269999999999996,"5min":2.2709999999999999,"15min":1.863},"min":{"1min":0.40699999999999997,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":55.851999999999997,"5min":55.851999999999997,"15min":55.851999999999997},"last":0.65600000000000003},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.1890000000000001,"15min":2.395},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":55.731000000000002,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.58399999999999996}]}]},{"osd":3,"up_from":149,"seq":639950127223,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":442692,"kb_used_data":34108,"kb_used_omap":2,"kb_used_meta":408573,"kb_avail":93302460,"statfs":{"total":95995035648,"available":95541719040,"internally_reserved":0,"allocated":34926592,"data_stored":33734478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":418378805},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":7.8659999999999997,"15min":3.262},"min":{"1min":0.19800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":10.367000000000001,"5min":187.852,"15min":187.852},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.23,"5min":8.4719999999999995,"15min":3.7200000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":4.6539999999999999,"5min":187.923,"15min":187.923},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":6.0439999999999996,"15min":2.9500000000000002},"min":{"1min":0.255,"5min":0.246,"15min":0.223},"max":{"1min":11.367000000000001,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.978,"5min":3.673,"15min":1.9379999999999999},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.8810000000000002,"5min":67.028000000000006,"15min":67.028000000000006},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":5.2350000000000003,"15min":2.649},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.57,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.3210000000000002,"5min":4.8769999999999998,"15min":2.1349999999999998},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5250000000000004,"5min":187.79499999999999,"15min":187.79499999999999},"last":0.42899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.544,"15min":1.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":6.7830000000000004,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.2389999999999999,"15min":1.964},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.9649999999999999,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.59899999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":3.2530000000000001,"15min":2.1080000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":3.0059999999999998,"5min":187.833,"15min":187.833},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.0649999999999999,"15min":2.036},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":187.952,"15min":187.952},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.6080000000000001,"15min":2.1560000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6280000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1890000000000001,"15min":1.9910000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8719999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":3.5169999999999999,"15min":1.8320000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":6.8239999999999998,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.242,"5min":3.4580000000000002,"15min":2.093},"min":{"1min":0.41899999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":5.431,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.63800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019259,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032692,"kb_used_data":145900,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":92712460,"statfs":{"total":95995035648,"available":94937559040,"internally_reserved":0,"allocated":149401600,"data_stored":148630864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":3.5310000000000001,"15min":2.2469999999999999},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.7480000000000002,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":3.9089999999999998,"15min":2.6549999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":4.3799999999999999,"5min":76.988,"15min":76.988},"last":0.64100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.98399999999999999,"15min":2.016},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.7519999999999998,"5min":5.0190000000000001,"15min":91.849999999999994},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.381,"5min":1.359,"15min":2.1579999999999999},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":5.0979999999999999,"5min":9.625,"15min":91.885000000000005},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":3.5089999999999999,"15min":2.3359999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":47.125,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":5.9249999999999998,"5min":3.794,"15min":2.5379999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":47.151000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":0.59899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.7029999999999998,"15min":2.2879999999999998},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.8360000000000003,"5min":45.591000000000001,"15min":91.697000000000003},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.341,"5min":2.3140000000000001,"15min":2.129},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":3.5230000000000001,"5min":45.548999999999999,"15min":94.356999999999999},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.95799999999999996,"15min":1.1419999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24299999999999999},"max":{"1min":6.2670000000000003,"5min":9.1720000000000006,"15min":10.673},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.91500000000000004,"15min":1.228},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.536,"5min":9.6379999999999999,"15min":12.552},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.83399999999999996,"15min":0.84899999999999998},"min":{"1min":0.29399999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":2.1179999999999999,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1830000000000001,"15min":1.286},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.859,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.7350000000000001,"15min":1.476},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":2.1880000000000002,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.63},{"interface":"front","average":{"1min":0.876,"5min":1.706,"15min":1.488},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":3.8450000000000002,"5min":62.573,"15min":62.573},"last":0.65600000000000003}]}]},{"osd":5,"up_from":0,"seq":163208757624,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:26:31.622 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:26:31.656 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:31.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:31.761 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:31.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[212750]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:31.785 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:31.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:31.786 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[220921]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:31.786 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:31.786 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:31.788 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:31.810 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:31.810 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:31.886 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:31.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:31 smithi150 ceph-mon[194178]: from='client.45959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:31.990 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:32.017 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:32.082 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:26:32.106 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:32] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T06:26:32.316 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:32.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:32.421 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:32.421 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:32.451 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:32.462 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:26:32.507 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:32.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:32.533 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:32.612 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:32.638 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:32.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:32.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:32.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:32.811 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:32.822 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:32.869 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:32.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:32.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:32.974 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:33.001 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:33.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:32 smithi107 ceph-mon[212750]: pgmap v922: 99 pgs: 6 creating+peering, 5 unknown, 88 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:33.032 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:32 smithi107 ceph-mon[220921]: pgmap v922: 99 pgs: 6 creating+peering, 5 unknown, 88 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:33.039 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:32 smithi150 ceph-mon[194178]: pgmap v922: 99 pgs: 6 creating+peering, 5 unknown, 88 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:33.046 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:33.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:33.149 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:33.178 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:33.474 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:33.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:33.578 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:33.609 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:33.704 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:33.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:33.764 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:26:33.789 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-10-09T06:26:33.807 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:33.837 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1556798204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1556798204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:34.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1556798204' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:34.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:26:34.188 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:34.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:34.292 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:34.323 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:34.369 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:34.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:34.379 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:34.380 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:34.473 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:34.501 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:34.597 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:34.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:34.701 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:34.727 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:34.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[212750]: pgmap v923: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[212750]: failed to encode map e300 with expected crc 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[212750]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[220921]: pgmap v923: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[220921]: failed to encode map e300 with expected crc 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[220921]: failed to encode map e300 with expected crc 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:26:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:34 smithi107 ceph-mon[220921]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T06:26:35.028 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:34 smithi150 ceph-mon[194178]: pgmap v923: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:35.029 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:34 smithi150 ceph-mon[194178]: failed to encode map e300 with expected crc 2023-10-09T06:26:35.029 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:34 smithi150 ceph-mon[194178]: failed to encode map e300 with expected crc 2023-10-09T06:26:35.029 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:34 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:26:35.029 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:34 smithi150 ceph-mon[194178]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T06:26:35.038 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:35.045 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:26:35.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:35.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:35.153 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:35.184 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:35.322 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:35.322 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1208,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":300,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":7,"osd_in_since":1696832504,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"creating+peering","count":3}],"num_pgs":99,"num_pools":5,"num_objects":24,"data_bytes":595872,"bytes_used":5246504960,"bytes_avail":666718744576,"bytes_total":671965249536,"inactive_pgs_ratio":0.030303031206130981},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":24,"modified":"2023-10-09T06:22:27.038508+0000","services":{}},"progress_events":{"7960b10a-aba7-4935-a6b3-a5cb11c5e685":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:26:35.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:35.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:35.391 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:35.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:35.468 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:35.531 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:35.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:35.635 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:35.663 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:35.705 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:35.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:35.712 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-10-09T06:26:35.715 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:26:35.715 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:26:35.762 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:26:35.762 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:26:35.762 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_418363 2023-10-09T06:26:35.763 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:26:35.763 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:26:35.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:35.818 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:26:35.874 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:36.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/663533821' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:36.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1961006749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.099 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/663533821' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:36.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1961006749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/663533821' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:26:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1961006749' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:36.220 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:36.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:36.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:36.379 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:36.763 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2590 2574 160.865 160.875 0.00480154 0.00619843 2023-10-09T06:26:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: pgmap v925: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:26:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:36.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:36 smithi150 ceph-mon[194178]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: pgmap v925: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:26:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[220921]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: pgmap v925: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-09T06:26:37.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:26:37.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:26:37.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:36 smithi107 ceph-mon[212750]: failed to encode map e301 with expected crc 2023-10-09T06:26:37.269 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:37.270 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:37.763 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 5001 4985 155.769 150.688 0.00299252 0.00624107 2023-10-09T06:26:37.921 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:37.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:37.937 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T06:26:37.937 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:26:38.025 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:38.073 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:38.137 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:38.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:38.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:38.245 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:38.314 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:38.418 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:38.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:38.481 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:26:38.528 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:38.599 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:38.695 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:38.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:38.707 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:38.760 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:26:38.764 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 15 6652 6637 138.19 103.25 0.00395591 0.00722444 2023-10-09T06:26:38.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:38.814 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:26:38.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:26:38.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:38 smithi107 ceph-mon[212750]: pgmap v927: 99 pgs: 99 active+clean; 55 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 7.4 MiB/s wr, 117 op/s 2023-10-09T06:26:38.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3961685948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:38.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:38 smithi107 ceph-mon[220921]: pgmap v927: 99 pgs: 99 active+clean; 55 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 7.4 MiB/s wr, 117 op/s 2023-10-09T06:26:38.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3961685948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:38.867 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:38.887 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:26:38.925 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:26:38.948 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:38.964 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:26:39.001 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:26:39.012 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:39.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:39.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:39.037 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:26:39.048 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.048 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.048 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.048 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.049 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.049 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.049 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.049 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:26:39.118 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:38 smithi150 ceph-mon[194178]: pgmap v927: 99 pgs: 99 active+clean; 55 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 7.4 MiB/s wr, 117 op/s 2023-10-09T06:26:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3961685948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:39.186 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:39.413 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:39.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:39.519 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:39.576 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:39.768 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 9333 9317 145.409 167.5 0.00453787 0.00686776 2023-10-09T06:26:40.007 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:26:40.584 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:40.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:40.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:40.689 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:40.738 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:40.767 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 12089 12073 150.769 172.25 0.00969559 0.00662507 2023-10-09T06:26:40.880 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:40.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:40 smithi150 ceph-mon[194178]: pgmap v928: 99 pgs: 99 active+clean; 285 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-09T06:26:40.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:40.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:40.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:40 smithi107 ceph-mon[212750]: pgmap v928: 99 pgs: 99 active+clean; 285 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-09T06:26:40.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:40 smithi107 ceph-mon[220921]: pgmap v928: 99 pgs: 99 active+clean; 285 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-09T06:26:40.986 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:41.021 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:41.117 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:41.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:41.223 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:41.234 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:26:41.234 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:26:41.234 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:26:41.234 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:26:41.234 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:26:41.269 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:41.483 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:41.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:41.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:41.586 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:41.624 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:41.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:41.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:41.768 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 14549 14533 151.261 153.75 0.00873924 0.00660578 2023-10-09T06:26:41.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:41.828 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:41.891 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:41.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:41.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:41.995 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:42.021 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:42.062 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:42.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:42.165 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:42.200 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:42.264 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:42.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:42.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:42.369 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:42] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T06:26:42.435 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:42.555 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:42.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:42.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:42.589 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:42.663 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:42.691 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:26:42.703 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:42.768 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 16408 16392 146.252 116.188 0.00967963 0.006829 2023-10-09T06:26:42.788 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:42.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:42.819 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:42 smithi107 ceph-mon[212750]: pgmap v929: 99 pgs: 99 active+clean; 327 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s 2023-10-09T06:26:42.819 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:42.819 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:42 smithi107 ceph-mon[220921]: pgmap v929: 99 pgs: 99 active+clean; 327 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s 2023-10-09T06:26:42.819 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:42.895 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:42.936 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:42 smithi150 ceph-mon[194178]: pgmap v929: 99 pgs: 99 active+clean; 327 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s 2023-10-09T06:26:42.936 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:42.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:43.005 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:43.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:43.111 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:43.162 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:43.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:43.235 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:43.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:43.342 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:43.433 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:43.533 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:43.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:43.641 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:43.711 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:43.769 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 18516 18500 144.408 131.75 0.0068982 0.00691761 2023-10-09T06:26:43.809 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:43.812 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:43.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:43.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:43.824 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:26:43.925 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:43.965 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:44.028 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:44.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:44.131 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:44.168 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:44.421 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:44.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:44.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:44.466 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:26:44.525 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:44.559 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:44.770 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 19625 19609 136.067 69.3125 0.00438652 0.00734414 2023-10-09T06:26:44.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:44 smithi107 ceph-mon[212750]: pgmap v930: 99 pgs: 99 active+clean; 694 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T06:26:44.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:44 smithi107 ceph-mon[212750]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:45.018 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:45.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:45.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:44 smithi107 ceph-mon[220921]: pgmap v930: 99 pgs: 99 active+clean; 694 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T06:26:45.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:44 smithi107 ceph-mon[220921]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:45.122 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:45.143 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:44 smithi150 ceph-mon[194178]: pgmap v930: 99 pgs: 99 active+clean; 694 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T06:26:45.143 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:44 smithi150 ceph-mon[194178]: from='client.55900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:45.166 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:45.342 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:45.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:45.428 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:26:45.428 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:26:45.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:26:45.429 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [] 2023-10-09T06:26:45.429 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:26:45.452 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:45.513 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:45.678 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:45.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:45.774 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 21491 21475 134.1 116.625 0.0185359 0.00745126 2023-10-09T06:26:45.782 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:45.845 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:45.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:45 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:45.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:45 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:45.960 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":930,"stamp":"2023-10-09T06:26:43.145576+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":727193504,"num_objects":11111,"num_object_clones":0,"num_object_copies":33333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":709597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":338010,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12913,"ondisk_log_size":12913,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7075336,"kb_used_data":2361104,"kb_used_omap":20,"kb_used_meta":4714155,"kb_avail":649140728,"statfs":{"total":671965249536,"available":664720105472,"internally_reserved":0,"allocated":2417770496,"data_stored":2410132385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4827295374},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":726597632,"num_objects":11087,"num_object_clones":0,"num_object_copies":33261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11087,"num_write_kb":709568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.364269"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786883+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:26:33.786883+0000","last_peered":"2023-10-09T06:26:33.786883+0000","last_clean":"2023-10-09T06:26:33.786883+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:26:33.786883+0000","last_undegraded":"2023-10-09T06:26:33.786883+0000","last_fullsized":"2023-10-09T06:26:33.786883+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816573+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:26:33.816573+0000","last_peered":"2023-10-09T06:26:33.816573+0000","last_clean":"2023-10-09T06:26:33.816573+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:26:33.816573+0000","last_undegraded":"2023-10-09T06:26:33.816573+0000","last_fullsized":"2023-10-09T06:26:33.816573+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":273,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817290+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:26:33.817290+0000","last_peered":"2023-10-09T06:26:33.817290+0000","last_clean":"2023-10-09T06:26:33.817290+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:26:33.817290+0000","last_undegraded":"2023-10-09T06:26:33.817290+0000","last_fullsized":"2023-10-09T06:26:33.817290+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":273,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783823+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:26:33.783823+0000","last_peered":"2023-10-09T06:26:33.783823+0000","last_clean":"2023-10-09T06:26:33.783823+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:26:33.783823+0000","last_undegraded":"2023-10-09T06:26:33.783823+0000","last_fullsized":"2023-10-09T06:26:33.783823+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815817+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:26:33.815817+0000","last_peered":"2023-10-09T06:26:33.815817+0000","last_clean":"2023-10-09T06:26:33.815817+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:26:33.815817+0000","last_undegraded":"2023-10-09T06:26:33.815817+0000","last_fullsized":"2023-10-09T06:26:33.815817+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814944+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:26:33.814944+0000","last_peered":"2023-10-09T06:26:33.814944+0000","last_clean":"2023-10-09T06:26:33.814944+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:26:33.814944+0000","last_undegraded":"2023-10-09T06:26:33.814944+0000","last_fullsized":"2023-10-09T06:26:33.814944+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.784272+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:26:33.784272+0000","last_peered":"2023-10-09T06:26:33.784272+0000","last_clean":"2023-10-09T06:26:33.784272+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:26:33.784272+0000","last_undegraded":"2023-10-09T06:26:33.784272+0000","last_fullsized":"2023-10-09T06:26:33.784272+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815870+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:26:33.815870+0000","last_peered":"2023-10-09T06:26:33.815870+0000","last_clean":"2023-10-09T06:26:33.815870+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:26:33.815870+0000","last_undegraded":"2023-10-09T06:26:33.815870+0000","last_fullsized":"2023-10-09T06:26:33.815870+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":229,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786405+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:26:33.786405+0000","last_peered":"2023-10-09T06:26:33.786405+0000","last_clean":"2023-10-09T06:26:33.786405+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:26:33.786405+0000","last_undegraded":"2023-10-09T06:26:33.786405+0000","last_fullsized":"2023-10-09T06:26:33.786405+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":237,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815018+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:26:33.815018+0000","last_peered":"2023-10-09T06:26:33.815018+0000","last_clean":"2023-10-09T06:26:33.815018+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:26:33.815018+0000","last_undegraded":"2023-10-09T06:26:33.815018+0000","last_fullsized":"2023-10-09T06:26:33.815018+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":150,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786482+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:26:33.786482+0000","last_peered":"2023-10-09T06:26:33.786482+0000","last_clean":"2023-10-09T06:26:33.786482+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:26:33.786482+0000","last_undegraded":"2023-10-09T06:26:33.786482+0000","last_fullsized":"2023-10-09T06:26:33.786482+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":232,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817665+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:26:33.817665+0000","last_peered":"2023-10-09T06:26:33.817665+0000","last_clean":"2023-10-09T06:26:33.817665+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:26:33.817665+0000","last_undegraded":"2023-10-09T06:26:33.817665+0000","last_fullsized":"2023-10-09T06:26:33.817665+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":307,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787083+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:26:33.787083+0000","last_peered":"2023-10-09T06:26:33.787083+0000","last_clean":"2023-10-09T06:26:33.787083+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:26:33.787083+0000","last_undegraded":"2023-10-09T06:26:33.787083+0000","last_fullsized":"2023-10-09T06:26:33.787083+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":271,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787232+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:26:33.787232+0000","last_peered":"2023-10-09T06:26:33.787232+0000","last_clean":"2023-10-09T06:26:33.787232+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:26:33.787232+0000","last_undegraded":"2023-10-09T06:26:33.787232+0000","last_fullsized":"2023-10-09T06:26:33.787232+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":150,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786200+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:26:33.786200+0000","last_peered":"2023-10-09T06:26:33.786200+0000","last_clean":"2023-10-09T06:26:33.786200+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:26:33.786200+0000","last_undegraded":"2023-10-09T06:26:33.786200+0000","last_fullsized":"2023-10-09T06:26:33.786200+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816730+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:26:33.816730+0000","last_peered":"2023-10-09T06:26:33.816730+0000","last_clean":"2023-10-09T06:26:33.816730+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:26:33.816730+0000","last_undegraded":"2023-10-09T06:26:33.816730+0000","last_fullsized":"2023-10-09T06:26:33.816730+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":271,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.784155+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:26:33.784155+0000","last_peered":"2023-10-09T06:26:33.784155+0000","last_clean":"2023-10-09T06:26:33.784155+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:26:33.784155+0000","last_undegraded":"2023-10-09T06:26:33.784155+0000","last_fullsized":"2023-10-09T06:26:33.784155+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786363+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:26:33.786363+0000","last_peered":"2023-10-09T06:26:33.786363+0000","last_clean":"2023-10-09T06:26:33.786363+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:26:33.786363+0000","last_undegraded":"2023-10-09T06:26:33.786363+0000","last_fullsized":"2023-10-09T06:26:33.786363+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786076+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:26:33.786076+0000","last_peered":"2023-10-09T06:26:33.786076+0000","last_clean":"2023-10-09T06:26:33.786076+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:26:33.786076+0000","last_undegraded":"2023-10-09T06:26:33.786076+0000","last_fullsized":"2023-10-09T06:26:33.786076+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783979+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:26:33.783979+0000","last_peered":"2023-10-09T06:26:33.783979+0000","last_clean":"2023-10-09T06:26:33.783979+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:26:33.783979+0000","last_undegraded":"2023-10-09T06:26:33.783979+0000","last_fullsized":"2023-10-09T06:26:33.783979+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":289,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814604+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:26:33.814604+0000","last_peered":"2023-10-09T06:26:33.814604+0000","last_clean":"2023-10-09T06:26:33.814604+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:26:33.814604+0000","last_undegraded":"2023-10-09T06:26:33.814604+0000","last_fullsized":"2023-10-09T06:26:33.814604+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816530+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:26:33.816530+0000","last_peered":"2023-10-09T06:26:33.816530+0000","last_clean":"2023-10-09T06:26:33.816530+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:26:33.816530+0000","last_undegraded":"2023-10-09T06:26:33.816530+0000","last_fullsized":"2023-10-09T06:26:33.816530+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"301'872","reported_seq":885,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:41.930486+0000","last_change":"2023-10-09T06:26:28.774987+0000","last_active":"2023-10-09T06:26:41.930486+0000","last_peered":"2023-10-09T06:26:41.930486+0000","last_clean":"2023-10-09T06:26:41.930486+0000","last_became_active":"2023-10-09T06:26:28.774691+0000","last_became_peered":"2023-10-09T06:26:28.774691+0000","last_unstale":"2023-10-09T06:26:41.930486+0000","last_undegraded":"2023-10-09T06:26:41.930486+0000","last_fullsized":"2023-10-09T06:26:41.930486+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":872,"log_dups_size":0,"ondisk_log_size":872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:17.484383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":872,"num_write_kb":55808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":162,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786506+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:26:33.786506+0000","last_peered":"2023-10-09T06:26:33.786506+0000","last_clean":"2023-10-09T06:26:33.786506+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:26:33.786506+0000","last_undegraded":"2023-10-09T06:26:33.786506+0000","last_fullsized":"2023-10-09T06:26:33.786506+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":211,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786098+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:26:33.786098+0000","last_peered":"2023-10-09T06:26:33.786098+0000","last_clean":"2023-10-09T06:26:33.786098+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:26:33.786098+0000","last_undegraded":"2023-10-09T06:26:33.786098+0000","last_fullsized":"2023-10-09T06:26:33.786098+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"301'669","reported_seq":682,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:40.388288+0000","last_change":"2023-10-09T06:26:28.769277+0000","last_active":"2023-10-09T06:26:40.388288+0000","last_peered":"2023-10-09T06:26:40.388288+0000","last_clean":"2023-10-09T06:26:40.388288+0000","last_became_active":"2023-10-09T06:26:28.769053+0000","last_became_peered":"2023-10-09T06:26:28.769053+0000","last_unstale":"2023-10-09T06:26:40.388288+0000","last_undegraded":"2023-10-09T06:26:40.388288+0000","last_fullsized":"2023-10-09T06:26:40.388288+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":669,"log_dups_size":0,"ondisk_log_size":669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:00.039037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43843584,"num_objects":669,"num_object_clones":0,"num_object_copies":2007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":669,"num_write_kb":42816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":148,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786424+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:26:33.786424+0000","last_peered":"2023-10-09T06:26:33.786424+0000","last_clean":"2023-10-09T06:26:33.786424+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:26:33.786424+0000","last_undegraded":"2023-10-09T06:26:33.786424+0000","last_fullsized":"2023-10-09T06:26:33.786424+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786832+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:26:33.786832+0000","last_peered":"2023-10-09T06:26:33.786832+0000","last_clean":"2023-10-09T06:26:33.786832+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:26:33.786832+0000","last_undegraded":"2023-10-09T06:26:33.786832+0000","last_fullsized":"2023-10-09T06:26:33.786832+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"301'351","reported_seq":362,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.166445+0000","last_change":"2023-10-09T06:26:28.778132+0000","last_active":"2023-10-09T06:26:38.166445+0000","last_peered":"2023-10-09T06:26:38.166445+0000","last_clean":"2023-10-09T06:26:38.166445+0000","last_became_active":"2023-10-09T06:26:28.777848+0000","last_became_peered":"2023-10-09T06:26:28.777848+0000","last_unstale":"2023-10-09T06:26:38.166445+0000","last_undegraded":"2023-10-09T06:26:38.166445+0000","last_fullsized":"2023-10-09T06:26:38.166445+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":351,"log_dups_size":0,"ondisk_log_size":351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:31.438186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23003136,"num_objects":351,"num_object_clones":0,"num_object_copies":1053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":351,"num_write_kb":22464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":162,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786364+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:26:33.786364+0000","last_peered":"2023-10-09T06:26:33.786364+0000","last_clean":"2023-10-09T06:26:33.786364+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:26:33.786364+0000","last_undegraded":"2023-10-09T06:26:33.786364+0000","last_fullsized":"2023-10-09T06:26:33.786364+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814829+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:26:33.814829+0000","last_peered":"2023-10-09T06:26:33.814829+0000","last_clean":"2023-10-09T06:26:33.814829+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:26:33.814829+0000","last_undegraded":"2023-10-09T06:26:33.814829+0000","last_fullsized":"2023-10-09T06:26:33.814829+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"301'299","reported_seq":311,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.161925+0000","last_change":"2023-10-09T06:26:28.775173+0000","last_active":"2023-10-09T06:26:38.161925+0000","last_peered":"2023-10-09T06:26:38.161925+0000","last_clean":"2023-10-09T06:26:38.161925+0000","last_became_active":"2023-10-09T06:26:28.774891+0000","last_became_peered":"2023-10-09T06:26:28.774891+0000","last_unstale":"2023-10-09T06:26:38.161925+0000","last_undegraded":"2023-10-09T06:26:38.161925+0000","last_fullsized":"2023-10-09T06:26:38.161925+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":299,"log_dups_size":0,"ondisk_log_size":299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:37.713365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19595264,"num_objects":299,"num_object_clones":0,"num_object_copies":897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":299,"num_write_kb":19136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":193,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816709+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:26:33.816709+0000","last_peered":"2023-10-09T06:26:33.816709+0000","last_clean":"2023-10-09T06:26:33.816709+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:26:33.816709+0000","last_undegraded":"2023-10-09T06:26:33.816709+0000","last_fullsized":"2023-10-09T06:26:33.816709+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":254,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783492+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:26:33.783492+0000","last_peered":"2023-10-09T06:26:33.783492+0000","last_clean":"2023-10-09T06:26:33.783492+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:26:33.783492+0000","last_undegraded":"2023-10-09T06:26:33.783492+0000","last_fullsized":"2023-10-09T06:26:33.783492+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"301'1076","reported_seq":1088,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:42.929093+0000","last_change":"2023-10-09T06:26:28.776488+0000","last_active":"2023-10-09T06:26:42.929093+0000","last_peered":"2023-10-09T06:26:42.929093+0000","last_clean":"2023-10-09T06:26:42.929093+0000","last_became_active":"2023-10-09T06:26:28.776212+0000","last_became_peered":"2023-10-09T06:26:28.776212+0000","last_unstale":"2023-10-09T06:26:42.929093+0000","last_undegraded":"2023-10-09T06:26:42.929093+0000","last_fullsized":"2023-10-09T06:26:42.929093+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1076,"log_dups_size":0,"ondisk_log_size":1076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:12.868016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1076,"num_write_kb":68864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":593,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.785929+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:26:33.785929+0000","last_peered":"2023-10-09T06:26:33.785929+0000","last_clean":"2023-10-09T06:26:33.785929+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:26:33.785929+0000","last_undegraded":"2023-10-09T06:26:33.785929+0000","last_fullsized":"2023-10-09T06:26:33.785929+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":265,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816268+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:26:33.816268+0000","last_peered":"2023-10-09T06:26:33.816268+0000","last_clean":"2023-10-09T06:26:33.816268+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:26:33.816268+0000","last_undegraded":"2023-10-09T06:26:33.816268+0000","last_fullsized":"2023-10-09T06:26:33.816268+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":261,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786988+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:26:33.786988+0000","last_peered":"2023-10-09T06:26:33.786988+0000","last_clean":"2023-10-09T06:26:33.786988+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:26:33.786988+0000","last_undegraded":"2023-10-09T06:26:33.786988+0000","last_fullsized":"2023-10-09T06:26:33.786988+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"301'908","reported_seq":921,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:41.703322+0000","last_change":"2023-10-09T06:26:28.774343+0000","last_active":"2023-10-09T06:26:41.703322+0000","last_peered":"2023-10-09T06:26:41.703322+0000","last_clean":"2023-10-09T06:26:41.703322+0000","last_became_active":"2023-10-09T06:26:28.773068+0000","last_became_peered":"2023-10-09T06:26:28.773068+0000","last_unstale":"2023-10-09T06:26:41.703322+0000","last_undegraded":"2023-10-09T06:26:41.703322+0000","last_fullsized":"2023-10-09T06:26:41.703322+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":908,"log_dups_size":0,"ondisk_log_size":908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:13.256859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59506688,"num_objects":908,"num_object_clones":0,"num_object_copies":2724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":908,"num_write_kb":58112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":585,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814711+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:26:33.814711+0000","last_peered":"2023-10-09T06:26:33.814711+0000","last_clean":"2023-10-09T06:26:33.814711+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:26:33.814711+0000","last_undegraded":"2023-10-09T06:26:33.814711+0000","last_fullsized":"2023-10-09T06:26:33.814711+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":261,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814735+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:26:33.814735+0000","last_peered":"2023-10-09T06:26:33.814735+0000","last_clean":"2023-10-09T06:26:33.814735+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:26:33.814735+0000","last_undegraded":"2023-10-09T06:26:33.814735+0000","last_fullsized":"2023-10-09T06:26:33.814735+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787272+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:26:33.787272+0000","last_peered":"2023-10-09T06:26:33.787272+0000","last_clean":"2023-10-09T06:26:33.787272+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:26:33.787272+0000","last_undegraded":"2023-10-09T06:26:33.787272+0000","last_fullsized":"2023-10-09T06:26:33.787272+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"301'1065","reported_seq":1077,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:42.931701+0000","last_change":"2023-10-09T06:26:28.775744+0000","last_active":"2023-10-09T06:26:42.931701+0000","last_peered":"2023-10-09T06:26:42.931701+0000","last_clean":"2023-10-09T06:26:42.931701+0000","last_became_active":"2023-10-09T06:26:28.775434+0000","last_became_peered":"2023-10-09T06:26:28.775434+0000","last_unstale":"2023-10-09T06:26:42.931701+0000","last_undegraded":"2023-10-09T06:26:42.931701+0000","last_fullsized":"2023-10-09T06:26:42.931701+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1065,"log_dups_size":0,"ondisk_log_size":1065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:30.642594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69795840,"num_objects":1065,"num_object_clones":0,"num_object_copies":3195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1065,"num_write_kb":68160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":282,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786734+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:26:33.786734+0000","last_peered":"2023-10-09T06:26:33.786734+0000","last_clean":"2023-10-09T06:26:33.786734+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:26:33.786734+0000","last_undegraded":"2023-10-09T06:26:33.786734+0000","last_fullsized":"2023-10-09T06:26:33.786734+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":261,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787441+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:26:33.787441+0000","last_peered":"2023-10-09T06:26:33.787441+0000","last_clean":"2023-10-09T06:26:33.787441+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:26:33.787441+0000","last_undegraded":"2023-10-09T06:26:33.787441+0000","last_fullsized":"2023-10-09T06:26:33.787441+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"301'904","reported_seq":917,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:41.915405+0000","last_change":"2023-10-09T06:26:28.776736+0000","last_active":"2023-10-09T06:26:41.915405+0000","last_peered":"2023-10-09T06:26:41.915405+0000","last_clean":"2023-10-09T06:26:41.915405+0000","last_became_active":"2023-10-09T06:26:28.776473+0000","last_became_peered":"2023-10-09T06:26:28.776473+0000","last_unstale":"2023-10-09T06:26:41.915405+0000","last_undegraded":"2023-10-09T06:26:41.915405+0000","last_fullsized":"2023-10-09T06:26:41.915405+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":904,"log_dups_size":0,"ondisk_log_size":904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:05.868023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59244544,"num_objects":904,"num_object_clones":0,"num_object_copies":2712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":904,"num_write_kb":57856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":581,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815430+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:26:33.815430+0000","last_peered":"2023-10-09T06:26:33.815430+0000","last_clean":"2023-10-09T06:26:33.815430+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:26:33.815430+0000","last_undegraded":"2023-10-09T06:26:33.815430+0000","last_fullsized":"2023-10-09T06:26:33.815430+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":258,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815710+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:26:33.815710+0000","last_peered":"2023-10-09T06:26:33.815710+0000","last_clean":"2023-10-09T06:26:33.815710+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:26:33.815710+0000","last_undegraded":"2023-10-09T06:26:33.815710+0000","last_fullsized":"2023-10-09T06:26:33.815710+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":233,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817696+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:26:33.817696+0000","last_peered":"2023-10-09T06:26:33.817696+0000","last_clean":"2023-10-09T06:26:33.817696+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:26:33.817696+0000","last_undegraded":"2023-10-09T06:26:33.817696+0000","last_fullsized":"2023-10-09T06:26:33.817696+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"301'880","reported_seq":893,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:41.704833+0000","last_change":"2023-10-09T06:26:28.775845+0000","last_active":"2023-10-09T06:26:41.704833+0000","last_peered":"2023-10-09T06:26:41.704833+0000","last_clean":"2023-10-09T06:26:41.704833+0000","last_became_active":"2023-10-09T06:26:28.775581+0000","last_became_peered":"2023-10-09T06:26:28.775581+0000","last_unstale":"2023-10-09T06:26:41.704833+0000","last_undegraded":"2023-10-09T06:26:41.704833+0000","last_fullsized":"2023-10-09T06:26:41.704833+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":880,"log_dups_size":0,"ondisk_log_size":880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:22.188138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57671680,"num_objects":880,"num_object_clones":0,"num_object_copies":2640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":880,"num_write_kb":56320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":523,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815031+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:26:33.815031+0000","last_peered":"2023-10-09T06:26:33.815031+0000","last_clean":"2023-10-09T06:26:33.815031+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:26:33.815031+0000","last_undegraded":"2023-10-09T06:26:33.815031+0000","last_fullsized":"2023-10-09T06:26:33.815031+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":284,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.818080+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:26:33.818080+0000","last_peered":"2023-10-09T06:26:33.818080+0000","last_clean":"2023-10-09T06:26:33.818080+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:26:33.818080+0000","last_undegraded":"2023-10-09T06:26:33.818080+0000","last_fullsized":"2023-10-09T06:26:33.818080+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":210,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816500+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:26:33.816500+0000","last_peered":"2023-10-09T06:26:33.816500+0000","last_clean":"2023-10-09T06:26:33.816500+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:26:33.816500+0000","last_undegraded":"2023-10-09T06:26:33.816500+0000","last_fullsized":"2023-10-09T06:26:33.816500+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":227,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817450+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:26:33.817450+0000","last_peered":"2023-10-09T06:26:33.817450+0000","last_clean":"2023-10-09T06:26:33.817450+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:26:33.817450+0000","last_undegraded":"2023-10-09T06:26:33.817450+0000","last_fullsized":"2023-10-09T06:26:33.817450+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"301'443","reported_seq":456,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.793920+0000","last_change":"2023-10-09T06:26:28.773695+0000","last_active":"2023-10-09T06:26:38.793920+0000","last_peered":"2023-10-09T06:26:38.793920+0000","last_clean":"2023-10-09T06:26:38.793920+0000","last_became_active":"2023-10-09T06:26:28.773467+0000","last_became_peered":"2023-10-09T06:26:28.773467+0000","last_unstale":"2023-10-09T06:26:38.793920+0000","last_undegraded":"2023-10-09T06:26:38.793920+0000","last_fullsized":"2023-10-09T06:26:38.793920+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":443,"log_dups_size":0,"ondisk_log_size":443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:33.681958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29032448,"num_objects":443,"num_object_clones":0,"num_object_copies":1329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":443,"num_write_kb":28352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":586,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783972+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:26:33.783972+0000","last_peered":"2023-10-09T06:26:33.783972+0000","last_clean":"2023-10-09T06:26:33.783972+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:26:33.783972+0000","last_undegraded":"2023-10-09T06:26:33.783972+0000","last_fullsized":"2023-10-09T06:26:33.783972+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":158,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817557+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:26:33.817557+0000","last_peered":"2023-10-09T06:26:33.817557+0000","last_clean":"2023-10-09T06:26:33.817557+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:26:33.817557+0000","last_undegraded":"2023-10-09T06:26:33.817557+0000","last_fullsized":"2023-10-09T06:26:33.817557+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":262,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787154+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:26:33.787154+0000","last_peered":"2023-10-09T06:26:33.787154+0000","last_clean":"2023-10-09T06:26:33.787154+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:26:33.787154+0000","last_undegraded":"2023-10-09T06:26:33.787154+0000","last_fullsized":"2023-10-09T06:26:33.787154+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":248,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.818053+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:26:33.818053+0000","last_peered":"2023-10-09T06:26:33.818053+0000","last_clean":"2023-10-09T06:26:33.818053+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:26:33.818053+0000","last_undegraded":"2023-10-09T06:26:33.818053+0000","last_fullsized":"2023-10-09T06:26:33.818053+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"301'414","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.816475+0000","last_change":"2023-10-09T06:26:28.772110+0000","last_active":"2023-10-09T06:26:38.816475+0000","last_peered":"2023-10-09T06:26:38.816475+0000","last_clean":"2023-10-09T06:26:38.816475+0000","last_became_active":"2023-10-09T06:26:28.771837+0000","last_became_peered":"2023-10-09T06:26:28.771837+0000","last_unstale":"2023-10-09T06:26:38.816475+0000","last_undegraded":"2023-10-09T06:26:38.816475+0000","last_fullsized":"2023-10-09T06:26:38.816475+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":414,"log_dups_size":0,"ondisk_log_size":414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:42.271885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27131904,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":414,"num_write_kb":26496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":593,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786032+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:26:33.786032+0000","last_peered":"2023-10-09T06:26:33.786032+0000","last_clean":"2023-10-09T06:26:33.786032+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:26:33.786032+0000","last_undegraded":"2023-10-09T06:26:33.786032+0000","last_fullsized":"2023-10-09T06:26:33.786032+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":267,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814872+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:26:33.814872+0000","last_peered":"2023-10-09T06:26:33.814872+0000","last_clean":"2023-10-09T06:26:33.814872+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:26:33.814872+0000","last_undegraded":"2023-10-09T06:26:33.814872+0000","last_fullsized":"2023-10-09T06:26:33.814872+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":274,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817571+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:26:33.817571+0000","last_peered":"2023-10-09T06:26:33.817571+0000","last_clean":"2023-10-09T06:26:33.817571+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:26:33.817571+0000","last_undegraded":"2023-10-09T06:26:33.817571+0000","last_fullsized":"2023-10-09T06:26:33.817571+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":270,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817735+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:26:33.817735+0000","last_peered":"2023-10-09T06:26:33.817735+0000","last_clean":"2023-10-09T06:26:33.817735+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:26:33.817735+0000","last_undegraded":"2023-10-09T06:26:33.817735+0000","last_fullsized":"2023-10-09T06:26:33.817735+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"301'427","reported_seq":440,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.823959+0000","last_change":"2023-10-09T06:26:28.773784+0000","last_active":"2023-10-09T06:26:38.823959+0000","last_peered":"2023-10-09T06:26:38.823959+0000","last_clean":"2023-10-09T06:26:38.823959+0000","last_became_active":"2023-10-09T06:26:28.773431+0000","last_became_peered":"2023-10-09T06:26:28.773431+0000","last_unstale":"2023-10-09T06:26:38.823959+0000","last_undegraded":"2023-10-09T06:26:38.823959+0000","last_fullsized":"2023-10-09T06:26:38.823959+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":427,"log_dups_size":0,"ondisk_log_size":427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:54.972414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27983872,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":427,"num_write_kb":27328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":234,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815063+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:26:33.815063+0000","last_peered":"2023-10-09T06:26:33.815063+0000","last_clean":"2023-10-09T06:26:33.815063+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:26:33.815063+0000","last_undegraded":"2023-10-09T06:26:33.815063+0000","last_fullsized":"2023-10-09T06:26:33.815063+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":574,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786033+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:26:33.786033+0000","last_peered":"2023-10-09T06:26:33.786033+0000","last_clean":"2023-10-09T06:26:33.786033+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:26:33.786033+0000","last_undegraded":"2023-10-09T06:26:33.786033+0000","last_fullsized":"2023-10-09T06:26:33.786033+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":228,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816012+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:26:33.816012+0000","last_peered":"2023-10-09T06:26:33.816012+0000","last_clean":"2023-10-09T06:26:33.816012+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:26:33.816012+0000","last_undegraded":"2023-10-09T06:26:33.816012+0000","last_fullsized":"2023-10-09T06:26:33.816012+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815127+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:26:33.815127+0000","last_peered":"2023-10-09T06:26:33.815127+0000","last_clean":"2023-10-09T06:26:33.815127+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:26:33.815127+0000","last_undegraded":"2023-10-09T06:26:33.815127+0000","last_fullsized":"2023-10-09T06:26:33.815127+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"301'414","reported_seq":427,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.809497+0000","last_change":"2023-10-09T06:26:28.778366+0000","last_active":"2023-10-09T06:26:38.809497+0000","last_peered":"2023-10-09T06:26:38.809497+0000","last_clean":"2023-10-09T06:26:38.809497+0000","last_became_active":"2023-10-09T06:26:28.777926+0000","last_became_peered":"2023-10-09T06:26:28.777926+0000","last_unstale":"2023-10-09T06:26:38.809497+0000","last_undegraded":"2023-10-09T06:26:38.809497+0000","last_fullsized":"2023-10-09T06:26:38.809497+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":414,"log_dups_size":0,"ondisk_log_size":414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:41.094025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27131904,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":414,"num_write_kb":26496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":584,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817535+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:26:33.817535+0000","last_peered":"2023-10-09T06:26:33.817535+0000","last_clean":"2023-10-09T06:26:33.817535+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:26:33.817535+0000","last_undegraded":"2023-10-09T06:26:33.817535+0000","last_fullsized":"2023-10-09T06:26:33.817535+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":144,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.785924+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:26:33.785924+0000","last_peered":"2023-10-09T06:26:33.785924+0000","last_clean":"2023-10-09T06:26:33.785924+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:26:33.785924+0000","last_undegraded":"2023-10-09T06:26:33.785924+0000","last_fullsized":"2023-10-09T06:26:33.785924+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":254,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783891+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:26:33.783891+0000","last_peered":"2023-10-09T06:26:33.783891+0000","last_clean":"2023-10-09T06:26:33.783891+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:26:33.783891+0000","last_undegraded":"2023-10-09T06:26:33.783891+0000","last_fullsized":"2023-10-09T06:26:33.783891+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783902+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:26:33.783902+0000","last_peered":"2023-10-09T06:26:33.783902+0000","last_clean":"2023-10-09T06:26:33.783902+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:26:33.783902+0000","last_undegraded":"2023-10-09T06:26:33.783902+0000","last_fullsized":"2023-10-09T06:26:33.783902+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"301'919","reported_seq":931,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:41.939327+0000","last_change":"2023-10-09T06:26:28.775392+0000","last_active":"2023-10-09T06:26:41.939327+0000","last_peered":"2023-10-09T06:26:41.939327+0000","last_clean":"2023-10-09T06:26:41.939327+0000","last_became_active":"2023-10-09T06:26:28.774522+0000","last_became_peered":"2023-10-09T06:26:28.774522+0000","last_unstale":"2023-10-09T06:26:41.939327+0000","last_undegraded":"2023-10-09T06:26:41.939327+0000","last_fullsized":"2023-10-09T06:26:41.939327+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":919,"log_dups_size":0,"ondisk_log_size":919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.490573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60227584,"num_objects":919,"num_object_clones":0,"num_object_copies":2757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":919,"num_write_kb":58816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":523,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.814921+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:26:33.814921+0000","last_peered":"2023-10-09T06:26:33.814921+0000","last_clean":"2023-10-09T06:26:33.814921+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:26:33.814921+0000","last_undegraded":"2023-10-09T06:26:33.814921+0000","last_fullsized":"2023-10-09T06:26:33.814921+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":144,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786449+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:26:33.786449+0000","last_peered":"2023-10-09T06:26:33.786449+0000","last_clean":"2023-10-09T06:26:33.786449+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:26:33.786449+0000","last_undegraded":"2023-10-09T06:26:33.786449+0000","last_fullsized":"2023-10-09T06:26:33.786449+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":283,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.818038+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:26:33.818038+0000","last_peered":"2023-10-09T06:26:33.818038+0000","last_clean":"2023-10-09T06:26:33.818038+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:26:33.818038+0000","last_undegraded":"2023-10-09T06:26:33.818038+0000","last_fullsized":"2023-10-09T06:26:33.818038+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":266,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783737+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:26:33.783737+0000","last_peered":"2023-10-09T06:26:33.783737+0000","last_clean":"2023-10-09T06:26:33.783737+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:26:33.783737+0000","last_undegraded":"2023-10-09T06:26:33.783737+0000","last_fullsized":"2023-10-09T06:26:33.783737+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"301'345","reported_seq":357,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:38.158335+0000","last_change":"2023-10-09T06:26:28.777254+0000","last_active":"2023-10-09T06:26:38.158335+0000","last_peered":"2023-10-09T06:26:38.158335+0000","last_clean":"2023-10-09T06:26:38.158335+0000","last_became_active":"2023-10-09T06:26:28.776915+0000","last_became_peered":"2023-10-09T06:26:28.776915+0000","last_unstale":"2023-10-09T06:26:38.158335+0000","last_undegraded":"2023-10-09T06:26:38.158335+0000","last_fullsized":"2023-10-09T06:26:38.158335+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":345,"log_dups_size":0,"ondisk_log_size":345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:04.121493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22609920,"num_objects":345,"num_object_clones":0,"num_object_copies":1035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":345,"num_write_kb":22080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":584,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787155+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:26:33.787155+0000","last_peered":"2023-10-09T06:26:33.787155+0000","last_clean":"2023-10-09T06:26:33.787155+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:26:33.787155+0000","last_undegraded":"2023-10-09T06:26:33.787155+0000","last_fullsized":"2023-10-09T06:26:33.787155+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":291,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816618+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:26:33.816618+0000","last_peered":"2023-10-09T06:26:33.816618+0000","last_clean":"2023-10-09T06:26:33.816618+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:26:33.816618+0000","last_undegraded":"2023-10-09T06:26:33.816618+0000","last_fullsized":"2023-10-09T06:26:33.816618+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":260,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.817849+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:26:33.817849+0000","last_peered":"2023-10-09T06:26:33.817849+0000","last_clean":"2023-10-09T06:26:33.817849+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:26:33.817849+0000","last_undegraded":"2023-10-09T06:26:33.817849+0000","last_fullsized":"2023-10-09T06:26:33.817849+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":280,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815559+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:26:33.815559+0000","last_peered":"2023-10-09T06:26:33.815559+0000","last_clean":"2023-10-09T06:26:33.815559+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:26:33.815559+0000","last_undegraded":"2023-10-09T06:26:33.815559+0000","last_fullsized":"2023-10-09T06:26:33.815559+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"301'1101","reported_seq":1113,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:43.051203+0000","last_change":"2023-10-09T06:26:28.775931+0000","last_active":"2023-10-09T06:26:43.051203+0000","last_peered":"2023-10-09T06:26:43.051203+0000","last_clean":"2023-10-09T06:26:43.051203+0000","last_became_active":"2023-10-09T06:26:28.775652+0000","last_became_peered":"2023-10-09T06:26:28.775652+0000","last_unstale":"2023-10-09T06:26:43.051203+0000","last_undegraded":"2023-10-09T06:26:43.051203+0000","last_fullsized":"2023-10-09T06:26:43.051203+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1101,"log_dups_size":0,"ondisk_log_size":1101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:39.386224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72155136,"num_objects":1101,"num_object_clones":0,"num_object_copies":3303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1101,"num_write_kb":70464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":605,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816797+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:26:33.816797+0000","last_peered":"2023-10-09T06:26:33.816797+0000","last_clean":"2023-10-09T06:26:33.816797+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:26:33.816797+0000","last_undegraded":"2023-10-09T06:26:33.816797+0000","last_fullsized":"2023-10-09T06:26:33.816797+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":285,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815192+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:26:33.815192+0000","last_peered":"2023-10-09T06:26:33.815192+0000","last_clean":"2023-10-09T06:26:33.815192+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:26:33.815192+0000","last_undegraded":"2023-10-09T06:26:33.815192+0000","last_fullsized":"2023-10-09T06:26:33.815192+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":258,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783742+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:26:33.783742+0000","last_peered":"2023-10-09T06:26:33.783742+0000","last_clean":"2023-10-09T06:26:33.783742+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:26:33.783742+0000","last_undegraded":"2023-10-09T06:26:33.783742+0000","last_fullsized":"2023-10-09T06:26:33.783742+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786740+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:26:33.786740+0000","last_peered":"2023-10-09T06:26:33.786740+0000","last_clean":"2023-10-09T06:26:33.786740+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:26:33.786740+0000","last_undegraded":"2023-10-09T06:26:33.786740+0000","last_fullsized":"2023-10-09T06:26:33.786740+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":260,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.783657+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:26:33.783657+0000","last_peered":"2023-10-09T06:26:33.783657+0000","last_clean":"2023-10-09T06:26:33.783657+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:26:33.783657+0000","last_undegraded":"2023-10-09T06:26:33.783657+0000","last_fullsized":"2023-10-09T06:26:33.783657+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":283,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815396+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:26:33.815396+0000","last_peered":"2023-10-09T06:26:33.815396+0000","last_clean":"2023-10-09T06:26:33.815396+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:26:33.815396+0000","last_undegraded":"2023-10-09T06:26:33.815396+0000","last_fullsized":"2023-10-09T06:26:33.815396+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":106,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787056+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:26:33.787056+0000","last_peered":"2023-10-09T06:26:33.787056+0000","last_clean":"2023-10-09T06:26:33.787056+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:26:33.787056+0000","last_undegraded":"2023-10-09T06:26:33.787056+0000","last_fullsized":"2023-10-09T06:26:33.787056+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":288,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787135+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:26:33.787135+0000","last_peered":"2023-10-09T06:26:33.787135+0000","last_clean":"2023-10-09T06:26:33.787135+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:26:33.787135+0000","last_undegraded":"2023-10-09T06:26:33.787135+0000","last_fullsized":"2023-10-09T06:26:33.787135+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":106,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.787326+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:26:33.787326+0000","last_peered":"2023-10-09T06:26:33.787326+0000","last_clean":"2023-10-09T06:26:33.787326+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:26:33.787326+0000","last_undegraded":"2023-10-09T06:26:33.787326+0000","last_fullsized":"2023-10-09T06:26:33.787326+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786979+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:26:33.786979+0000","last_peered":"2023-10-09T06:26:33.786979+0000","last_clean":"2023-10-09T06:26:33.786979+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:26:33.786979+0000","last_undegraded":"2023-10-09T06:26:33.786979+0000","last_fullsized":"2023-10-09T06:26:33.786979+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":259,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.816341+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:26:33.816341+0000","last_peered":"2023-10-09T06:26:33.816341+0000","last_clean":"2023-10-09T06:26:33.816341+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:26:33.816341+0000","last_undegraded":"2023-10-09T06:26:33.816341+0000","last_fullsized":"2023-10-09T06:26:33.816341+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.815353+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:26:33.815353+0000","last_peered":"2023-10-09T06:26:33.815353+0000","last_clean":"2023-10-09T06:26:33.815353+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:26:33.815353+0000","last_undegraded":"2023-10-09T06:26:33.815353+0000","last_fullsized":"2023-10-09T06:26:33.815353+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":273,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.784188+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:26:33.784188+0000","last_peered":"2023-10-09T06:26:33.784188+0000","last_clean":"2023-10-09T06:26:33.784188+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:26:33.784188+0000","last_undegraded":"2023-10-09T06:26:33.784188+0000","last_fullsized":"2023-10-09T06:26:33.784188+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":261,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-09T06:26:33.786643+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:26:33.786643+0000","last_peered":"2023-10-09T06:26:33.786643+0000","last_clean":"2023-10-09T06:26:33.786643+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:26:33.786643+0000","last_undegraded":"2023-10-09T06:26:33.786643+0000","last_fullsized":"2023-10-09T06:26:33.786643+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":726597632,"num_objects":11087,"num_object_clones":0,"num_object_copies":33261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11087,"num_write_kb":709568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2401632256,"data_stored":2401632256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11087,"ondisk_log_size":11087,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234352,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1033640,"kb_used_data":125216,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":92711512,"statfs":{"total":95995035648,"available":94936588288,"internally_reserved":0,"allocated":128221184,"data_stored":127438193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2149999999999999,"5min":8.8960000000000008,"15min":3.8580000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":89.027000000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.25},{"interface":"front","average":{"1min":7.0759999999999996,"5min":9.2319999999999993,"15min":3.8980000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.218},"max":{"1min":89.135999999999996,"5min":219.399,"15min":219.399},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":4.1980000000000004,"5min":3.3839999999999999,"15min":2.8479999999999999},"min":{"1min":0.25,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":49.808,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.694},{"interface":"front","average":{"1min":4.2930000000000001,"5min":3.835,"15min":2.6619999999999999},"min":{"1min":0.33900000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":49.476999999999997,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.81999999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:26:33 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":11.959,"15min":5.1870000000000003},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.2559999999999998,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":12.417,"15min":5.407},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.9299999999999997,"5min":215.869,"15min":215.869},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.359,"15min":1.627},"min":{"1min":0.35599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4710000000000001,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.6739999999999999,"5min":2.4409999999999998,"15min":1.669},"min":{"1min":0.40500000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6760000000000002,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.73199999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":2.0870000000000002,"15min":1.694},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.3620000000000001,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2,"15min":1.419},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9729999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.169,"15min":1.7230000000000001},"min":{"1min":0.33900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.952,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.151,"15min":1.45},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":4.3280000000000003,"5min":5.7130000000000001,"15min":48.774999999999999},"last":0.46500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.458,"15min":1.3580000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7169999999999996,"5min":8.4890000000000008,"15min":11.878},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.2310000000000001,"15min":1.216},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33000000000000002},"max":{"1min":3.0539999999999998,"5min":9.4900000000000002,"15min":12.141999999999999},"last":0.53100000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495990,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386204,"kb_used_data":491284,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92358948,"statfs":{"total":95995035648,"available":94575562752,"internally_reserved":0,"allocated":503074816,"data_stored":502304477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":3.7690000000000001,"15min":3.0790000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":4.2560000000000002,"5min":110.926,"15min":110.926},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.2869999999999999,"5min":3.7480000000000002,"15min":3.2669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2859999999999996,"5min":110.911,"15min":110.911},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.7669999999999999,"15min":3.456},"min":{"1min":0.308,"5min":0.215,"15min":0.215},"max":{"1min":4.4939999999999998,"5min":66.796999999999997,"15min":114.202},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.244,"5min":2.4889999999999999,"15min":3.2810000000000001},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.7389999999999999,"5min":66.704999999999998,"15min":114.179},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":3.6549999999999998,"15min":2.7029999999999998},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.5910000000000002,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.6240000000000001,"5min":3.6720000000000002,"15min":2.5950000000000002},"min":{"1min":0.34799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.9699999999999998,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.46300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.25,"15min":2.02},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":6.9290000000000003,"5min":110.358,"15min":110.358},"last":0.318},{"interface":"front","average":{"1min":1.804,"5min":3.27,"15min":2.069},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.9359999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.42899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":1.1299999999999999,"15min":1.28},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.1890000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.03,"5min":1.474,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":4.2119999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.40999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.456,"15min":1.3640000000000001},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.2460000000000004,"5min":22.379999999999999,"15min":22.379999999999999},"last":0.377},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.488,"15min":1.407},"min":{"1min":0.377,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.8920000000000003,"5min":24.401,"15min":24.401},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:26:05 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":2.3639999999999999,"15min":1.536},"min":{"1min":0.35399999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":4.7610000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":122.881},{"interface":"front","average":{"1min":1.115,"5min":2.343,"15min":1.732},"min":{"1min":0.371,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":4.7510000000000003,"5min":100.995,"15min":100.995},"last":122.92100000000001}]}]},{"osd":1,"up_from":110,"seq":472446402726,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006496,"kb_used_data":511192,"kb_used_omap":4,"kb_used_meta":495291,"kb_avail":92738656,"statfs":{"total":95995035648,"available":94964383744,"internally_reserved":0,"allocated":523460608,"data_stored":522155943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":507178894},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":6.7030000000000003,"15min":3.9319999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.2389999999999999,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.255},{"interface":"front","average":{"1min":1.77,"5min":7.8129999999999997,"15min":4.468},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":9.6170000000000009,"5min":145.982,"15min":149.02699999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":5.4480000000000004,"15min":3.0590000000000002},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.193},"max":{"1min":4.5960000000000001,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.6459999999999999,"5min":6.5979999999999999,"15min":3.8479999999999999},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.219},"max":{"1min":11.522,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.27700000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:26:06 2023","interfaces":[{"interface":"back","average":{"1min":4.2439999999999998,"5min":4.0350000000000001,"15min":2.585},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":16.047999999999998,"5min":77.331999999999994,"15min":153.18199999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.6549999999999998,"15min":1.9319999999999999},"min":{"1min":0.23799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.21,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.39800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":4.6630000000000003,"15min":3.6259999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.6159999999999997,"5min":99.391000000000005,"15min":199.43799999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.6519999999999999,"5min":4.7789999999999999,"15min":3.569},"min":{"1min":0.38600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1920000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":0.623}]},{"osd":5,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":2.355,"15min":2.3820000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7450000000000001,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.506,"5min":3.2120000000000002,"15min":2.7829999999999999},"min":{"1min":0.38900000000000001,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":4.7640000000000002,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.74199999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":3.669,"15min":2.8519999999999999},"min":{"1min":0.38300000000000001,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":4.883,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.3999999999999999,"5min":3.3319999999999999,"15min":2.6560000000000001},"min":{"1min":0.377,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.6550000000000002,"5min":77.378,"15min":77.378},"last":0.78300000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":3.129,"15min":2.6030000000000002},"min":{"1min":0.41899999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6820000000000004,"5min":77.456999999999994,"15min":77.456999999999994},"last":55.231000000000002},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.7469999999999999,"15min":2.137},"min":{"1min":0.375,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7009999999999996,"5min":77.359999999999999,"15min":77.359999999999999},"last":55.253999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024159,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085528,"kb_used_data":387728,"kb_used_omap":5,"kb_used_meta":697786,"kb_avail":92659624,"statfs":{"total":95995035648,"available":94883454976,"internally_reserved":0,"allocated":397033472,"data_stored":395505479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":714533498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.0139999999999998,"15min":2.8359999999999999},"min":{"1min":0.20699999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":9.1820000000000004,"5min":13.916,"15min":185.17400000000001},"last":1.04},{"interface":"front","average":{"1min":2.7599999999999998,"5min":2.3879999999999999,"15min":3.3599999999999999},"min":{"1min":0.251,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":9.3369999999999997,"5min":13.945,"15min":186.691},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5800000000000001,"5min":2.6669999999999998,"15min":2.3610000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":57.162999999999997,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":4.8360000000000003,"5min":3.0649999999999999,"15min":2.5510000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.20699999999999999},"max":{"1min":57.585000000000001,"5min":57.585000000000001,"15min":57.585000000000001},"last":0.94999999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:25:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2010000000000001,"5min":3.3999999999999999,"15min":2.9140000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2989999999999995,"5min":52.683,"15min":185.89400000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.9950000000000001,"5min":3.4289999999999998,"15min":3.0259999999999998},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":9.3759999999999994,"5min":52.941000000000003,"15min":186.72900000000001},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5860000000000001,"15min":1.516},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.024,"5min":7.9660000000000002,"15min":13.244},"last":1.115},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.833,"15min":1.9410000000000001},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":4.9770000000000003,"5min":8.4890000000000008,"15min":12.752000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.0990000000000002,"15min":1.9239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8450000000000002,"5min":62.984999999999999,"15min":62.984999999999999},"last":1.143},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.7090000000000001,"15min":2.504},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":5.8650000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":0.63900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.282,"15min":2.319},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.1360000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.498,"5min":1.6299999999999999,"15min":1.8400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.819,"5min":7.9980000000000002,"15min":13.315},"last":1.083}]},{"osd":7,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.5099999999999998,"15min":2.1920000000000002},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":4.7279999999999998,"5min":62.262,"15min":62.262},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.202,"5min":2.9910000000000001,"15min":2.5619999999999998},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":7.1059999999999999,"5min":62.229999999999997,"15min":62.229999999999997},"last":0.92500000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814028,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":725956,"kb_used_data":295356,"kb_used_omap":7,"kb_used_meta":430584,"kb_avail":93019196,"statfs":{"total":95995035648,"available":95251656704,"internally_reserved":0,"allocated":302444544,"data_stored":301163281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":440918097},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":4.5810000000000004,"15min":3.915},"min":{"1min":0.23499999999999999,"5min":0.183,"15min":0.157},"max":{"1min":4.4189999999999996,"5min":150.32599999999999,"15min":163.40100000000001},"last":1.391},{"interface":"front","average":{"1min":1.4670000000000001,"5min":3.5030000000000001,"15min":3.798},"min":{"1min":0.223,"5min":0.182,"15min":0.182},"max":{"1min":7.5590000000000002,"5min":110.66500000000001,"15min":163.363},"last":0.45100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":4.0359999999999996,"15min":3.6299999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":6.968,"5min":205.148,"15min":222.50299999999999},"last":1.5329999999999999},{"interface":"front","average":{"1min":1.806,"5min":4.0229999999999997,"15min":3.9199999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.77,"5min":205.267,"15min":222.19399999999999},"last":0.41399999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":2.8780000000000001,"15min":1.8140000000000001},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":13.664999999999999,"5min":69.936999999999998,"15min":81.183999999999997},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.0150000000000001,"5min":2.9369999999999998,"15min":1.8700000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.728,"5min":69.906999999999996,"15min":81.061999999999998},"last":1.343}]},{"osd":4,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.8530000000000002,"15min":2.1440000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":3.6850000000000001,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.5109999999999999,"5min":3.0009999999999999,"15min":2.4249999999999998},"min":{"1min":0.34599999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.8250000000000002,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.29899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.429,"15min":1.4930000000000001},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.309},"max":{"1min":3.859,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.514,"15min":2.0270000000000001},"min":{"1min":0.42799999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":5.8940000000000001,"5min":6.9349999999999996,"15min":10.548},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":3.5529999999999999,"15min":2.5230000000000001},"min":{"1min":0.40000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.8689999999999998,"5min":138.291,"15min":138.291},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":3.4129999999999998,"15min":2.157},"min":{"1min":0.47799999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":3.9980000000000002,"5min":138.01599999999999,"15min":138.01599999999999},"last":1.429}]},{"osd":7,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3029999999999999,"15min":1.8360000000000001},"min":{"1min":0.44500000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":4.9889999999999999,"5min":55.851999999999997,"15min":55.851999999999997},"last":1.4550000000000001},{"interface":"front","average":{"1min":1.226,"5min":2.141,"15min":2.2589999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.899,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.54100000000000004}]}]},{"osd":3,"up_from":149,"seq":639950127225,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":611564,"kb_used_data":211172,"kb_used_omap":2,"kb_used_meta":400381,"kb_avail":93133588,"statfs":{"total":95995035648,"available":95368794112,"internally_reserved":0,"allocated":216240128,"data_stored":215040328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":409990197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":7.8659999999999997,"15min":3.262},"min":{"1min":0.19800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":10.367000000000001,"5min":187.852,"15min":187.852},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.23,"5min":8.4719999999999995,"15min":3.7200000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":4.6539999999999999,"5min":187.923,"15min":187.923},"last":0.71699999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":6.0439999999999996,"15min":2.9500000000000002},"min":{"1min":0.255,"5min":0.246,"15min":0.223},"max":{"1min":11.367000000000001,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.978,"5min":3.673,"15min":1.9379999999999999},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.8810000000000002,"5min":67.028000000000006,"15min":67.028000000000006},"last":1.601}]},{"osd":2,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":5.2350000000000003,"15min":2.649},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.57,"5min":187.76400000000001,"15min":187.76400000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":2.3210000000000002,"5min":4.8769999999999998,"15min":2.1349999999999998},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5250000000000004,"5min":187.79499999999999,"15min":187.79499999999999},"last":0.373}]},{"osd":4,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.544,"15min":1.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":6.7830000000000004,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.2389999999999999,"15min":1.964},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.9649999999999999,"5min":187.67400000000001,"15min":187.67400000000001},"last":0.33300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":3.2530000000000001,"15min":2.1080000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":3.0059999999999998,"5min":187.833,"15min":187.833},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.0649999999999999,"15min":2.036},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":187.952,"15min":187.952},"last":0.56100000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.6080000000000001,"15min":2.1560000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6280000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1890000000000001,"15min":1.9910000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8719999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":3.5169999999999999,"15min":1.8320000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":6.8239999999999998,"5min":192.20400000000001,"15min":192.20400000000001},"last":0.433},{"interface":"front","average":{"1min":1.242,"5min":3.4580000000000002,"15min":2.093},"min":{"1min":0.41899999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":5.431,"5min":187.86500000000001,"15min":187.86500000000001},"last":0.51000000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019262,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1225948,"kb_used_data":339156,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":92519204,"statfs":{"total":95995035648,"available":94739664896,"internally_reserved":0,"allocated":347295744,"data_stored":346524684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":3.5310000000000001,"15min":2.2469999999999999},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.7480000000000002,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.1279999999999999,"5min":3.9089999999999998,"15min":2.6549999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":4.3799999999999999,"5min":76.988,"15min":76.988},"last":0.95199999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.98399999999999999,"15min":2.016},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.7519999999999998,"5min":5.0190000000000001,"15min":91.849999999999994},"last":0.87},{"interface":"front","average":{"1min":1.381,"5min":1.359,"15min":2.1579999999999999},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":5.0979999999999999,"5min":9.625,"15min":91.885000000000005},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":3.5089999999999999,"15min":2.3359999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":47.125,"5min":69.640000000000001,"15min":69.640000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":5.9249999999999998,"5min":3.794,"15min":2.5379999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":47.151000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":0.60699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.7029999999999998,"15min":2.2879999999999998},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.8360000000000003,"5min":45.591000000000001,"15min":91.697000000000003},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.341,"5min":2.3140000000000001,"15min":2.129},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":3.5230000000000001,"5min":45.548999999999999,"15min":94.356999999999999},"last":1.4590000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.95799999999999996,"15min":1.1419999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24299999999999999},"max":{"1min":6.2670000000000003,"5min":9.1720000000000006,"15min":10.673},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.91500000000000004,"15min":1.228},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.536,"5min":9.6379999999999999,"15min":12.552},"last":0.57799999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.83399999999999996,"15min":0.84899999999999998},"min":{"1min":0.29399999999999998,"5min":0.27700000000000002,"15min":0.26500000000000001},"max":{"1min":2.1179999999999999,"5min":8.8369999999999997,"15min":9.7680000000000007},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1830000000000001,"15min":1.286},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.859,"5min":15.321999999999999,"15min":15.321999999999999},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.7350000000000001,"15min":1.476},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":2.1880000000000002,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.876,"5min":1.706,"15min":1.488},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":3.8450000000000002,"5min":62.573,"15min":62.573},"last":0.499}]}]},{"osd":5,"up_from":0,"seq":163208757626,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":318242816,"data_stored":318242816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":548143104,"data_stored":548143104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":347537408,"data_stored":347537408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":227409920,"data_stored":227409920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":275906560,"data_stored":275906560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":554303488,"data_stored":554303488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":130088960,"data_stored":130088960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:26:46.000 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:46.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:46.106 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:46.137 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:45 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:26:46.141 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:46.204 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:46.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:46.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:46.228 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:46.310 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:46.383 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:46.542 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:26:46.543 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:46.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:46.651 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:26:46.690 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:46.772 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 22970 22954 130.315 92.4375 0.00960053 0.00766745 2023-10-09T06:26:46.772 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:26:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:46.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:26:46.882 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[220921]: pgmap v931: 99 pgs: 99 active+clean; 1022 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3839053999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[212750]: pgmap v931: 99 pgs: 99 active+clean; 1022 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:26:46.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:46.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:46.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3839053999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:46.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:46.915 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:46.996 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:47.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:47.102 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:46 smithi150 ceph-mon[194178]: pgmap v931: 99 pgs: 99 active+clean; 1022 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:26:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:26:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3839053999' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:26:47.181 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:47.288 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:47.625 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:26:47.737 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:47.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:47.772 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 24982 24966 129.934 125.75 0.00908823 0.00769035 2023-10-09T06:26:47.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:47.877 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:48.511 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: pgmap v932: 99 pgs: 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:26:48.511 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:48.512 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:48.512 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T06:26:48.512 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.512 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[212750]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: pgmap v932: 99 pgs: 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.513 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:48 smithi107 ceph-mon[220921]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: pgmap v932: 99 pgs: 99 active+clean; 1.2 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:26:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:26:48.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: osdmap e302: 8 total, 8 up, 7 in 2023-10-09T06:26:48.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:48 smithi150 ceph-mon[194178]: failed to encode map e302 with expected crc 2023-10-09T06:26:48.772 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 26362 26346 126.575 86.25 0.00285697 0.00787747 2023-10-09T06:26:48.967 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:26:49.409 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:49.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:49.513 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:49.567 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:49.772 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 27897 27881 124.387 95.9375 0.0176522 0.00803274 2023-10-09T06:26:49.846 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:26:49.846 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:26:49.847 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:26:49.847 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:26:49.908 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:49.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:50.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:50.088 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:50.104 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:50.229 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:50.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:50.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:50.335 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:50.372 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:50.603 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:26:50.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:50 smithi107 ceph-mon[220921]: pgmap v934: 99 pgs: 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-09T06:26:50.604 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:50 smithi107 ceph-mon[212750]: pgmap v934: 99 pgs: 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-09T06:26:50.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:50 smithi150 ceph-mon[194178]: pgmap v934: 99 pgs: 99 active+clean; 1.5 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-09T06:26:50.668 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:26:50.711 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:26:50.756 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:26:50.783 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 29306 29290 121.89 88.0625 0.00355013 0.00819685 2023-10-09T06:26:50.800 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:26:50.837 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:26:50.873 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:26:50.908 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.918 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:26:50.919 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:26:51.092 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:51.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:51.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:51.198 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:51.209 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:26:51.264 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4131928660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:51.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4131928660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:51.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4131928660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:26:51.781 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 31097 31081 121.268 111.938 0.00626318 0.00824106 2023-10-09T06:26:51.859 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:51.888 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:26:52.238 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:26:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:26:52] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.43.0" 2023-10-09T06:26:52.239 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:52 smithi107 ceph-mon[212750]: pgmap v935: 99 pgs: 99 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T06:26:52.445 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:52 smithi150 ceph-mon[194178]: pgmap v935: 99 pgs: 99 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T06:26:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:52 smithi107 ceph-mon[220921]: pgmap v935: 99 pgs: 99 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T06:26:52.681 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:52.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:52.781 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 32593 32577 119.636 93.5 0.00202838 0.0083277 2023-10-09T06:26:52.791 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:52.822 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:53.487 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:53.497 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:53.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:53.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:53.600 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:26:53.666 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:53.781 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 33819 33803 117.248 76.625 0.00514835 0.00852461 2023-10-09T06:26:53.867 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:26:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:53.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:26:53.976 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:54.022 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:54.108 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:54.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:54.154 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:54.154 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:54.214 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:26:54.293 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:54.610 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:26:54.627 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:26:54.686 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:26:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:54.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:26:54.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 35228 35212 115.713 88.0625 0.00998146 0.00863641 2023-10-09T06:26:54.790 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:54.859 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:54.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[212750]: pgmap v936: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:26:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:54.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:54 smithi150 ceph-mon[194178]: pgmap v936: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:26:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:54.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[220921]: pgmap v936: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-09T06:26:54.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:54.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:55.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:26:55.332 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:26:55.540 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:55.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:26:55.782709+0000 min lat: 0.00108834 max lat: 0.264003 avg lat: 0.00880038 2023-10-09T06:26:55.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:26:55.782 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 36393 36377 113.57 72.8125 0.006767 0.00880038 2023-10-09T06:26:55.989 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:55 smithi150 ceph-mon[194178]: from='client.46013 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:56.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:55 smithi107 ceph-mon[212750]: from='client.46013 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:56.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:55 smithi107 ceph-mon[220921]: from='client.46013 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:26:56.487 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:26:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:56.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:26:56.569 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":936,"stamp":"2023-10-09T06:26:53.153093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1900418976,"num_objects":29013,"num_object_clones":0,"num_object_copies":87039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29050,"num_write_kb":1855325,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":338010,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":30815,"ondisk_log_size":30815,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10530060,"kb_used_data":5769108,"kb_used_omap":20,"kb_used_meta":4760875,"kb_avail":645686004,"statfs":{"total":671965249536,"available":661182468096,"internally_reserved":0,"allocated":5907566592,"data_stored":5899916846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4875136654},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,2,10,2,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1173225472,"num_objects":17902,"num_object_clones":0,"num_object_copies":53706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17902,"num_write_kb":1145728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007517"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":281,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.813057+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:26:35.813057+0000","last_peered":"2023-10-09T06:26:35.813057+0000","last_clean":"2023-10-09T06:26:35.813057+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:26:35.813057+0000","last_undegraded":"2023-10-09T06:26:35.813057+0000","last_fullsized":"2023-10-09T06:26:35.813057+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:43.833967+0000","last_change":"2023-10-09T06:16:44.958058+0000","last_active":"2023-10-09T06:26:43.833967+0000","last_peered":"2023-10-09T06:26:43.833967+0000","last_clean":"2023-10-09T06:26:43.833967+0000","last_became_active":"2023-10-09T06:16:44.957537+0000","last_became_peered":"2023-10-09T06:16:44.957537+0000","last_unstale":"2023-10-09T06:26:43.833967+0000","last_undegraded":"2023-10-09T06:26:43.833967+0000","last_fullsized":"2023-10-09T06:26:43.833967+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.445870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.802501+0000","last_change":"2023-10-09T06:16:44.955580+0000","last_active":"2023-10-09T06:26:35.802501+0000","last_peered":"2023-10-09T06:26:35.802501+0000","last_clean":"2023-10-09T06:26:35.802501+0000","last_became_active":"2023-10-09T06:16:44.955255+0000","last_became_peered":"2023-10-09T06:16:44.955255+0000","last_unstale":"2023-10-09T06:26:35.802501+0000","last_undegraded":"2023-10-09T06:26:35.802501+0000","last_fullsized":"2023-10-09T06:26:35.802501+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:43.791386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807414+0000","last_change":"2023-10-09T06:19:06.761480+0000","last_active":"2023-10-09T06:26:35.807414+0000","last_peered":"2023-10-09T06:26:35.807414+0000","last_clean":"2023-10-09T06:26:35.807414+0000","last_became_active":"2023-10-09T06:19:06.761166+0000","last_became_peered":"2023-10-09T06:19:06.761166+0000","last_unstale":"2023-10-09T06:26:35.807414+0000","last_undegraded":"2023-10-09T06:26:35.807414+0000","last_fullsized":"2023-10-09T06:26:35.807414+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:34.174935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806587+0000","last_change":"2023-10-09T06:16:44.959728+0000","last_active":"2023-10-09T06:26:35.806587+0000","last_peered":"2023-10-09T06:26:35.806587+0000","last_clean":"2023-10-09T06:26:35.806587+0000","last_became_active":"2023-10-09T06:16:44.958593+0000","last_became_peered":"2023-10-09T06:16:44.958593+0000","last_unstale":"2023-10-09T06:26:35.806587+0000","last_undegraded":"2023-10-09T06:26:35.806587+0000","last_fullsized":"2023-10-09T06:26:35.806587+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:06.110322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":281,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814720+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:26:35.814720+0000","last_peered":"2023-10-09T06:26:35.814720+0000","last_clean":"2023-10-09T06:26:35.814720+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:26:35.814720+0000","last_undegraded":"2023-10-09T06:26:35.814720+0000","last_fullsized":"2023-10-09T06:26:35.814720+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":263,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807034+0000","last_change":"2023-10-09T06:21:45.410858+0000","last_active":"2023-10-09T06:26:35.807034+0000","last_peered":"2023-10-09T06:26:35.807034+0000","last_clean":"2023-10-09T06:26:35.807034+0000","last_became_active":"2023-10-09T06:21:45.410276+0000","last_became_peered":"2023-10-09T06:21:45.410276+0000","last_unstale":"2023-10-09T06:26:35.807034+0000","last_undegraded":"2023-10-09T06:26:35.807034+0000","last_fullsized":"2023-10-09T06:26:35.807034+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:49.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806516+0000","last_change":"2023-10-09T06:16:44.963275+0000","last_active":"2023-10-09T06:26:35.806516+0000","last_peered":"2023-10-09T06:26:35.806516+0000","last_clean":"2023-10-09T06:26:35.806516+0000","last_became_active":"2023-10-09T06:16:44.962493+0000","last_became_peered":"2023-10-09T06:16:44.962493+0000","last_unstale":"2023-10-09T06:26:35.806516+0000","last_undegraded":"2023-10-09T06:26:35.806516+0000","last_fullsized":"2023-10-09T06:26:35.806516+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:22.099479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812367+0000","last_change":"2023-10-09T06:21:45.407135+0000","last_active":"2023-10-09T06:26:35.812367+0000","last_peered":"2023-10-09T06:26:35.812367+0000","last_clean":"2023-10-09T06:26:35.812367+0000","last_became_active":"2023-10-09T06:21:45.406660+0000","last_became_peered":"2023-10-09T06:21:45.406660+0000","last_unstale":"2023-10-09T06:26:35.812367+0000","last_undegraded":"2023-10-09T06:26:35.812367+0000","last_fullsized":"2023-10-09T06:26:35.812367+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:56.820254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":238,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.808287+0000","last_change":"2023-10-09T06:19:06.768489+0000","last_active":"2023-10-09T06:26:35.808287+0000","last_peered":"2023-10-09T06:26:35.808287+0000","last_clean":"2023-10-09T06:26:35.808287+0000","last_became_active":"2023-10-09T06:19:06.768163+0000","last_became_peered":"2023-10-09T06:19:06.768163+0000","last_unstale":"2023-10-09T06:26:35.808287+0000","last_undegraded":"2023-10-09T06:26:35.808287+0000","last_fullsized":"2023-10-09T06:26:35.808287+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:06.992876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":151,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812839+0000","last_change":"2023-10-09T06:19:06.806686+0000","last_active":"2023-10-09T06:26:35.812839+0000","last_peered":"2023-10-09T06:26:35.812839+0000","last_clean":"2023-10-09T06:26:35.812839+0000","last_became_active":"2023-10-09T06:19:06.757051+0000","last_became_peered":"2023-10-09T06:19:06.757051+0000","last_unstale":"2023-10-09T06:26:35.812839+0000","last_undegraded":"2023-10-09T06:26:35.812839+0000","last_fullsized":"2023-10-09T06:26:35.812839+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:59:27.870765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":233,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803770+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:26:35.803770+0000","last_peered":"2023-10-09T06:26:35.803770+0000","last_clean":"2023-10-09T06:26:35.803770+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:26:35.803770+0000","last_undegraded":"2023-10-09T06:26:35.803770+0000","last_fullsized":"2023-10-09T06:26:35.803770+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":308,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812394+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:26:35.812394+0000","last_peered":"2023-10-09T06:26:35.812394+0000","last_clean":"2023-10-09T06:26:35.812394+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:26:35.812394+0000","last_undegraded":"2023-10-09T06:26:35.812394+0000","last_fullsized":"2023-10-09T06:26:35.812394+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":272,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812983+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:26:35.812983+0000","last_peered":"2023-10-09T06:26:35.812983+0000","last_clean":"2023-10-09T06:26:35.812983+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:26:35.812983+0000","last_undegraded":"2023-10-09T06:26:35.812983+0000","last_fullsized":"2023-10-09T06:26:35.812983+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":151,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812185+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:26:35.812185+0000","last_peered":"2023-10-09T06:26:35.812185+0000","last_clean":"2023-10-09T06:26:35.812185+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:26:35.812185+0000","last_undegraded":"2023-10-09T06:26:35.812185+0000","last_fullsized":"2023-10-09T06:26:35.812185+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":258,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806053+0000","last_change":"2023-10-09T06:19:06.684692+0000","last_active":"2023-10-09T06:26:35.806053+0000","last_peered":"2023-10-09T06:26:35.806053+0000","last_clean":"2023-10-09T06:26:35.806053+0000","last_became_active":"2023-10-09T06:19:06.684410+0000","last_became_peered":"2023-10-09T06:19:06.684410+0000","last_unstale":"2023-10-09T06:26:35.806053+0000","last_undegraded":"2023-10-09T06:26:35.806053+0000","last_fullsized":"2023-10-09T06:26:35.806053+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:14.736764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":272,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807796+0000","last_change":"2023-10-09T06:19:06.666354+0000","last_active":"2023-10-09T06:26:35.807796+0000","last_peered":"2023-10-09T06:26:35.807796+0000","last_clean":"2023-10-09T06:26:35.807796+0000","last_became_active":"2023-10-09T06:19:06.666054+0000","last_became_peered":"2023-10-09T06:19:06.666054+0000","last_unstale":"2023-10-09T06:26:35.807796+0000","last_undegraded":"2023-10-09T06:26:35.807796+0000","last_fullsized":"2023-10-09T06:26:35.807796+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:58.374098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812677+0000","last_change":"2023-10-09T06:21:45.413416+0000","last_active":"2023-10-09T06:26:35.812677+0000","last_peered":"2023-10-09T06:26:35.812677+0000","last_clean":"2023-10-09T06:26:35.812677+0000","last_became_active":"2023-10-09T06:21:45.413197+0000","last_became_peered":"2023-10-09T06:21:45.413197+0000","last_unstale":"2023-10-09T06:26:35.812677+0000","last_undegraded":"2023-10-09T06:26:35.812677+0000","last_fullsized":"2023-10-09T06:26:35.812677+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:48.758410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":145,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811892+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:26:35.811892+0000","last_peered":"2023-10-09T06:26:35.811892+0000","last_clean":"2023-10-09T06:26:35.811892+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:26:35.811892+0000","last_undegraded":"2023-10-09T06:26:35.811892+0000","last_fullsized":"2023-10-09T06:26:35.811892+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":272,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807550+0000","last_change":"2023-10-09T06:19:06.760963+0000","last_active":"2023-10-09T06:26:35.807550+0000","last_peered":"2023-10-09T06:26:35.807550+0000","last_clean":"2023-10-09T06:26:35.807550+0000","last_became_active":"2023-10-09T06:19:06.760651+0000","last_became_peered":"2023-10-09T06:19:06.760651+0000","last_unstale":"2023-10-09T06:26:35.807550+0000","last_undegraded":"2023-10-09T06:26:35.807550+0000","last_fullsized":"2023-10-09T06:26:35.807550+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:10.297991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":290,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814206+0000","last_change":"2023-10-09T06:21:45.429317+0000","last_active":"2023-10-09T06:26:35.814206+0000","last_peered":"2023-10-09T06:26:35.814206+0000","last_clean":"2023-10-09T06:26:35.814206+0000","last_became_active":"2023-10-09T06:21:45.413178+0000","last_became_peered":"2023-10-09T06:21:45.413178+0000","last_unstale":"2023-10-09T06:26:35.814206+0000","last_undegraded":"2023-10-09T06:26:35.814206+0000","last_fullsized":"2023-10-09T06:26:35.814206+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:31.126963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806852+0000","last_change":"2023-10-09T06:19:06.676268+0000","last_active":"2023-10-09T06:26:35.806852+0000","last_peered":"2023-10-09T06:26:35.806852+0000","last_clean":"2023-10-09T06:26:35.806852+0000","last_became_active":"2023-10-09T06:19:06.675954+0000","last_became_peered":"2023-10-09T06:19:06.675954+0000","last_unstale":"2023-10-09T06:26:35.806852+0000","last_undegraded":"2023-10-09T06:26:35.806852+0000","last_fullsized":"2023-10-09T06:26:35.806852+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:49.122634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"302'1905","reported_seq":1919,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:51.889773+0000","last_change":"2023-10-09T06:26:28.774987+0000","last_active":"2023-10-09T06:26:51.889773+0000","last_peered":"2023-10-09T06:26:51.889773+0000","last_clean":"2023-10-09T06:26:51.889773+0000","last_became_active":"2023-10-09T06:26:28.774691+0000","last_became_peered":"2023-10-09T06:26:28.774691+0000","last_unstale":"2023-10-09T06:26:51.889773+0000","last_undegraded":"2023-10-09T06:26:51.889773+0000","last_fullsized":"2023-10-09T06:26:51.889773+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1905,"log_dups_size":0,"ondisk_log_size":1905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:17.484383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124846080,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1905,"num_write_kb":121920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":163,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812116+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:26:35.812116+0000","last_peered":"2023-10-09T06:26:35.812116+0000","last_clean":"2023-10-09T06:26:35.812116+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:26:35.812116+0000","last_undegraded":"2023-10-09T06:26:35.812116+0000","last_fullsized":"2023-10-09T06:26:35.812116+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":212,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811796+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:26:35.811796+0000","last_peered":"2023-10-09T06:26:35.811796+0000","last_clean":"2023-10-09T06:26:35.811796+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:26:35.811796+0000","last_undegraded":"2023-10-09T06:26:35.811796+0000","last_fullsized":"2023-10-09T06:26:35.811796+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"302'1799","reported_seq":1813,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:50.342594+0000","last_change":"2023-10-09T06:26:28.769277+0000","last_active":"2023-10-09T06:26:50.342594+0000","last_peered":"2023-10-09T06:26:50.342594+0000","last_clean":"2023-10-09T06:26:50.342594+0000","last_became_active":"2023-10-09T06:26:28.769053+0000","last_became_peered":"2023-10-09T06:26:28.769053+0000","last_unstale":"2023-10-09T06:26:50.342594+0000","last_undegraded":"2023-10-09T06:26:50.342594+0000","last_fullsized":"2023-10-09T06:26:50.342594+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1799,"log_dups_size":0,"ondisk_log_size":1799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:00.039037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1799,"num_write_kb":115136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":149,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812128+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:26:35.812128+0000","last_peered":"2023-10-09T06:26:35.812128+0000","last_clean":"2023-10-09T06:26:35.812128+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:26:35.812128+0000","last_undegraded":"2023-10-09T06:26:35.812128+0000","last_fullsized":"2023-10-09T06:26:35.812128+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811761+0000","last_change":"2023-10-09T06:16:44.957272+0000","last_active":"2023-10-09T06:26:35.811761+0000","last_peered":"2023-10-09T06:26:35.811761+0000","last_clean":"2023-10-09T06:26:35.811761+0000","last_became_active":"2023-10-09T06:16:44.956967+0000","last_became_peered":"2023-10-09T06:16:44.956967+0000","last_unstale":"2023-10-09T06:26:35.811761+0000","last_undegraded":"2023-10-09T06:26:35.811761+0000","last_fullsized":"2023-10-09T06:26:35.811761+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:01.685297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"302'1647","reported_seq":1660,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.117049+0000","last_change":"2023-10-09T06:26:28.778132+0000","last_active":"2023-10-09T06:26:48.117049+0000","last_peered":"2023-10-09T06:26:48.117049+0000","last_clean":"2023-10-09T06:26:48.117049+0000","last_became_active":"2023-10-09T06:26:28.777848+0000","last_became_peered":"2023-10-09T06:26:28.777848+0000","last_unstale":"2023-10-09T06:26:48.117049+0000","last_undegraded":"2023-10-09T06:26:48.117049+0000","last_fullsized":"2023-10-09T06:26:48.117049+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1647,"log_dups_size":0,"ondisk_log_size":1647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:31.438186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1647,"num_write_kb":105408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":163,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812601+0000","last_change":"2023-10-09T06:21:45.424916+0000","last_active":"2023-10-09T06:26:35.812601+0000","last_peered":"2023-10-09T06:26:35.812601+0000","last_clean":"2023-10-09T06:26:35.812601+0000","last_became_active":"2023-10-09T06:21:45.414473+0000","last_became_peered":"2023-10-09T06:21:45.414473+0000","last_unstale":"2023-10-09T06:26:35.812601+0000","last_undegraded":"2023-10-09T06:26:35.812601+0000","last_fullsized":"2023-10-09T06:26:35.812601+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:01.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":281,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814489+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:26:35.814489+0000","last_peered":"2023-10-09T06:26:35.814489+0000","last_clean":"2023-10-09T06:26:35.814489+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:26:35.814489+0000","last_undegraded":"2023-10-09T06:26:35.814489+0000","last_fullsized":"2023-10-09T06:26:35.814489+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"302'1566","reported_seq":1579,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.170343+0000","last_change":"2023-10-09T06:26:28.775173+0000","last_active":"2023-10-09T06:26:48.170343+0000","last_peered":"2023-10-09T06:26:48.170343+0000","last_clean":"2023-10-09T06:26:48.170343+0000","last_became_active":"2023-10-09T06:26:28.774891+0000","last_became_peered":"2023-10-09T06:26:28.774891+0000","last_unstale":"2023-10-09T06:26:48.170343+0000","last_undegraded":"2023-10-09T06:26:48.170343+0000","last_fullsized":"2023-10-09T06:26:48.170343+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1566,"log_dups_size":0,"ondisk_log_size":1566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:37.713365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1566,"num_write_kb":100224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.805978+0000","last_change":"2023-10-09T06:21:45.405334+0000","last_active":"2023-10-09T06:26:35.805978+0000","last_peered":"2023-10-09T06:26:35.805978+0000","last_clean":"2023-10-09T06:26:35.805978+0000","last_became_active":"2023-10-09T06:21:45.405012+0000","last_became_peered":"2023-10-09T06:21:45.405012+0000","last_unstale":"2023-10-09T06:26:35.805978+0000","last_undegraded":"2023-10-09T06:26:35.805978+0000","last_fullsized":"2023-10-09T06:26:35.805978+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:39.530336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":255,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806786+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:26:35.806786+0000","last_peered":"2023-10-09T06:26:35.806786+0000","last_clean":"2023-10-09T06:26:35.806786+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:26:35.806786+0000","last_undegraded":"2023-10-09T06:26:35.806786+0000","last_fullsized":"2023-10-09T06:26:35.806786+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"302'2082","reported_seq":2095,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:52.918563+0000","last_change":"2023-10-09T06:26:28.776488+0000","last_active":"2023-10-09T06:26:52.918563+0000","last_peered":"2023-10-09T06:26:52.918563+0000","last_clean":"2023-10-09T06:26:52.918563+0000","last_became_active":"2023-10-09T06:26:28.776212+0000","last_became_peered":"2023-10-09T06:26:28.776212+0000","last_unstale":"2023-10-09T06:26:52.918563+0000","last_undegraded":"2023-10-09T06:26:52.918563+0000","last_fullsized":"2023-10-09T06:26:52.918563+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":2082,"log_dups_size":0,"ondisk_log_size":2082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:12.868016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136445952,"num_objects":2082,"num_object_clones":0,"num_object_copies":6246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2082,"num_write_kb":133248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":594,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811791+0000","last_change":"2023-10-09T06:21:33.751433+0000","last_active":"2023-10-09T06:26:35.811791+0000","last_peered":"2023-10-09T06:26:35.811791+0000","last_clean":"2023-10-09T06:26:35.811791+0000","last_became_active":"2023-10-09T06:21:17.848232+0000","last_became_peered":"2023-10-09T06:21:17.848232+0000","last_unstale":"2023-10-09T06:26:35.811791+0000","last_undegraded":"2023-10-09T06:26:35.811791+0000","last_fullsized":"2023-10-09T06:26:35.811791+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:20:37.902792+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":266,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.809382+0000","last_change":"2023-10-09T06:16:44.960277+0000","last_active":"2023-10-09T06:26:35.809382+0000","last_peered":"2023-10-09T06:26:35.809382+0000","last_clean":"2023-10-09T06:26:35.809382+0000","last_became_active":"2023-10-09T06:16:44.959130+0000","last_became_peered":"2023-10-09T06:16:44.959130+0000","last_unstale":"2023-10-09T06:26:35.809382+0000","last_undegraded":"2023-10-09T06:26:35.809382+0000","last_fullsized":"2023-10-09T06:26:35.809382+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:51.214385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":262,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812216+0000","last_change":"2023-10-09T06:16:44.960879+0000","last_active":"2023-10-09T06:26:35.812216+0000","last_peered":"2023-10-09T06:26:35.812216+0000","last_clean":"2023-10-09T06:26:35.812216+0000","last_became_active":"2023-10-09T06:16:44.960514+0000","last_became_peered":"2023-10-09T06:16:44.960514+0000","last_unstale":"2023-10-09T06:26:35.812216+0000","last_undegraded":"2023-10-09T06:26:35.812216+0000","last_fullsized":"2023-10-09T06:26:35.812216+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:19:28.446745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"302'1922","reported_seq":1936,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:51.681770+0000","last_change":"2023-10-09T06:26:28.774343+0000","last_active":"2023-10-09T06:26:51.681770+0000","last_peered":"2023-10-09T06:26:51.681770+0000","last_clean":"2023-10-09T06:26:51.681770+0000","last_became_active":"2023-10-09T06:26:28.773068+0000","last_became_peered":"2023-10-09T06:26:28.773068+0000","last_unstale":"2023-10-09T06:26:51.681770+0000","last_undegraded":"2023-10-09T06:26:51.681770+0000","last_fullsized":"2023-10-09T06:26:51.681770+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1922,"log_dups_size":0,"ondisk_log_size":1922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:13.256859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125960192,"num_objects":1922,"num_object_clones":0,"num_object_copies":5766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1922,"num_write_kb":123008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":586,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.804598+0000","last_change":"2023-10-09T06:22:41.675346+0000","last_active":"2023-10-09T06:26:35.804598+0000","last_peered":"2023-10-09T06:26:35.804598+0000","last_clean":"2023-10-09T06:26:35.804598+0000","last_became_active":"2023-10-09T06:21:48.116394+0000","last_became_peered":"2023-10-09T06:21:48.116394+0000","last_unstale":"2023-10-09T06:26:35.804598+0000","last_undegraded":"2023-10-09T06:26:35.804598+0000","last_fullsized":"2023-10-09T06:26:35.804598+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:53.520517+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":262,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814347+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:26:35.814347+0000","last_peered":"2023-10-09T06:26:35.814347+0000","last_clean":"2023-10-09T06:26:35.814347+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:26:35.814347+0000","last_undegraded":"2023-10-09T06:26:35.814347+0000","last_fullsized":"2023-10-09T06:26:35.814347+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":279,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812486+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:26:35.812486+0000","last_peered":"2023-10-09T06:26:35.812486+0000","last_clean":"2023-10-09T06:26:35.812486+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:26:35.812486+0000","last_undegraded":"2023-10-09T06:26:35.812486+0000","last_fullsized":"2023-10-09T06:26:35.812486+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"302'2023","reported_seq":2036,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:52.980274+0000","last_change":"2023-10-09T06:26:28.775744+0000","last_active":"2023-10-09T06:26:52.980274+0000","last_peered":"2023-10-09T06:26:52.980274+0000","last_clean":"2023-10-09T06:26:52.980274+0000","last_became_active":"2023-10-09T06:26:28.775434+0000","last_became_peered":"2023-10-09T06:26:28.775434+0000","last_unstale":"2023-10-09T06:26:52.980274+0000","last_undegraded":"2023-10-09T06:26:52.980274+0000","last_fullsized":"2023-10-09T06:26:52.980274+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":2023,"log_dups_size":0,"ondisk_log_size":2023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:30.642594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2023,"num_write_kb":129472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":283,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811565+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:26:35.811565+0000","last_peered":"2023-10-09T06:26:35.811565+0000","last_clean":"2023-10-09T06:26:35.811565+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:26:35.811565+0000","last_undegraded":"2023-10-09T06:26:35.811565+0000","last_fullsized":"2023-10-09T06:26:35.811565+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":262,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812973+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:26:35.812973+0000","last_peered":"2023-10-09T06:26:35.812973+0000","last_clean":"2023-10-09T06:26:35.812973+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:26:35.812973+0000","last_undegraded":"2023-10-09T06:26:35.812973+0000","last_fullsized":"2023-10-09T06:26:35.812973+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"302'1927","reported_seq":1941,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:52.010665+0000","last_change":"2023-10-09T06:26:28.776736+0000","last_active":"2023-10-09T06:26:52.010665+0000","last_peered":"2023-10-09T06:26:52.010665+0000","last_clean":"2023-10-09T06:26:52.010665+0000","last_became_active":"2023-10-09T06:26:28.776473+0000","last_became_peered":"2023-10-09T06:26:28.776473+0000","last_unstale":"2023-10-09T06:26:52.010665+0000","last_undegraded":"2023-10-09T06:26:52.010665+0000","last_fullsized":"2023-10-09T06:26:52.010665+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1927,"log_dups_size":0,"ondisk_log_size":1927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:05.868023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126287872,"num_objects":1927,"num_object_clones":0,"num_object_copies":5781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1927,"num_write_kb":123328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":582,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.804800+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:26:35.804800+0000","last_peered":"2023-10-09T06:26:35.804800+0000","last_clean":"2023-10-09T06:26:35.804800+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:26:35.804800+0000","last_undegraded":"2023-10-09T06:26:35.804800+0000","last_fullsized":"2023-10-09T06:26:35.804800+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":259,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806359+0000","last_change":"2023-10-09T06:19:06.683262+0000","last_active":"2023-10-09T06:26:35.806359+0000","last_peered":"2023-10-09T06:26:35.806359+0000","last_clean":"2023-10-09T06:26:35.806359+0000","last_became_active":"2023-10-09T06:19:06.682957+0000","last_became_peered":"2023-10-09T06:19:06.682957+0000","last_unstale":"2023-10-09T06:26:35.806359+0000","last_undegraded":"2023-10-09T06:26:35.806359+0000","last_fullsized":"2023-10-09T06:26:35.806359+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:09:49.734186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":234,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803507+0000","last_change":"2023-10-09T06:16:44.960353+0000","last_active":"2023-10-09T06:26:35.803507+0000","last_peered":"2023-10-09T06:26:35.803507+0000","last_clean":"2023-10-09T06:26:35.803507+0000","last_became_active":"2023-10-09T06:16:44.960054+0000","last_became_peered":"2023-10-09T06:16:44.960054+0000","last_unstale":"2023-10-09T06:26:35.803507+0000","last_undegraded":"2023-10-09T06:26:35.803507+0000","last_fullsized":"2023-10-09T06:26:35.803507+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:56.091932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"302'1915","reported_seq":1928,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:51.707341+0000","last_change":"2023-10-09T06:26:28.775845+0000","last_active":"2023-10-09T06:26:51.707341+0000","last_peered":"2023-10-09T06:26:51.707341+0000","last_clean":"2023-10-09T06:26:51.707341+0000","last_became_active":"2023-10-09T06:26:28.775581+0000","last_became_peered":"2023-10-09T06:26:28.775581+0000","last_unstale":"2023-10-09T06:26:51.707341+0000","last_undegraded":"2023-10-09T06:26:51.707341+0000","last_fullsized":"2023-10-09T06:26:51.707341+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1915,"log_dups_size":0,"ondisk_log_size":1915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:22.188138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125501440,"num_objects":1915,"num_object_clones":0,"num_object_copies":5745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1915,"num_write_kb":122560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":524,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814791+0000","last_change":"2023-10-09T06:21:31.860736+0000","last_active":"2023-10-09T06:26:35.814791+0000","last_peered":"2023-10-09T06:26:35.814791+0000","last_clean":"2023-10-09T06:26:35.814791+0000","last_became_active":"2023-10-09T06:21:17.839226+0000","last_became_peered":"2023-10-09T06:21:17.839226+0000","last_unstale":"2023-10-09T06:26:35.814791+0000","last_undegraded":"2023-10-09T06:26:35.814791+0000","last_fullsized":"2023-10-09T06:26:35.814791+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:41.303993+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":285,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803373+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:26:35.803373+0000","last_peered":"2023-10-09T06:26:35.803373+0000","last_clean":"2023-10-09T06:26:35.803373+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:26:35.803373+0000","last_undegraded":"2023-10-09T06:26:35.803373+0000","last_fullsized":"2023-10-09T06:26:35.803373+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":211,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.805701+0000","last_change":"2023-10-09T06:21:45.408765+0000","last_active":"2023-10-09T06:26:35.805701+0000","last_peered":"2023-10-09T06:26:35.805701+0000","last_clean":"2023-10-09T06:26:35.805701+0000","last_became_active":"2023-10-09T06:21:45.408352+0000","last_became_peered":"2023-10-09T06:21:45.408352+0000","last_unstale":"2023-10-09T06:26:35.805701+0000","last_undegraded":"2023-10-09T06:26:35.805701+0000","last_fullsized":"2023-10-09T06:26:35.805701+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:52.140414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":228,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803436+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:26:35.803436+0000","last_peered":"2023-10-09T06:26:35.803436+0000","last_clean":"2023-10-09T06:26:35.803436+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:26:35.803436+0000","last_undegraded":"2023-10-09T06:26:35.803436+0000","last_fullsized":"2023-10-09T06:26:35.803436+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"302'1703","reported_seq":1717,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.724344+0000","last_change":"2023-10-09T06:26:28.773695+0000","last_active":"2023-10-09T06:26:48.724344+0000","last_peered":"2023-10-09T06:26:48.724344+0000","last_clean":"2023-10-09T06:26:48.724344+0000","last_became_active":"2023-10-09T06:26:28.773467+0000","last_became_peered":"2023-10-09T06:26:28.773467+0000","last_unstale":"2023-10-09T06:26:48.724344+0000","last_undegraded":"2023-10-09T06:26:48.724344+0000","last_fullsized":"2023-10-09T06:26:48.724344+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1703,"log_dups_size":0,"ondisk_log_size":1703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:24:33.681958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111607808,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1703,"num_write_kb":108992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":587,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807090+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:26:35.807090+0000","last_peered":"2023-10-09T06:26:35.807090+0000","last_clean":"2023-10-09T06:26:35.807090+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:26:35.807090+0000","last_undegraded":"2023-10-09T06:26:35.807090+0000","last_fullsized":"2023-10-09T06:26:35.807090+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":159,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803575+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:26:35.803575+0000","last_peered":"2023-10-09T06:26:35.803575+0000","last_clean":"2023-10-09T06:26:35.803575+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:26:35.803575+0000","last_undegraded":"2023-10-09T06:26:35.803575+0000","last_fullsized":"2023-10-09T06:26:35.803575+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":263,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812913+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:26:35.812913+0000","last_peered":"2023-10-09T06:26:35.812913+0000","last_clean":"2023-10-09T06:26:35.812913+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:26:35.812913+0000","last_undegraded":"2023-10-09T06:26:35.812913+0000","last_fullsized":"2023-10-09T06:26:35.812913+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":249,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803300+0000","last_change":"2023-10-09T06:16:44.955848+0000","last_active":"2023-10-09T06:26:35.803300+0000","last_peered":"2023-10-09T06:26:35.803300+0000","last_clean":"2023-10-09T06:26:35.803300+0000","last_became_active":"2023-10-09T06:16:44.955199+0000","last_became_peered":"2023-10-09T06:16:44.955199+0000","last_unstale":"2023-10-09T06:26:35.803300+0000","last_undegraded":"2023-10-09T06:26:35.803300+0000","last_fullsized":"2023-10-09T06:26:35.803300+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:54.509256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"302'1580","reported_seq":1594,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.705178+0000","last_change":"2023-10-09T06:26:28.772110+0000","last_active":"2023-10-09T06:26:48.705178+0000","last_peered":"2023-10-09T06:26:48.705178+0000","last_clean":"2023-10-09T06:26:48.705178+0000","last_became_active":"2023-10-09T06:26:28.771837+0000","last_became_peered":"2023-10-09T06:26:28.771837+0000","last_unstale":"2023-10-09T06:26:48.705178+0000","last_undegraded":"2023-10-09T06:26:48.705178+0000","last_fullsized":"2023-10-09T06:26:48.705178+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1580,"log_dups_size":0,"ondisk_log_size":1580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:42.271885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103546880,"num_objects":1580,"num_object_clones":0,"num_object_copies":4740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1580,"num_write_kb":101120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":594,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811943+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:26:35.811943+0000","last_peered":"2023-10-09T06:26:35.811943+0000","last_clean":"2023-10-09T06:26:35.811943+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:26:35.811943+0000","last_undegraded":"2023-10-09T06:26:35.811943+0000","last_fullsized":"2023-10-09T06:26:35.811943+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":268,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.804276+0000","last_change":"2023-10-09T06:21:34.272975+0000","last_active":"2023-10-09T06:26:35.804276+0000","last_peered":"2023-10-09T06:26:35.804276+0000","last_clean":"2023-10-09T06:26:35.804276+0000","last_became_active":"2023-10-09T06:21:34.272689+0000","last_became_peered":"2023-10-09T06:21:34.272689+0000","last_unstale":"2023-10-09T06:26:35.804276+0000","last_undegraded":"2023-10-09T06:26:35.804276+0000","last_fullsized":"2023-10-09T06:26:35.804276+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:31.894517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.802579+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:26:35.802579+0000","last_peered":"2023-10-09T06:26:35.802579+0000","last_clean":"2023-10-09T06:26:35.802579+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:26:35.802579+0000","last_undegraded":"2023-10-09T06:26:35.802579+0000","last_fullsized":"2023-10-09T06:26:35.802579+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":271,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803584+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:26:35.803584+0000","last_peered":"2023-10-09T06:26:35.803584+0000","last_clean":"2023-10-09T06:26:35.803584+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:26:35.803584+0000","last_undegraded":"2023-10-09T06:26:35.803584+0000","last_fullsized":"2023-10-09T06:26:35.803584+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"302'1643","reported_seq":1657,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.728178+0000","last_change":"2023-10-09T06:26:28.773784+0000","last_active":"2023-10-09T06:26:48.728178+0000","last_peered":"2023-10-09T06:26:48.728178+0000","last_clean":"2023-10-09T06:26:48.728178+0000","last_became_active":"2023-10-09T06:26:28.773431+0000","last_became_peered":"2023-10-09T06:26:28.773431+0000","last_unstale":"2023-10-09T06:26:48.728178+0000","last_undegraded":"2023-10-09T06:26:48.728178+0000","last_fullsized":"2023-10-09T06:26:48.728178+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1643,"log_dups_size":0,"ondisk_log_size":1643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:54.972414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1643,"num_write_kb":105152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":235,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.804501+0000","last_change":"2023-10-09T06:21:45.413030+0000","last_active":"2023-10-09T06:26:35.804501+0000","last_peered":"2023-10-09T06:26:35.804501+0000","last_clean":"2023-10-09T06:26:35.804501+0000","last_became_active":"2023-10-09T06:21:45.412769+0000","last_became_peered":"2023-10-09T06:21:45.412769+0000","last_unstale":"2023-10-09T06:26:35.804501+0000","last_undegraded":"2023-10-09T06:26:35.804501+0000","last_fullsized":"2023-10-09T06:26:35.804501+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:28.225852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":575,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811880+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:26:35.811880+0000","last_peered":"2023-10-09T06:26:35.811880+0000","last_clean":"2023-10-09T06:26:35.811880+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:26:35.811880+0000","last_undegraded":"2023-10-09T06:26:35.811880+0000","last_fullsized":"2023-10-09T06:26:35.811880+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.805667+0000","last_change":"2023-10-09T06:21:45.405340+0000","last_active":"2023-10-09T06:26:35.805667+0000","last_peered":"2023-10-09T06:26:35.805667+0000","last_clean":"2023-10-09T06:26:35.805667+0000","last_became_active":"2023-10-09T06:21:45.405044+0000","last_became_peered":"2023-10-09T06:21:45.405044+0000","last_unstale":"2023-10-09T06:26:35.805667+0000","last_undegraded":"2023-10-09T06:26:35.805667+0000","last_fullsized":"2023-10-09T06:26:35.805667+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.594872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":264,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814602+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:26:35.814602+0000","last_peered":"2023-10-09T06:26:35.814602+0000","last_clean":"2023-10-09T06:26:35.814602+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:26:35.814602+0000","last_undegraded":"2023-10-09T06:26:35.814602+0000","last_fullsized":"2023-10-09T06:26:35.814602+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"302'1659","reported_seq":1673,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.723225+0000","last_change":"2023-10-09T06:26:28.778366+0000","last_active":"2023-10-09T06:26:48.723225+0000","last_peered":"2023-10-09T06:26:48.723225+0000","last_clean":"2023-10-09T06:26:48.723225+0000","last_became_active":"2023-10-09T06:26:28.777926+0000","last_became_peered":"2023-10-09T06:26:28.777926+0000","last_unstale":"2023-10-09T06:26:48.723225+0000","last_undegraded":"2023-10-09T06:26:48.723225+0000","last_fullsized":"2023-10-09T06:26:48.723225+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1659,"log_dups_size":0,"ondisk_log_size":1659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:41.094025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1659,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":585,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.802886+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:26:35.802886+0000","last_peered":"2023-10-09T06:26:35.802886+0000","last_clean":"2023-10-09T06:26:35.802886+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:26:35.802886+0000","last_undegraded":"2023-10-09T06:26:35.802886+0000","last_fullsized":"2023-10-09T06:26:35.802886+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":145,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811696+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:26:35.811696+0000","last_peered":"2023-10-09T06:26:35.811696+0000","last_clean":"2023-10-09T06:26:35.811696+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:26:35.811696+0000","last_undegraded":"2023-10-09T06:26:35.811696+0000","last_fullsized":"2023-10-09T06:26:35.811696+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":255,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807006+0000","last_change":"2023-10-09T06:16:44.960166+0000","last_active":"2023-10-09T06:26:35.807006+0000","last_peered":"2023-10-09T06:26:35.807006+0000","last_clean":"2023-10-09T06:26:35.807006+0000","last_became_active":"2023-10-09T06:16:44.959694+0000","last_became_peered":"2023-10-09T06:16:44.959694+0000","last_unstale":"2023-10-09T06:26:35.807006+0000","last_undegraded":"2023-10-09T06:26:35.807006+0000","last_fullsized":"2023-10-09T06:26:35.807006+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:33.703556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807479+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:26:35.807479+0000","last_peered":"2023-10-09T06:26:35.807479+0000","last_clean":"2023-10-09T06:26:35.807479+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:26:35.807479+0000","last_undegraded":"2023-10-09T06:26:35.807479+0000","last_fullsized":"2023-10-09T06:26:35.807479+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"302'1912","reported_seq":1926,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:52.011068+0000","last_change":"2023-10-09T06:26:28.775392+0000","last_active":"2023-10-09T06:26:52.011068+0000","last_peered":"2023-10-09T06:26:52.011068+0000","last_clean":"2023-10-09T06:26:52.011068+0000","last_became_active":"2023-10-09T06:26:28.774522+0000","last_became_peered":"2023-10-09T06:26:28.774522+0000","last_unstale":"2023-10-09T06:26:52.011068+0000","last_undegraded":"2023-10-09T06:26:52.011068+0000","last_fullsized":"2023-10-09T06:26:52.011068+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1912,"log_dups_size":0,"ondisk_log_size":1912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.490573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125304832,"num_objects":1912,"num_object_clones":0,"num_object_copies":5736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1912,"num_write_kb":122368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":524,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814138+0000","last_change":"2023-10-09T06:21:30.865209+0000","last_active":"2023-10-09T06:26:35.814138+0000","last_peered":"2023-10-09T06:26:35.814138+0000","last_clean":"2023-10-09T06:26:35.814138+0000","last_became_active":"2023-10-09T06:21:17.848456+0000","last_became_peered":"2023-10-09T06:21:17.848456+0000","last_unstale":"2023-10-09T06:26:35.814138+0000","last_undegraded":"2023-10-09T06:26:35.814138+0000","last_fullsized":"2023-10-09T06:26:35.814138+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:11.618743+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":145,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812749+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:26:35.812749+0000","last_peered":"2023-10-09T06:26:35.812749+0000","last_clean":"2023-10-09T06:26:35.812749+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:26:35.812749+0000","last_undegraded":"2023-10-09T06:26:35.812749+0000","last_fullsized":"2023-10-09T06:26:35.812749+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":284,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.803207+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:26:35.803207+0000","last_peered":"2023-10-09T06:26:35.803207+0000","last_clean":"2023-10-09T06:26:35.803207+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:26:35.803207+0000","last_undegraded":"2023-10-09T06:26:35.803207+0000","last_fullsized":"2023-10-09T06:26:35.803207+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807009+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:26:35.807009+0000","last_peered":"2023-10-09T06:26:35.807009+0000","last_clean":"2023-10-09T06:26:35.807009+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:26:35.807009+0000","last_undegraded":"2023-10-09T06:26:35.807009+0000","last_fullsized":"2023-10-09T06:26:35.807009+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"302'1617","reported_seq":1628,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:48.158705+0000","last_change":"2023-10-09T06:26:28.777254+0000","last_active":"2023-10-09T06:26:48.158705+0000","last_peered":"2023-10-09T06:26:48.158705+0000","last_clean":"2023-10-09T06:26:48.158705+0000","last_became_active":"2023-10-09T06:26:28.776915+0000","last_became_peered":"2023-10-09T06:26:28.776915+0000","last_unstale":"2023-10-09T06:26:48.158705+0000","last_undegraded":"2023-10-09T06:26:48.158705+0000","last_fullsized":"2023-10-09T06:26:48.158705+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":1617,"log_dups_size":0,"ondisk_log_size":1617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:04.121493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105971712,"num_objects":1617,"num_object_clones":0,"num_object_copies":4851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1617,"num_write_kb":103488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":585,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812385+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:26:35.812385+0000","last_peered":"2023-10-09T06:26:35.812385+0000","last_clean":"2023-10-09T06:26:35.812385+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:26:35.812385+0000","last_undegraded":"2023-10-09T06:26:35.812385+0000","last_fullsized":"2023-10-09T06:26:35.812385+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":292,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806934+0000","last_change":"2023-10-09T06:19:06.770771+0000","last_active":"2023-10-09T06:26:35.806934+0000","last_peered":"2023-10-09T06:26:35.806934+0000","last_clean":"2023-10-09T06:26:35.806934+0000","last_became_active":"2023-10-09T06:19:06.675960+0000","last_became_peered":"2023-10-09T06:19:06.675960+0000","last_unstale":"2023-10-09T06:26:35.806934+0000","last_undegraded":"2023-10-09T06:26:35.806934+0000","last_fullsized":"2023-10-09T06:26:35.806934+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:48.625361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":261,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.802604+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:26:35.802604+0000","last_peered":"2023-10-09T06:26:35.802604+0000","last_clean":"2023-10-09T06:26:35.802604+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:26:35.802604+0000","last_undegraded":"2023-10-09T06:26:35.802604+0000","last_fullsized":"2023-10-09T06:26:35.802604+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":282,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:43.833831+0000","last_change":"2023-10-09T06:19:06.675511+0000","last_active":"2023-10-09T06:26:43.833831+0000","last_peered":"2023-10-09T06:26:43.833831+0000","last_clean":"2023-10-09T06:26:43.833831+0000","last_became_active":"2023-10-09T06:19:06.675150+0000","last_became_peered":"2023-10-09T06:19:06.675150+0000","last_unstale":"2023-10-09T06:26:43.833831+0000","last_undegraded":"2023-10-09T06:26:43.833831+0000","last_fullsized":"2023-10-09T06:26:43.833831+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:04.203385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"302'2089","reported_seq":2102,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-10-09T06:26:53.045671+0000","last_change":"2023-10-09T06:26:28.775931+0000","last_active":"2023-10-09T06:26:53.045671+0000","last_peered":"2023-10-09T06:26:53.045671+0000","last_clean":"2023-10-09T06:26:53.045671+0000","last_became_active":"2023-10-09T06:26:28.775652+0000","last_became_peered":"2023-10-09T06:26:28.775652+0000","last_unstale":"2023-10-09T06:26:53.045671+0000","last_undegraded":"2023-10-09T06:26:53.045671+0000","last_fullsized":"2023-10-09T06:26:53.045671+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":2089,"log_dups_size":0,"ondisk_log_size":2089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:39.386224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136904704,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2089,"num_write_kb":133696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":606,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806119+0000","last_change":"2023-10-09T06:21:48.113246+0000","last_active":"2023-10-09T06:26:35.806119+0000","last_peered":"2023-10-09T06:26:35.806119+0000","last_clean":"2023-10-09T06:26:35.806119+0000","last_became_active":"2023-10-09T06:21:48.112865+0000","last_became_peered":"2023-10-09T06:21:48.112865+0000","last_unstale":"2023-10-09T06:26:35.806119+0000","last_undegraded":"2023-10-09T06:26:35.806119+0000","last_fullsized":"2023-10-09T06:26:35.806119+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:21.118082+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":286,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.814870+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:26:35.814870+0000","last_peered":"2023-10-09T06:26:35.814870+0000","last_clean":"2023-10-09T06:26:35.814870+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:26:35.814870+0000","last_undegraded":"2023-10-09T06:26:35.814870+0000","last_fullsized":"2023-10-09T06:26:35.814870+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":259,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.807344+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:26:35.807344+0000","last_peered":"2023-10-09T06:26:35.807344+0000","last_clean":"2023-10-09T06:26:35.807344+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:26:35.807344+0000","last_undegraded":"2023-10-09T06:26:35.807344+0000","last_fullsized":"2023-10-09T06:26:35.807344+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811690+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:26:35.811690+0000","last_peered":"2023-10-09T06:26:35.811690+0000","last_clean":"2023-10-09T06:26:35.811690+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:26:35.811690+0000","last_undegraded":"2023-10-09T06:26:35.811690+0000","last_fullsized":"2023-10-09T06:26:35.811690+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":261,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806935+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:26:35.806935+0000","last_peered":"2023-10-09T06:26:35.806935+0000","last_clean":"2023-10-09T06:26:35.806935+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:26:35.806935+0000","last_undegraded":"2023-10-09T06:26:35.806935+0000","last_fullsized":"2023-10-09T06:26:35.806935+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":284,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.804720+0000","last_change":"2023-10-09T06:21:45.401008+0000","last_active":"2023-10-09T06:26:35.804720+0000","last_peered":"2023-10-09T06:26:35.804720+0000","last_clean":"2023-10-09T06:26:35.804720+0000","last_became_active":"2023-10-09T06:21:45.400695+0000","last_became_peered":"2023-10-09T06:21:45.400695+0000","last_unstale":"2023-10-09T06:26:35.804720+0000","last_undegraded":"2023-10-09T06:26:35.804720+0000","last_fullsized":"2023-10-09T06:26:35.804720+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:07.675994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":107,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812304+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:26:35.812304+0000","last_peered":"2023-10-09T06:26:35.812304+0000","last_clean":"2023-10-09T06:26:35.812304+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:26:35.812304+0000","last_undegraded":"2023-10-09T06:26:35.812304+0000","last_fullsized":"2023-10-09T06:26:35.812304+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":289,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812840+0000","last_change":"2023-10-09T06:16:44.956492+0000","last_active":"2023-10-09T06:26:35.812840+0000","last_peered":"2023-10-09T06:26:35.812840+0000","last_clean":"2023-10-09T06:26:35.812840+0000","last_became_active":"2023-10-09T06:16:44.956223+0000","last_became_peered":"2023-10-09T06:16:44.956223+0000","last_unstale":"2023-10-09T06:26:35.812840+0000","last_undegraded":"2023-10-09T06:26:35.812840+0000","last_fullsized":"2023-10-09T06:26:35.812840+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:07.028821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":107,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.812570+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:26:35.812570+0000","last_peered":"2023-10-09T06:26:35.812570+0000","last_clean":"2023-10-09T06:26:35.812570+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:26:35.812570+0000","last_undegraded":"2023-10-09T06:26:35.812570+0000","last_fullsized":"2023-10-09T06:26:35.812570+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":279,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.813201+0000","last_change":"2023-10-09T06:16:44.956388+0000","last_active":"2023-10-09T06:26:35.813201+0000","last_peered":"2023-10-09T06:26:35.813201+0000","last_clean":"2023-10-09T06:26:35.813201+0000","last_became_active":"2023-10-09T06:16:44.956080+0000","last_became_peered":"2023-10-09T06:16:44.956080+0000","last_unstale":"2023-10-09T06:26:35.813201+0000","last_undegraded":"2023-10-09T06:26:35.813201+0000","last_fullsized":"2023-10-09T06:26:35.813201+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.508255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":261,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:43.833978+0000","last_change":"2023-10-09T06:16:44.961893+0000","last_active":"2023-10-09T06:26:43.833978+0000","last_peered":"2023-10-09T06:26:43.833978+0000","last_clean":"2023-10-09T06:26:43.833978+0000","last_became_active":"2023-10-09T06:16:44.961572+0000","last_became_peered":"2023-10-09T06:16:44.961572+0000","last_unstale":"2023-10-09T06:26:43.833978+0000","last_undegraded":"2023-10-09T06:26:43.833978+0000","last_fullsized":"2023-10-09T06:26:43.833978+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:05.481140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.808659+0000","last_change":"2023-10-09T06:16:44.953905+0000","last_active":"2023-10-09T06:26:35.808659+0000","last_peered":"2023-10-09T06:26:35.808659+0000","last_clean":"2023-10-09T06:26:35.808659+0000","last_became_active":"2023-10-09T06:16:44.953591+0000","last_became_peered":"2023-10-09T06:16:44.953591+0000","last_unstale":"2023-10-09T06:26:35.808659+0000","last_undegraded":"2023-10-09T06:26:35.808659+0000","last_fullsized":"2023-10-09T06:26:35.808659+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:58.153492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.806931+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:26:35.806931+0000","last_peered":"2023-10-09T06:26:35.806931+0000","last_clean":"2023-10-09T06:26:35.806931+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:26:35.806931+0000","last_undegraded":"2023-10-09T06:26:35.806931+0000","last_fullsized":"2023-10-09T06:26:35.806931+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":262,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-09T06:26:35.811319+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:26:35.811319+0000","last_peered":"2023-10-09T06:26:35.811319+0000","last_clean":"2023-10-09T06:26:35.811319+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:26:35.811319+0000","last_undegraded":"2023-10-09T06:26:35.811319+0000","last_fullsized":"2023-10-09T06:26:35.811319+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1899823104,"num_objects":28989,"num_object_clones":0,"num_object_copies":86967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":28989,"num_write_kb":1855296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5835456512,"data_stored":5835456512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28989,"ondisk_log_size":28989,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":10330,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234354,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507888,"kb_used_data":603560,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":92237264,"statfs":{"total":95995035648,"available":94450958336,"internally_reserved":0,"allocated":618045440,"data_stored":617262388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:51 2023","interfaces":[{"interface":"back","average":{"1min":7.2149999999999999,"5min":8.8960000000000008,"15min":3.8580000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":89.027000000000001,"5min":219.41900000000001,"15min":219.41900000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":7.0759999999999996,"5min":9.2319999999999993,"15min":3.8980000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.218},"max":{"1min":89.135999999999996,"5min":219.399,"15min":219.399},"last":0.28399999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":4.1980000000000004,"5min":3.3839999999999999,"15min":2.8479999999999999},"min":{"1min":0.25,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":49.808,"5min":141.44300000000001,"15min":141.44300000000001},"last":1.026},{"interface":"front","average":{"1min":4.2930000000000001,"5min":3.835,"15min":2.6619999999999999},"min":{"1min":0.33900000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":49.476999999999997,"5min":140.93700000000001,"15min":140.93700000000001},"last":0.71099999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:26:33 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":11.959,"15min":5.1870000000000003},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.2559999999999998,"5min":215.34800000000001,"15min":215.34800000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.1360000000000001,"5min":12.417,"15min":5.407},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.9299999999999997,"5min":215.869,"15min":215.869},"last":1.262}]},{"osd":3,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.873,"5min":2.359,"15min":1.627},"min":{"1min":0.35599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4710000000000001,"5min":50.966999999999999,"15min":50.966999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.6739999999999999,"5min":2.4409999999999998,"15min":1.669},"min":{"1min":0.40500000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.6760000000000002,"5min":51.133000000000003,"15min":51.133000000000003},"last":0.93700000000000006}]},{"osd":4,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":2.0870000000000002,"15min":1.694},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.3620000000000001,"5min":94.105999999999995,"15min":94.105999999999995},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2,"15min":1.419},"min":{"1min":0.315,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.9729999999999999,"5min":93.713999999999999,"15min":93.713999999999999},"last":0.32100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.169,"15min":1.7230000000000001},"min":{"1min":0.33900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.952,"5min":8.9649999999999999,"15min":50.304000000000002},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.151,"15min":1.45},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":4.3280000000000003,"5min":5.7130000000000001,"15min":48.774999999999999},"last":0.42699999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:26:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.458,"15min":1.3580000000000001},"min":{"1min":0.374,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7169999999999996,"5min":8.4890000000000008,"15min":11.878},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.2310000000000001,"15min":1.216},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33000000000000002},"max":{"1min":3.0539999999999998,"5min":9.4900000000000002,"15min":12.141999999999999},"last":0.442}]}]},{"osd":6,"up_from":46,"seq":197568495992,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1936356,"kb_used_data":1025052,"kb_used_omap":0,"kb_used_meta":911296,"kb_avail":91808796,"statfs":{"total":95995035648,"available":94012207104,"internally_reserved":0,"allocated":1049653248,"data_stored":1048882848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933167104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":3.7690000000000001,"15min":3.0790000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":4.2560000000000002,"5min":110.926,"15min":110.926},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.2869999999999999,"5min":3.7480000000000002,"15min":3.2669999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.2859999999999996,"5min":110.911,"15min":110.911},"last":2.1579999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":2.7669999999999999,"15min":3.456},"min":{"1min":0.308,"5min":0.215,"15min":0.215},"max":{"1min":4.4939999999999998,"5min":66.796999999999997,"15min":114.202},"last":1.6739999999999999},{"interface":"front","average":{"1min":1.244,"5min":2.4889999999999999,"15min":3.2810000000000001},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":5.7389999999999999,"5min":66.704999999999998,"15min":114.179},"last":1.331}]},{"osd":2,"last update":"Mon Oct 9 06:26:34 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":3.6549999999999998,"15min":2.7029999999999998},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.5910000000000002,"5min":75.191999999999993,"15min":75.191999999999993},"last":1.7230000000000001},{"interface":"front","average":{"1min":1.6240000000000001,"5min":3.6720000000000002,"15min":2.5950000000000002},"min":{"1min":0.34799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.9699999999999998,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.97699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.25,"15min":2.02},"min":{"1min":0.33300000000000002,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":6.9290000000000003,"5min":110.358,"15min":110.358},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.804,"5min":3.27,"15min":2.069},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":6.9359999999999999,"5min":110.43899999999999,"15min":110.43899999999999},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:26:43 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.0489999999999999,"15min":1.284},"min":{"1min":0.32900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":2.3500000000000001,"5min":7.2060000000000004,"15min":17.643999999999998},"last":1.597},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.333,"15min":1.2410000000000001},"min":{"1min":0.318,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":0.71999999999999997,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.371}]},{"osd":5,"last update":"Mon Oct 9 06:26:43 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.3560000000000001,"15min":1.385},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":3.746,"5min":22.379999999999999,"15min":22.379999999999999},"last":1.639},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.331,"15min":1.371},"min":{"1min":0.36399999999999999,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.98599999999999999,"5min":24.401,"15min":24.401},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:05 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":2.3639999999999999,"15min":1.536},"min":{"1min":0.35399999999999998,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":4.7610000000000001,"5min":100.55800000000001,"15min":100.55800000000001},"last":1.7649999999999999},{"interface":"front","average":{"1min":1.115,"5min":2.343,"15min":1.732},"min":{"1min":0.371,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":4.7510000000000003,"5min":100.995,"15min":100.995},"last":1.8160000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402728,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1662760,"kb_used_data":1153504,"kb_used_omap":4,"kb_used_meta":509243,"kb_avail":92082392,"statfs":{"total":95995035648,"available":94292369408,"internally_reserved":0,"allocated":1181188096,"data_stored":1179883370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":521465742},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":6.7030000000000003,"15min":3.9319999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":5.2389999999999999,"5min":145.96600000000001,"15min":148.98400000000001},"last":0.317},{"interface":"front","average":{"1min":1.77,"5min":7.8129999999999997,"15min":4.468},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":9.6170000000000009,"5min":145.982,"15min":149.02699999999999},"last":0.24199999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":5.4480000000000004,"15min":3.0590000000000002},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.193},"max":{"1min":4.5960000000000001,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.6459999999999999,"5min":6.5979999999999999,"15min":3.8479999999999999},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.219},"max":{"1min":11.522,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:26:06 2023","interfaces":[{"interface":"back","average":{"1min":4.2439999999999998,"5min":4.0350000000000001,"15min":2.585},"min":{"1min":0.248,"5min":0.215,"15min":0.215},"max":{"1min":16.047999999999998,"5min":77.331999999999994,"15min":153.18199999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.8720000000000001,"5min":2.6549999999999998,"15min":1.9319999999999999},"min":{"1min":0.23799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.21,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.51300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":4.6630000000000003,"15min":3.6259999999999999},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.6159999999999997,"5min":99.391000000000005,"15min":199.43799999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.6519999999999999,"5min":4.7789999999999999,"15min":3.569},"min":{"1min":0.38600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.1920000000000002,"5min":99.462000000000003,"15min":199.32900000000001},"last":0.81200000000000006}]},{"osd":5,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":2.355,"15min":2.3820000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32900000000000001},"max":{"1min":4.7450000000000001,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.506,"5min":3.2120000000000002,"15min":2.7829999999999999},"min":{"1min":0.38900000000000001,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":4.7640000000000002,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.34799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":3.669,"15min":2.8519999999999999},"min":{"1min":0.38300000000000001,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":4.883,"5min":77.292000000000002,"15min":77.292000000000002},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.3999999999999999,"5min":3.3319999999999999,"15min":2.6560000000000001},"min":{"1min":0.377,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.6550000000000002,"5min":77.378,"15min":77.378},"last":1.2969999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":3.129,"15min":2.6030000000000002},"min":{"1min":0.41899999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6820000000000004,"5min":77.456999999999994,"15min":77.456999999999994},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.7469999999999999,"15min":2.137},"min":{"1min":0.375,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7009999999999996,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.59899999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024161,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1594084,"kb_used_data":888092,"kb_used_omap":5,"kb_used_meta":705978,"kb_avail":92151068,"statfs":{"total":95995035648,"available":94362693632,"internally_reserved":0,"allocated":909406208,"data_stored":907874418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":722922106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:26:21 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":2.0139999999999998,"15min":2.8359999999999999},"min":{"1min":0.20699999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":9.1820000000000004,"5min":13.916,"15min":185.17400000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.7599999999999998,"5min":2.3879999999999999,"15min":3.3599999999999999},"min":{"1min":0.251,"5min":0.224,"15min":0.17499999999999999},"max":{"1min":9.3369999999999997,"5min":13.945,"15min":186.691},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:26:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5800000000000001,"5min":2.6669999999999998,"15min":2.3610000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":57.162999999999997,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.27700000000000002},{"interface":"front","average":{"1min":4.8360000000000003,"5min":3.0649999999999999,"15min":2.5510000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.20699999999999999},"max":{"1min":57.585000000000001,"5min":57.585000000000001,"15min":57.585000000000001},"last":2.6030000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:25:58 2023","interfaces":[{"interface":"back","average":{"1min":3.2010000000000001,"5min":3.3999999999999999,"15min":2.9140000000000001},"min":{"1min":0.35299999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.2989999999999995,"5min":52.683,"15min":185.89400000000001},"last":6.3819999999999997},{"interface":"front","average":{"1min":2.9950000000000001,"5min":3.4289999999999998,"15min":3.0259999999999998},"min":{"1min":0.37,"5min":0.25600000000000001,"15min":0.219},"max":{"1min":9.3759999999999994,"5min":52.941000000000003,"15min":186.72900000000001},"last":0.76800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":1.5860000000000001,"15min":1.516},"min":{"1min":0.27100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.024,"5min":7.9660000000000002,"15min":13.244},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.833,"15min":1.9410000000000001},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":4.9770000000000003,"5min":8.4890000000000008,"15min":12.752000000000001},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.0990000000000002,"15min":1.9239999999999999},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":3.8450000000000002,"5min":62.984999999999999,"15min":62.984999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.7090000000000001,"15min":2.504},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.26700000000000002},"max":{"1min":5.8650000000000002,"5min":64.983999999999995,"15min":64.983999999999995},"last":1.774}]},{"osd":6,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3559999999999999,"5min":2.282,"15min":2.319},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.1360000000000001,"5min":11.928000000000001,"15min":32.079000000000001},"last":1.7310000000000001},{"interface":"front","average":{"1min":1.498,"5min":1.6299999999999999,"15min":1.8400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.819,"5min":7.9980000000000002,"15min":13.315},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:25:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.5099999999999998,"15min":2.1920000000000002},"min":{"1min":0.40100000000000002,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":4.7279999999999998,"5min":62.262,"15min":62.262},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.202,"5min":2.9910000000000001,"15min":2.5619999999999998},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":7.1059999999999999,"5min":62.229999999999997,"15min":62.229999999999997},"last":1.796}]}]},{"osd":2,"up_from":128,"seq":549755814030,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143312,"kb_used_data":708616,"kb_used_omap":7,"kb_used_meta":434680,"kb_avail":92601840,"statfs":{"total":95995035648,"available":94824284160,"internally_reserved":0,"allocated":725622784,"data_stored":724337724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":445112401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":4.5810000000000004,"15min":3.915},"min":{"1min":0.23499999999999999,"5min":0.183,"15min":0.157},"max":{"1min":4.4189999999999996,"5min":150.32599999999999,"15min":163.40100000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.4670000000000001,"5min":3.5030000000000001,"15min":3.798},"min":{"1min":0.223,"5min":0.182,"15min":0.182},"max":{"1min":7.5590000000000002,"5min":110.66500000000001,"15min":163.363},"last":5.8380000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":4.0359999999999996,"15min":3.6299999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.188},"max":{"1min":6.968,"5min":205.148,"15min":222.50299999999999},"last":1.4239999999999999},{"interface":"front","average":{"1min":1.806,"5min":4.0229999999999997,"15min":3.9199999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":13.77,"5min":205.267,"15min":222.19399999999999},"last":5.9580000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":2.8780000000000001,"15min":1.8140000000000001},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.22700000000000001},"max":{"1min":13.664999999999999,"5min":69.936999999999998,"15min":81.183999999999997},"last":5.9180000000000001},{"interface":"front","average":{"1min":3.0150000000000001,"5min":2.9369999999999998,"15min":1.8700000000000001},"min":{"1min":0.29099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":13.728,"5min":69.906999999999996,"15min":81.061999999999998},"last":5.9379999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.8530000000000002,"15min":2.1440000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":3.6850000000000001,"5min":90.938999999999993,"15min":90.938999999999993},"last":5.798},{"interface":"front","average":{"1min":1.5109999999999999,"5min":3.0009999999999999,"15min":2.4249999999999998},"min":{"1min":0.34599999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.8250000000000002,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.75}]},{"osd":5,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.429,"15min":1.4930000000000001},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.309},"max":{"1min":3.859,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":5.8949999999999996},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.514,"15min":2.0270000000000001},"min":{"1min":0.42799999999999999,"5min":0.33600000000000002,"15min":0.33300000000000002},"max":{"1min":5.8940000000000001,"5min":6.9349999999999996,"15min":10.548},"last":0.80600000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":3.5529999999999999,"15min":2.5230000000000001},"min":{"1min":0.40000000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.8689999999999998,"5min":138.291,"15min":138.291},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.97899999999999998,"5min":3.4129999999999998,"15min":2.157},"min":{"1min":0.47799999999999998,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":3.9980000000000002,"5min":138.01599999999999,"15min":138.01599999999999},"last":0.66800000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.3029999999999999,"15min":1.8360000000000001},"min":{"1min":0.44500000000000001,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":4.9889999999999999,"5min":55.851999999999997,"15min":55.851999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.226,"5min":2.141,"15min":2.2589999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.899,"5min":55.731000000000002,"15min":55.731000000000002},"last":5.8700000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127227,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240440,"kb_used_data":827760,"kb_used_omap":2,"kb_used_meta":412669,"kb_avail":92504712,"statfs":{"total":95995035648,"available":94724825088,"internally_reserved":0,"allocated":847626240,"data_stored":846422643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":422573109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,8],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":7.8659999999999997,"15min":3.262},"min":{"1min":0.19800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":10.367000000000001,"5min":187.852,"15min":187.852},"last":2.7010000000000001},{"interface":"front","average":{"1min":1.23,"5min":8.4719999999999995,"15min":3.7200000000000002},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":4.6539999999999999,"5min":187.923,"15min":187.923},"last":0.29599999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":6.0439999999999996,"15min":2.9500000000000002},"min":{"1min":0.255,"5min":0.246,"15min":0.223},"max":{"1min":11.367000000000001,"5min":187.89099999999999,"15min":187.89099999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.978,"5min":3.673,"15min":1.9379999999999999},"min":{"1min":0.27800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.8810000000000002,"5min":67.028000000000006,"15min":67.028000000000006},"last":1.895}]},{"osd":2,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":5.2350000000000003,"15min":2.649},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.57,"5min":187.76400000000001,"15min":187.76400000000001},"last":4.0250000000000004},{"interface":"front","average":{"1min":2.3210000000000002,"5min":4.8769999999999998,"15min":2.1349999999999998},"min":{"1min":0.23599999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5250000000000004,"5min":187.79499999999999,"15min":187.79499999999999},"last":1.9910000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.544,"15min":1.1000000000000001},"min":{"1min":0.29999999999999999,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":6.7830000000000004,"5min":22.585999999999999,"15min":22.585999999999999},"last":1.9410000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.2389999999999999,"15min":1.964},"min":{"1min":0.39400000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.9649999999999999,"5min":187.67400000000001,"15min":187.67400000000001},"last":7.9089999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":3.2530000000000001,"15min":2.1080000000000001},"min":{"1min":0.44500000000000001,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":3.0059999999999998,"5min":187.833,"15min":187.833},"last":7.9429999999999996},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.0649999999999999,"15min":2.036},"min":{"1min":0.33200000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8319999999999999,"5min":187.952,"15min":187.952},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.6080000000000001,"15min":2.1560000000000001},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.6280000000000001,"5min":187.78200000000001,"15min":187.78200000000001},"last":7.8179999999999996},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1890000000000001,"15min":1.9910000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8719999999999999,"5min":187.81800000000001,"15min":187.81800000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":3.5169999999999999,"15min":1.8320000000000001},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":6.8239999999999998,"5min":192.20400000000001,"15min":192.20400000000001},"last":1.9690000000000001},{"interface":"front","average":{"1min":1.242,"5min":3.4580000000000002,"15min":2.093},"min":{"1min":0.41899999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":5.431,"5min":187.86500000000001,"15min":187.86500000000001},"last":7.8730000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019264,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445220,"kb_used_data":562524,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":92299932,"statfs":{"total":95995035648,"available":94515130368,"internally_reserved":0,"allocated":576024576,"data_stored":575253455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":3.6909999999999998,"15min":2.2240000000000002},"min":{"1min":0.255,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":37.485999999999997,"5min":76.944999999999993,"15min":76.944999999999993},"last":37.485999999999997},{"interface":"front","average":{"1min":2.5419999999999998,"5min":3.9470000000000001,"15min":2.6360000000000001},"min":{"1min":0.33500000000000002,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":37.502000000000002,"5min":76.988,"15min":76.988},"last":37.502000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.98399999999999999,"15min":2.016},"min":{"1min":0.27800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.7519999999999998,"5min":5.0190000000000001,"15min":91.849999999999994},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.381,"5min":1.359,"15min":2.1579999999999999},"min":{"1min":0.311,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":5.0979999999999999,"5min":9.625,"15min":91.885000000000005},"last":0.35399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":3.5089999999999999,"15min":2.3359999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":47.125,"5min":69.640000000000001,"15min":69.640000000000001},"last":2.6339999999999999},{"interface":"front","average":{"1min":5.9249999999999998,"5min":3.794,"15min":2.5379999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28399999999999997},"max":{"1min":47.151000000000003,"5min":69.676000000000002,"15min":69.676000000000002},"last":2.9209999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.7029999999999998,"15min":2.2879999999999998},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.8360000000000003,"5min":45.591000000000001,"15min":91.697000000000003},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.341,"5min":2.3140000000000001,"15min":2.129},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":3.5230000000000001,"5min":45.548999999999999,"15min":94.356999999999999},"last":0.38600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.95799999999999996,"15min":1.1419999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.24299999999999999},"max":{"1min":6.2670000000000003,"5min":9.1720000000000006,"15min":10.673},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.91500000000000004,"15min":1.228},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.536,"5min":9.6379999999999999,"15min":12.552},"last":0.79300000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.60999999999999999,"15min":0.84899999999999998},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26500000000000001},"max":{"1min":0.83799999999999997,"5min":2.1179999999999999,"15min":9.7680000000000007},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.875,"15min":1.2889999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2050000000000001,"5min":8.859,"15min":15.321999999999999},"last":0.434}]},{"osd":7,"last update":"Mon Oct 9 06:26:03 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.7350000000000001,"15min":1.476},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":2.1880000000000002,"5min":62.658000000000001,"15min":62.658000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.876,"5min":1.706,"15min":1.488},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":3.8450000000000002,"5min":62.573,"15min":62.573},"last":0.48599999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757628,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":822476800,"data_stored":822476800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1154809856,"data_stored":1154809856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":754057216,"data_stored":754057216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":865730560,"data_stored":865730560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":531890176,"data_stored":531890176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1072431104,"data_stored":1072431104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":634060800,"data_stored":634060800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:26:56.576 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:26:56.576 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:26:56.576 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T06:26:56.576 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:26:56.576 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:26:56.576 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T06:26:56.597 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:26:56.690 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:56.783 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 37685 37669 112 80.75 0.0114099 0.00892281 2023-10-09T06:26:56.944 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:56 smithi107 ceph-mon[212750]: pgmap v937: 99 pgs: 99 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:26:56.944 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:56 smithi107 ceph-mon[220921]: pgmap v937: 99 pgs: 99 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:26:56.966 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:26:57.086 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:26:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:57.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:26:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:56 smithi150 ceph-mon[194178]: pgmap v937: 99 pgs: 99 active+clean; 2.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-09T06:26:57.194 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:57.258 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:57.389 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:57.417 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:57.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:57.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:57.510 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:26:57.510 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:26:57.521 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:26:57.555 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:57.672 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:26:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:57.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:26:57.778 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:26:57.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 39042 39026 110.756 84.8125 0.0066368 0.00902254 2023-10-09T06:26:57.815 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:58.694 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:26:58.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 40234 40218 109.181 74.5 0.00928483 0.00915345 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[212750]: pgmap v938: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3161626695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[220921]: pgmap v938: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3161626695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:58 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:58 smithi150 ceph-mon[194178]: pgmap v938: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T06:26:59.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3161626695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:58 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:26:59.378 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:26:59.384 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:26:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:59.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:26:59.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:26:59.528 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:59.620 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:26:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:26:59.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.689 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:59.690 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:59.726 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:26:59.785 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:26:59.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 41402 41386 107.674 73 0.0147361 0.00927766 2023-10-09T06:26:59.865 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:26:59.869 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:26:59.869 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:26:59.869 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: failed to encode map e303 with expected crc 2023-10-09T06:26:59.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:26:59.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:26:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:00.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: osdmap e303: 8 total, 8 up, 7 in 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: failed to encode map e303 with expected crc 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: failed to encode map e303 with expected crc 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: failed to encode map e303 with expected crc 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: failed to encode map e303 with expected crc 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: failed to encode map e303 with expected crc 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:00.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:26:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:00.267 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:00.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:00.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:00.377 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:00.406 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:00.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 42411 42395 105.891 63.0625 0.00705939 0.00943733 2023-10-09T06:27:00.796 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:00.830 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:00.832 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:00.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:00.879 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:00.919 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:00.935 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:00.954 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:00.973 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:00 smithi150 ceph-mon[194178]: pgmap v940: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:27:00.974 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:00.974 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:00.974 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:00.982 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:00.998 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:01.038 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:01.078 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.085 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:27:01.105 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:01.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:01.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[212750]: pgmap v940: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:27:01.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:01.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:01.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:01.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[220921]: pgmap v940: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T06:27:01.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:27:01.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:27:01.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:01.210 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:01.246 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:01.366 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:01.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:01.470 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:01.544 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:01.659 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:01.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:01.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:01.765 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:01.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 43865 43849 105.314 90.875 0.00915283 0.00949051 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/211264142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/211264142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:01.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:01.833 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:01.980 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/211264142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:01.980 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:01.980 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:01.986 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:01.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:01.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:02.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:02.138 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:02.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:02] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.43.0" 2023-10-09T06:27:02.230 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:02.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:02.334 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:02.381 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:02.405 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:27:02.496 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:02.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:02.588 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:45048 11.5 11:a5bea26d:::benchmark_data_smithi107_418363_object45047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:02.583573+0000", 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.0034545919999999998, 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034833720000000002, 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:27:02.589 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 45048 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.583573+0000", 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:27:02.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.583573+0000", 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.583571+0000", 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584813+0000", 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0012411080000000001 2023-10-09T06:27:02.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584825+0000", 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2852e-05 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584832+0000", 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1659999999999999e-06 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584952+0000", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012037800000000001 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.585070+0000", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011763 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.585206+0000", 2023-10-09T06:27:02.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013636900000000001 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586859+0000", 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0016528910000000001 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:45046 11.3 e303/297 11:c5836dcf:::benchmark_data_smithi107_418363_object45045:head v 303'2904, mlcod=303'2902)", 2023-10-09T06:27:02.594 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:02.584862+0000", 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.0021656140000000002, 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002251627, 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584862+0000", 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584862+0000", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584862+0000", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584881+0000", 2023-10-09T06:27:02.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9426e-05 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584888+0000", 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3900000000000004e-06 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.584893+0000", 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4150000000000001e-06 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.585560+0000", 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00066724500000000001 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.585579+0000", 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8991000000000002e-05 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:45051 11.4 11:22663cfc:::benchmark_data_smithi107_418363_object45050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:02.586678+0000", 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00034984799999999999, 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000477706, 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 45051 2023-10-09T06:27:02.599 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586678+0000", 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586678+0000", 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586677+0000", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586747+0000", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0548000000000003e-05 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586763+0000", 2023-10-09T06:27:02.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5985999999999999e-05 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586770+0000", 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1969999999999999e-06 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586833+0000", 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3046000000000006e-05 2023-10-09T06:27:02.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586966+0000", 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013211000000000001 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.587096+0000", 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000130328 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:27:02.603 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:45052 11.f 11:fb126fc0:::benchmark_data_smithi107_418363_object45051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:02.586791+0000", 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00023641499999999999, 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045302500000000002, 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:27:02.604 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 45052 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586791+0000", 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586791+0000", 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586791+0000", 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586818+0000", 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7347000000000001e-05 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586854+0000", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6128999999999999e-05 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586864+0000", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9699999999999994e-06 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586911+0000", 2023-10-09T06:27:02.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6978000000000003e-05 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.587028+0000", 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116428 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.587137+0000", 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010949499999999999 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.608 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:45049 11.3 e303/297 11:c43659dd:::benchmark_data_smithi107_418363_object45048:head v 303'2905, mlcod=303'2902)", 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:02.586898+0000", 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00012986599999999999, 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00039255600000000001, 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.609 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586898+0000", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586898+0000", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586897+0000", 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:02.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586921+0000", 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4340000000000001e-05 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586929+0000", 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4089999999999997e-06 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.586934+0000", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1619999999999999e-06 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.587194+0000", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00026036199999999998 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:02.587211+0000", 2023-10-09T06:27:02.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6484999999999999e-05 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: ], 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 5 2023-10-09T06:27:02.613 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:27:02.613 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:02.689 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:02.785 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 45286 45270 104.703 88.8125 0.0123581 0.00954385 2023-10-09T06:27:02.794 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:02 smithi150 ceph-mon[194178]: pgmap v941: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:27:02.799 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:02.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:02.913 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:02.947 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:03.001 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:03.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:03.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:03.062 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:03.105 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:03.131 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:03.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:02 smithi107 ceph-mon[212750]: pgmap v941: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:27:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:02 smithi107 ceph-mon[220921]: pgmap v941: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T06:27:03.172 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:03.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:03.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:03.276 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:03.310 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:03.615 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:27:03.615 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:03.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:03.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:03.719 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:03.776 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:03.787 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 46394 46378 103.432 69.25 0.0116954 0.00966159 2023-10-09T06:27:03.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:03.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:03.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:03.998 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.064 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:04.123 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:27:04.195 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:04.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:04.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:04.295 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:27:04.301 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:04.327 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:04.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:04.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:04.531 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.619 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:04.789 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 47478 47462 102.198 67.75 0.00977533 0.00977895 2023-10-09T06:27:04.792 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:04.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:04 smithi107 ceph-mon[212750]: pgmap v942: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:27:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:04 smithi107 ceph-mon[220921]: pgmap v942: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:27:04.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:04.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:04 smithi150 ceph-mon[194178]: pgmap v942: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:27:04.954 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:05.063 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:05.071 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:05.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:05.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:05.175 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:05.520 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:05.619 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:05.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:05.731 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:05.771 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:05.789 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 48468 48452 100.854 61.875 0.0167743 0.00990912 2023-10-09T06:27:05.846 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:05.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:05.953 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:06.007 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:06.007 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:06.007 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:27:06.008 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.978799 2023-10-09T06:27:06.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0.9787992486263793 2023-10-09T06:27:06.015 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:06.077 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:27:06.077 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:27:06.077 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:27:06.077 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:27:06.077 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:27:06.166 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:06.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:06.272 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:06.306 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:06.348 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:06.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:06.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:06.452 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:06.478 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:06.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:06 smithi107 ceph-mon[212750]: pgmap v943: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:27:06.735 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:06.772 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:27:06.788 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 49346 49330 99.3721 54.875 0.00446716 0.010057 2023-10-09T06:27:06.875 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:06.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:06.981 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:06.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:06 smithi150 ceph-mon[194178]: pgmap v943: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:27:06.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:07.016 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:27:07.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:06 smithi107 ceph-mon[220921]: pgmap v943: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T06:27:07.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:07.029 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:07.125 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:07.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:07.231 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:07.258 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:07.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:07.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:07.418 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:07.470 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:07.537 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:07.539 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:27:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:07.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:07.643 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:07.737 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:07.789 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 50535 50519 98.5894 74.3125 0.0110128 0.010136 2023-10-09T06:27:07.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:07 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:07.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:07.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:07 smithi150 ceph-mon[194178]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T06:27:07.817 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:07 smithi150 ceph-mon[194178]: failed to encode map e304 with expected crc 2023-10-09T06:27:07.894 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:07.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[220921]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[220921]: failed to encode map e304 with expected crc 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3810796436' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:07.985 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[212750]: osdmap e304: 8 total, 8 up, 7 in 2023-10-09T06:27:07.986 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:07 smithi107 ceph-mon[212750]: failed to encode map e304 with expected crc 2023-10-09T06:27:08.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:08.073 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:08.081 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:08.190 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:08.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:08.240 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:08.274 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:08.297 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:08.309 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:08.325 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:08.328 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:08.351 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:08.371 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:08.372 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:08.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:08.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:08.475 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:08.501 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:08.556 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:08.557 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:08.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:08.597 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:08.609 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.610 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:27:08.661 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:08.693 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:08.789 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 51451 51435 97.3374 57.25 0.00402083 0.0102675 2023-10-09T06:27:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:08 smithi107 ceph-mon[212750]: pgmap v945: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:27:08.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3357283269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:08 smithi107 ceph-mon[220921]: pgmap v945: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:27:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3357283269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:08.932 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:08.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:08.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:09.038 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:09.097 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:08 smithi150 ceph-mon[194178]: pgmap v945: 99 pgs: 99 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T06:27:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3357283269' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:09.460 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:09.863 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:09.863 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:27:09.864 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 52549 52533 96.4916 68.625 0.0291087 0.0103581 2023-10-09T06:27:09.901 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:10.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:10.683 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:27:11.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:10 smithi107 ceph-mon[212750]: pgmap v946: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:27:11.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:10 smithi107 ceph-mon[220921]: pgmap v946: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:27:11.021 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 53523 53507 95.4704 60.875 0.00695424 0.0104684 2023-10-09T06:27:11.029 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:11.029 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:11.062 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:10 smithi150 ceph-mon[194178]: pgmap v946: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:27:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:11.334 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:11.334 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:11.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:11.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:11.543 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:11.816 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:11.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:11.819 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 54619 54603 94.7155 68.5 0.0130704 0.010552 2023-10-09T06:27:11.819 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.3 primary-affinity to 0.978799 (8641462) 2023-10-09T06:27:11.920 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:12.066 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3717379411' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.110 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.111 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:12.132 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:12.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:12.163 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:12] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.43.0" 2023-10-09T06:27:12.163 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3717379411' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.163 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.163 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3717379411' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.164 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]: dispatch 2023-10-09T06:27:12.263 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:12.290 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:12.332 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:12.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:12.439 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:12.478 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:12.793 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[212750]: pgmap v947: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]': finished 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[212750]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[212750]: failed to encode map e305 with expected crc 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[220921]: pgmap v947: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[220921]: failed to encode map e305 with expected crc 2023-10-09T06:27:12.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]': finished 2023-10-09T06:27:12.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[220921]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T06:27:12.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:12 smithi107 ceph-mon[220921]: failed to encode map e305 with expected crc 2023-10-09T06:27:12.795 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 55601 55585 93.815 61.375 0.0052023 0.0106545 2023-10-09T06:27:13.021 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:27:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:12 smithi150 ceph-mon[194178]: pgmap v947: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T06:27:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:12 smithi150 ceph-mon[194178]: failed to encode map e305 with expected crc 2023-10-09T06:27:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9787992486263793}]': finished 2023-10-09T06:27:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:12 smithi150 ceph-mon[194178]: osdmap e305: 8 total, 8 up, 7 in 2023-10-09T06:27:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:12 smithi150 ceph-mon[194178]: failed to encode map e305 with expected crc 2023-10-09T06:27:13.429 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:13.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:13.543 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:13.572 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:13.622 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:13.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:13.665 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:13.730 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:13.761 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:13.794 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 56628 56612 93.0357 64.1875 0.00720494 0.0107434 2023-10-09T06:27:13.808 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:13.816 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:13.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:13.922 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:13.954 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:14.011 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:14.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:14.117 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:14.214 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:14.296 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:14.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:14.402 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:14.433 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:14.489 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:14.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:14.594 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:14.670 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:14.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 57715 57699 92.3858 67.9375 0.0123981 0.0108185 2023-10-09T06:27:14.800 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[212750]: pgmap v949: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:27:14.800 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1998644370' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:14.800 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:14.838 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:27:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[220921]: pgmap v949: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:27:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1998644370' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:15.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:15.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:14 smithi150 ceph-mon[194178]: pgmap v949: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:27:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1998644370' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:15.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:27:15.797812+0000 min lat: 0.00108834 max lat: 0.325949 avg lat: 0.0109107 2023-10-09T06:27:15.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:27:15.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 58698 58682 91.6123 61.4375 0.0109923 0.0109107 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[212750]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.055 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:15 smithi107 ceph-mon[220921]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: osdmap e306: 8 total, 8 up, 7 in 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:15 smithi150 ceph-mon[194178]: failed to encode map e306 with expected crc 2023-10-09T06:27:16.741 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:16.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:16.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 59729 59713 90.9491 64.4375 0.00935114 0.0109904 2023-10-09T06:27:16.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[212750]: pgmap v951: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:27:16.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:16.848 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:17.019 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:17.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:16 smithi150 ceph-mon[194178]: pgmap v951: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:27:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[220921]: pgmap v951: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:27:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:17.286 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:17.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:17.403 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:17.432 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:17.487 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:17.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:17.592 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:17.673 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:17.789 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:17.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:17.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:17.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 60824 60808 90.4133 68.4375 0.0107023 0.0110553 2023-10-09T06:27:17.884 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:17.884 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:17.898 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:17.963 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:18.093 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:18.098 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:18.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:18.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:18.140 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:18.185 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:18.205 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:18.240 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:18.282 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:18.292 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:18.329 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:18.369 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:18.396 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:18.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:18.400 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:18.400 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.400 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.401 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:27:18.501 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:18.527 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:18.555 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:18.555 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:18.555 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 6, 7] 2023-10-09T06:27:18.555 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T06:27:18.569 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:18.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:18.674 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:18.710 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:18.769 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:18.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:18.797 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 61647 61631 89.5075 51.4375 0.0176642 0.0111676 2023-10-09T06:27:18.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:18 smithi107 ceph-mon[212750]: pgmap v952: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2023-10-09T06:27:18.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3251882024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:18.877 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:18.964 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:18 smithi107 ceph-mon[220921]: pgmap v952: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2023-10-09T06:27:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3251882024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:19.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:18 smithi150 ceph-mon[194178]: pgmap v952: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 55 MiB/s wr, 880 op/s 2023-10-09T06:27:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3251882024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:19.202 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:27:19.482 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:19.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:19.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:27:19.586 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:19.680 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:19.798 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 62543 62527 88.7451 56 0.0270146 0.0112628 2023-10-09T06:27:19.831 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:19.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:19.944 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:19.972 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:20.015 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:20.033 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:20.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:20.084 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:20.141 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:20.228 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:20.419 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:20.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:20.536 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:20.573 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:20.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:20.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:20.749 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:20.804 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 63582 63566 88.2126 64.9375 0.0147086 0.0113302 2023-10-09T06:27:20.838 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:20 smithi107 ceph-mon[212750]: pgmap v953: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:27:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:20 smithi107 ceph-mon[220921]: pgmap v953: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:27:21.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:20 smithi150 ceph-mon[194178]: pgmap v953: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:27:21.793 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:27:21.800 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 64596 64580 87.6729 63.375 0.0105958 0.0113948 2023-10-09T06:27:22.154 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:22] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T06:27:22.433 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:22.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:22.537 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:22.573 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:22.639 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:22.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:22.744 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:22.801 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 65543 65527 87.0654 59.1875 0.0151663 0.0114803 2023-10-09T06:27:22.812 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:22.861 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:22.969 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:22.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:22.987 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:23.084 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:22 smithi107 ceph-mon[212750]: pgmap v954: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:27:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:22 smithi107 ceph-mon[220921]: pgmap v954: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:27:23.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:22 smithi150 ceph-mon[194178]: pgmap v954: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:27:23.154 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:23.329 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:23.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:23.436 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:23.506 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:23.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:23.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:23.762 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:23.790 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:23.801 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 66501 66485 86.4992 59.875 0.0139773 0.0115554 2023-10-09T06:27:23.846 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:23.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:23.952 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:23.987 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:24.041 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:24.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:24.146 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:24.228 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:24.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 67293 67277 85.7444 49.5 0.00456775 0.0116404 2023-10-09T06:27:24.873 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 conmon[212706]: 2023-10-09T06:27:24.573+0000 7f7c794f9700 -1 mon.a@0(leader).osd e306 definitely_dead 0 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[212750]: pgmap v955: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3752193580' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[212750]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[220921]: pgmap v955: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:24.874 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3752193580' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:24.875 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:24.875 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:24 smithi107 ceph-mon[220921]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:27:24.885 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:27:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:24 smithi150 ceph-mon[194178]: pgmap v955: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1k op/s 2023-10-09T06:27:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:27:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3752193580' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T06:27:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:24 smithi150 ceph-mon[194178]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T06:27:25.610 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.3. 2023-10-09T06:27:25.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 67312 67296 84.0544 1.1875 0.0210774 0.0116601 2023-10-09T06:27:26.113 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:26.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:26.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:26.115 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: osdmap e307: 8 total, 8 up, 6 in 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[212750]: osdmap e308: 8 total, 8 up, 6 in 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:26.116 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:27:26.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: osdmap e307: 8 total, 8 up, 6 in 2023-10-09T06:27:26.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:25 smithi107 ceph-mon[220921]: osdmap e308: 8 total, 8 up, 6 in 2023-10-09T06:27:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:26.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1889969542' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: osdmap e307: 8 total, 8 up, 6 in 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: failed to encode map e307 with expected crc 2023-10-09T06:27:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:25 smithi150 ceph-mon[194178]: osdmap e308: 8 total, 8 up, 6 in 2023-10-09T06:27:26.218 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:26.251 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:26.300 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:26.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:26.406 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:26.434 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:26.581 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:26.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:26.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:26.685 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:26.723 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:26.773 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:26.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:26.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 67312 67296 82.4073 0 - 0.0116601 2023-10-09T06:27:26.880 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: pgmap v957: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[212750]: osdmap e309: 8 total, 8 up, 6 in 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: pgmap v957: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.962 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: failed to encode map e308 with expected crc 2023-10-09T06:27:26.963 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:26 smithi107 ceph-mon[220921]: osdmap e309: 8 total, 8 up, 6 in 2023-10-09T06:27:26.972 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: pgmap v957: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: failed to encode map e308 with expected crc 2023-10-09T06:27:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:26 smithi150 ceph-mon[194178]: osdmap e309: 8 total, 8 up, 6 in 2023-10-09T06:27:27.228 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:27.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:27.345 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:27.391 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:27.454 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:27.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:27.492 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:27.519 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:27.546 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:27.565 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:27.581 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:27.585 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:27.585 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:27.598 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:27.613 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:27.636 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:27.652 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:27.655 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:27.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:27.689 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:27.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.690 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.691 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:27:27.762 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:27.788 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:27.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 67540 67524 81.0974 7.125 0.00555151 0.0121504 2023-10-09T06:27:27.833 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:27.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:27.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:27.967 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:28.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: osdmap e310: 8 total, 8 up, 6 in 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/806262285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: osdmap e310: 8 total, 8 up, 6 in 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/806262285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: osdmap e310: 8 total, 8 up, 6 in 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: failed to encode map e310 with expected crc 2023-10-09T06:27:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/806262285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:28.577 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:27:28.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 67540 67524 79.5683 0 - 0.0121504 2023-10-09T06:27:28.954 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:28.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:29.017 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:27:29.057 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:29.085 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:29.130 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:29.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:29.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: pgmap v960: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 7 peering, 88 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s; 8468/199830 objects degraded (4.238%) 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 8468/199830 objects degraded (4.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: osdmap e311: 8 total, 8 up, 6 in 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: pgmap v960: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 7 peering, 88 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s; 8468/199830 objects degraded (4.238%) 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 8468/199830 objects degraded (4.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: osdmap e311: 8 total, 8 up, 6 in 2023-10-09T06:27:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:28 smithi107 ceph-mon[220921]: failed to encode map e311 with expected crc 2023-10-09T06:27:29.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: failed to encode map e309 with expected crc 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: pgmap v960: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 7 peering, 88 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s; 8468/199830 objects degraded (4.238%) 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: failed to encode map e310 with expected crc 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 8468/199830 objects degraded (4.238%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: osdmap e311: 8 total, 8 up, 6 in 2023-10-09T06:27:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:28 smithi150 ceph-mon[194178]: failed to encode map e311 with expected crc 2023-10-09T06:27:29.237 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:29.268 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:29.358 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:29.374 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:29.389 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:29.478 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.507 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:29.555 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:29.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:29.661 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:29.688 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:29.738 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:29.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:29.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:29.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 67540 67524 78.0958 0 - 0.0121504 2023-10-09T06:27:29.843 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:29.872 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:30.020 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:30.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:30.124 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:30.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[212750]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[220921]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[220921]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:29 smithi107 ceph-mon[220921]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.152 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:30.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:29 smithi150 ceph-mon[194178]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:29 smithi150 ceph-mon[194178]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:29 smithi150 ceph-mon[194178]: failed to encode map e311 with expected crc 2023-10-09T06:27:30.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 67540 67524 76.6768 0 - 0.0121504 2023-10-09T06:27:30.856 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:30.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:30.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:30.959 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:30.990 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:31.035 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:31.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:31.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:31.139 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:31.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:30 smithi107 ceph-mon[212750]: pgmap v963: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 8 peering, 85 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 24 MiB/s wr, 404 op/s; 8469/201096 objects degraded (4.211%); 4219/201096 objects misplaced (2.098%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:27:31.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:30 smithi107 ceph-mon[220921]: pgmap v963: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 8 peering, 85 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 24 MiB/s wr, 404 op/s; 8469/201096 objects degraded (4.211%); 4219/201096 objects misplaced (2.098%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:27:31.309 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:27:31.309 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:30 smithi150 ceph-mon[194178]: pgmap v963: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 8 peering, 85 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 21 KiB/s rd, 24 MiB/s wr, 404 op/s; 8469/201096 objects degraded (4.211%); 4219/201096 objects misplaced (2.098%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:27:31.338 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:31.447 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:31.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:31.554 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:31.588 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:31.654 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:31.654 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:31.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:31.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:31.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 67540 67524 75.3083 0 - 0.0121504 2023-10-09T06:27:31.803 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:31.830 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:31.904 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:31.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:31.961 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7] out_osds: [5, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:31.961 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:31.961 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T06:27:31.962 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T06:27:32.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:32.043 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:32.292 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:32.294 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:32] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T06:27:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:32.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:32.364 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:32.384 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:27:32.396 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:32.423 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:32.486 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:32.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:32.509 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:32.591 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:32.612 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:27:32.619 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:32.666 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:32.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:32.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:32.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 67540 67524 73.988 0 - 0.0121504 2023-10-09T06:27:32.804 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:32.852 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: pgmap v964: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 255 op/s; 8469/201135 objects degraded (4.211%); 4219/201135 objects misplaced (2.098%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:27:32.852 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:32.852 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:32.852 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2752267323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: pgmap v964: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 255 op/s; 8469/201135 objects degraded (4.211%); 4219/201135 objects misplaced (2.098%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2752267323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:32 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: pgmap v964: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 255 op/s; 8469/201135 objects degraded (4.211%); 4219/201135 objects misplaced (2.098%); 832 KiB/s, 13 objects/s recovering 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2752267323' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:32 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T06:27:33.500 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:33.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:33.607 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:33.634 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:33.683 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:33.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:33.790 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:33.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 67540 67524 72.713 0 - 0.0121504 2023-10-09T06:27:33.818 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:33.866 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:33.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:33.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:33.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: osdmap e312: 8 total, 8 up, 6 in 2023-10-09T06:27:33.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.869 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.870 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:33 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:33.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:33.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: osdmap e312: 8 total, 8 up, 6 in 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: osdmap e312: 8 total, 8 up, 6 in 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:33 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:33.971 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:34.001 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:34.549 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:34.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:34.601 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:34.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:34.687 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:34.802 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 67540 67524 71.4813 0 - 0.0121504 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: pgmap v966: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 14 KiB/s rd, 5.6 MiB/s wr, 99 op/s; 12663/201426 objects degraded (6.287%); 4219/201426 objects misplaced (2.095%); 780 KiB/s, 12 objects/s recovering 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12663/201426 objects degraded (6.287%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: pgmap v966: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 14 KiB/s rd, 5.6 MiB/s wr, 99 op/s; 12663/201426 objects degraded (6.287%); 4219/201426 objects misplaced (2.095%); 780 KiB/s, 12 objects/s recovering 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:34 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12663/201426 objects degraded (6.287%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: pgmap v966: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 92 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 14 KiB/s rd, 5.6 MiB/s wr, 99 op/s; 12663/201426 objects degraded (6.287%); 4219/201426 objects misplaced (2.095%); 780 KiB/s, 12 objects/s recovering 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: failed to encode map e312 with expected crc 2023-10-09T06:27:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:34 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12663/201426 objects degraded (6.287%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T06:27:35.105 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:35.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:35.212 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:35.242 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:35.301 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:35 smithi107 conmon[212706]: 2023-10-09T06:27:35.184+0000 7f7c794f9700 -1 mon.a@0(leader).osd e312 definitely_dead 0 2023-10-09T06:27:35.628 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.3. 2023-10-09T06:27:35.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:27:35.803601+0000 min lat: 0.00108834 max lat: 2.55426 avg lat: 0.0121504 2023-10-09T06:27:35.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:27:35.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 67540 67524 70.2907 0 - 0.0121504 2023-10-09T06:27:35.892 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:35.892 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:35.893 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:35.893 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:35.893 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:35.893 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:35.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:27:35.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:27:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T06:27:36.041 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:36.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:36.125 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:36.143 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:36.144 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.144 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.144 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.144 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.144 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:27:36.145 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:36.170 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:36.213 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:36.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:36.344 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:36.370 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:36.412 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:36.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:36.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:36.517 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:36.524 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:27:36.548 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:36.803 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 67540 67524 69.139 0 - 0.0121504 2023-10-09T06:27:36.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: pgmap v967: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 75 op/s; 12663/201426 objects degraded (6.287%); 3828/201426 objects misplaced (1.900%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1594497664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: pgmap v967: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 75 op/s; 12663/201426 objects degraded (6.287%); 3828/201426 objects misplaced (1.900%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1594497664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: pgmap v967: 99 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 75 op/s; 12663/201426 objects degraded (6.287%); 3828/201426 objects misplaced (1.900%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3442130945' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: osdmap e313: 8 total, 8 up, 7 in 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: failed to encode map e313 with expected crc 2023-10-09T06:27:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1594497664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:37.161 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T06:27:37.198 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:37.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:37.229 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:27:37.229 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:27:37.229 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:27:37.229 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:27:37.302 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:37.333 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:37.478 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:37.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:37.582 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:37.610 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:37.700 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:37.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:37.804 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:37.808 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 67627 67611 68.1061 0.54375 0.00592243 0.0146776 2023-10-09T06:27:37.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:37.899 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:37.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:27:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: osdmap e314: 8 total, 8 up, 7 in 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: failed to encode map e314 with expected crc 2023-10-09T06:27:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:37.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:37.989 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:38.005 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:27:38.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:38.124 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:38.367 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:38.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:38.474 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:38.559 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:38.791 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:38.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: pgmap v970: 99 pgs: 4 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.2 MiB/s wr, 83 op/s; 4296/202644 objects degraded (2.120%); 3828/202644 objects misplaced (1.889%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: pgmap v970: 99 pgs: 4 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.2 MiB/s wr, 83 op/s; 4296/202644 objects degraded (2.120%); 3828/202644 objects misplaced (1.889%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:27:38.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.806 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.806 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:27:38.806 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.806 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:38 smithi107 ceph-mon[220921]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.808 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 68565 68549 67.9556 58.625 0.0107779 0.0147102 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: pgmap v970: 99 pgs: 4 remapped+peering, 6 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 5.2 MiB/s wr, 83 op/s; 4296/202644 objects degraded (2.120%); 3828/202644 objects misplaced (1.889%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: osdmap e315: 8 total, 8 up, 7 in 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:38 smithi150 ceph-mon[194178]: failed to encode map e315 with expected crc 2023-10-09T06:27:38.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:38.939 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:38.997 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:39.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:39.107 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:39.172 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:39.235 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:39.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:39.258 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:27:39.339 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:39.370 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:39.429 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:39.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:39.536 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:39.619 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:39.797 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:39.808 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 69537 69521 67.843 60.75 0.012638 0.0147344 2023-10-09T06:27:39.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:39.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:39.853 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[212750]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T06:27:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[212750]: failed to encode map e315 with expected crc 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[220921]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[220921]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[220921]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:39 smithi107 ceph-mon[220921]: failed to encode map e315 with expected crc 2023-10-09T06:27:39.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:39 smithi150 ceph-mon[194178]: osdmap e316: 8 total, 8 up, 7 in 2023-10-09T06:27:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:39 smithi150 ceph-mon[194178]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:39 smithi150 ceph-mon[194178]: failed to encode map e316 with expected crc 2023-10-09T06:27:39.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:39 smithi150 ceph-mon[194178]: failed to encode map e315 with expected crc 2023-10-09T06:27:39.912 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:40.033 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:40.202 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:40.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:40.311 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:40.341 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:40.388 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:40.396 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:40.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:40.502 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:40.532 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:40.588 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:40.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:40.695 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:40.762 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: failed to encode map e316 with expected crc 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: pgmap v973: 99 pgs: 88 active+clean, 1 active+recovery_wait+degraded, 6 peering, 1 remapped+peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 12 MiB/s wr, 194 op/s; 12/203832 objects degraded (0.006%); 6/203832 objects misplaced (0.003%); 3.8 MiB/s, 63 objects/s recovering 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 12/203832 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[212750]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T06:27:40.763 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: failed to encode map e316 with expected crc 2023-10-09T06:27:40.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: pgmap v973: 99 pgs: 88 active+clean, 1 active+recovery_wait+degraded, 6 peering, 1 remapped+peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 12 MiB/s wr, 194 op/s; 12/203832 objects degraded (0.006%); 6/203832 objects misplaced (0.003%); 3.8 MiB/s, 63 objects/s recovering 2023-10-09T06:27:40.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 12/203832 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:27:40.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.764 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:40 smithi107 ceph-mon[220921]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T06:27:40.790 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:40.808 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 70433 70417 67.6609 56 0.00845156 0.0147732 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: failed to encode map e316 with expected crc 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: pgmap v973: 99 pgs: 88 active+clean, 1 active+recovery_wait+degraded, 6 peering, 1 remapped+peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 12 MiB/s wr, 194 op/s; 12/203832 objects degraded (0.006%); 6/203832 objects misplaced (0.003%); 3.8 MiB/s, 63 objects/s recovering 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 12/203832 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: failed to encode map e317 with expected crc 2023-10-09T06:27:40.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:40 smithi150 ceph-mon[194178]: osdmap e317: 8 total, 8 up, 7 in 2023-10-09T06:27:41.758 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:27:41.765 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:41.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4181054625' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:41.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:41.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4181054625' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.809 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 71219 71203 67.3795 49.125 0.0144716 0.0148355 2023-10-09T06:27:41.877 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4181054625' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:27:41.911 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:41.968 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:41.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:42.076 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:42.103 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:42] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T06:27:42.155 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:42.162 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:27:42.162 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:27:42.162 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_6 2023-10-09T06:27:42.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:27:42.811 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 72166 72150 67.2553 59.1875 0.0148217 0.0148628 2023-10-09T06:27:42.909 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:42.922 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: pgmap v975: 99 pgs: 92 active+clean, 1 active+recovery_wait+degraded, 3 peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 198 op/s; 12/205431 objects degraded (0.006%); 6/205431 objects misplaced (0.003%); 3.5 MiB/s, 58 objects/s recovering 2023-10-09T06:27:42.922 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:42.922 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: failed to encode map e318 with expected crc 2023-10-09T06:27:42.923 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:42.923 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T06:27:42.923 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[220921]: failed to encode map e318 with expected crc 2023-10-09T06:27:42.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:42.924 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: pgmap v975: 99 pgs: 92 active+clean, 1 active+recovery_wait+degraded, 3 peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 198 op/s; 12/205431 objects degraded (0.006%); 6/205431 objects misplaced (0.003%); 3.5 MiB/s, 58 objects/s recovering 2023-10-09T06:27:42.924 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:42.924 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: failed to encode map e318 with expected crc 2023-10-09T06:27:42.924 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:42.924 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T06:27:42.925 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:42 smithi107 ceph-mon[212750]: failed to encode map e318 with expected crc 2023-10-09T06:27:43.025 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.048 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: pgmap v975: 99 pgs: 92 active+clean, 1 active+recovery_wait+degraded, 3 peering, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 198 op/s; 12/205431 objects degraded (0.006%); 6/205431 objects misplaced (0.003%); 3.5 MiB/s, 58 objects/s recovering 2023-10-09T06:27:43.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:27:43.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: failed to encode map e318 with expected crc 2023-10-09T06:27:43.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:27:43.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: osdmap e318: 8 total, 8 up, 7 in 2023-10-09T06:27:43.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:42 smithi150 ceph-mon[194178]: failed to encode map e318 with expected crc 2023-10-09T06:27:43.051 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:43.093 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:43.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:43.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:43.269 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:43.541 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:43.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:43.645 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:43.675 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:43.738 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:43.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:43.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 73005 72989 67.0373 52.4375 0.0189452 0.0149113 2023-10-09T06:27:43.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:43.912 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:44.130 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:44.149 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:44.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:44.214 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:44.232 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:27:44.232 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:27:44.233 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:27:44.233 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:27:44.233 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:27:44.258 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:44.261 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:44.302 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:44.340 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:44.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:44.389 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:44.429 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:44.470 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:44.500 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.514 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:27:44.611 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:44.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:44.719 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:44.754 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:44.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 73839 73823 66.8213 52.125 0.0139825 0.0149596 2023-10-09T06:27:44.817 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:44.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:44.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:44 smithi107 ceph-mon[212750]: pgmap v977: 99 pgs: 95 active+clean, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 435 op/s; 12/209979 objects degraded (0.006%); 6/209979 objects misplaced (0.003%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:27:44.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2942933092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:44.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:44 smithi107 ceph-mon[220921]: pgmap v977: 99 pgs: 95 active+clean, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 435 op/s; 12/209979 objects degraded (0.006%); 6/209979 objects misplaced (0.003%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:27:44.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2942933092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:44.923 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:44.999 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:44 smithi150 ceph-mon[194178]: pgmap v977: 99 pgs: 95 active+clean, 1 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering, 1 active+undersized+remapped; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 435 op/s; 12/209979 objects degraded (0.006%); 6/209979 objects misplaced (0.003%); 3.3 MiB/s, 55 objects/s recovering 2023-10-09T06:27:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2942933092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:45.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 74683 74667 66.6203 52.75 0.00320695 0.0149996 2023-10-09T06:27:45.814 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:27:45.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:45 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/209979 objects degraded (0.006%), 1 pg degraded) 2023-10-09T06:27:45.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:45 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/209979 objects degraded (0.006%), 1 pg degraded) 2023-10-09T06:27:46.062 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:27:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:45 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/209979 objects degraded (0.006%), 1 pg degraded) 2023-10-09T06:27:46.246 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:46.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:46.353 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:46.468 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:46.722 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[212750]: pgmap v978: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 647 op/s; 948 KiB/s, 15 objects/s recovering 2023-10-09T06:27:46.722 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:46.722 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:46.723 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[220921]: pgmap v978: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 647 op/s; 948 KiB/s, 15 objects/s recovering 2023-10-09T06:27:46.723 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:46.723 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:46.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 75405 75389 66.3177 45.125 0.0207214 0.0150746 2023-10-09T06:27:46.915 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:46.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:47.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:47.055 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:46 smithi150 ceph-mon[194178]: pgmap v978: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 647 op/s; 948 KiB/s, 15 objects/s recovering 2023-10-09T06:27:47.055 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:47.055 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:27:47.057 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:47.114 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:47.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:47.168 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:47.220 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:27:47.255 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:47.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:27:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:47.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:27:47.419 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:47.453 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:47.524 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:47.783 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:47.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:47.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 76297 76281 66.1709 55.75 0.00378447 0.0151072 2023-10-09T06:27:47.893 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:47.967 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:48.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:48.102 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:48 smithi107 ceph-mon[212750]: pgmap v979: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 592 KiB/s, 9 objects/s recovering 2023-10-09T06:27:48.266 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:48.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:48.310 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:27:48.332 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:27:48.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:48.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:48 smithi107 ceph-mon[220921]: pgmap v979: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 592 KiB/s, 9 objects/s recovering 2023-10-09T06:27:48.376 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:27:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:48 smithi150 ceph-mon[194178]: pgmap v979: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 592 KiB/s, 9 objects/s recovering 2023-10-09T06:27:48.412 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:48.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:27:48.816 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 76984 76968 65.8527 42.9375 0.0201157 0.0151808 2023-10-09T06:27:48.968 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:48.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:48.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:49.080 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:49.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:49 smithi107 ceph-mon[212750]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:49.168 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:49 smithi107 ceph-mon[220921]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:49 smithi150 ceph-mon[194178]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:27:49.644 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:49.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:49.753 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:49.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 77909 77893 65.7441 57.8125 0.0222574 0.0152053 2023-10-09T06:27:49.826 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:49.920 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":979,"stamp":"2023-10-09T06:27:47.176053+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4846786464,"num_objects":73971,"num_object_clones":0,"num_object_copies":221913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74008,"num_write_kb":4732637,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":828,"num_bytes_recovered":51521946,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75773,"ondisk_log_size":75773,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19158984,"kb_used_data":14265936,"kb_used_omap":20,"kb_used_meta":4892971,"kb_avail":637057080,"statfs":{"total":671965249536,"available":652346449920,"internally_reserved":0,"allocated":14608318464,"data_stored":14600253312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5010402958},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":394985472,"num_objects":6027,"num_object_clones":0,"num_object_copies":18081,"num_objects_missing_on_primary":-18,"num_objects_missing":-18,"num_objects_degraded":-12,"num_objects_misplaced":-6,"num_objects_unfound":0,"num_objects_dirty":6027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6027,"num_write_kb":385728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":4849664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003928"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":297,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645027+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:27:39.645027+0000","last_peered":"2023-10-09T06:27:39.645027+0000","last_clean":"2023-10-09T06:27:39.645027+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:27:39.645027+0000","last_undegraded":"2023-10-09T06:27:39.645027+0000","last_fullsized":"2023-10-09T06:27:39.645027+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657627+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:27:39.657627+0000","last_peered":"2023-10-09T06:27:39.657627+0000","last_clean":"2023-10-09T06:27:39.657627+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:27:39.657627+0000","last_undegraded":"2023-10-09T06:27:39.657627+0000","last_fullsized":"2023-10-09T06:27:39.657627+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":309,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649124+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:27:39.649124+0000","last_peered":"2023-10-09T06:27:39.649124+0000","last_clean":"2023-10-09T06:27:39.649124+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:27:39.649124+0000","last_undegraded":"2023-10-09T06:27:39.649124+0000","last_fullsized":"2023-10-09T06:27:39.649124+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":308,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645289+0000","last_change":"2023-10-09T06:27:36.639854+0000","last_active":"2023-10-09T06:27:39.645289+0000","last_peered":"2023-10-09T06:27:39.645289+0000","last_clean":"2023-10-09T06:27:39.645289+0000","last_became_active":"2023-10-09T06:27:36.639562+0000","last_became_peered":"2023-10-09T06:27:36.639562+0000","last_unstale":"2023-10-09T06:27:39.645289+0000","last_undegraded":"2023-10-09T06:27:39.645289+0000","last_fullsized":"2023-10-09T06:27:39.645289+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:12.869880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.655751+0000","last_change":"2023-10-09T06:27:36.661666+0000","last_active":"2023-10-09T06:27:39.655751+0000","last_peered":"2023-10-09T06:27:39.655751+0000","last_clean":"2023-10-09T06:27:39.655751+0000","last_became_active":"2023-10-09T06:27:36.660973+0000","last_became_peered":"2023-10-09T06:27:36.660973+0000","last_unstale":"2023-10-09T06:27:39.655751+0000","last_undegraded":"2023-10-09T06:27:39.655751+0000","last_fullsized":"2023-10-09T06:27:39.655751+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:06.391336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":297,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646812+0000","last_change":"2023-10-09T06:23:17.735626+0000","last_active":"2023-10-09T06:27:39.646812+0000","last_peered":"2023-10-09T06:27:39.646812+0000","last_clean":"2023-10-09T06:27:39.646812+0000","last_became_active":"2023-10-09T06:23:17.735257+0000","last_became_peered":"2023-10-09T06:23:17.735257+0000","last_unstale":"2023-10-09T06:27:39.646812+0000","last_undegraded":"2023-10-09T06:27:39.646812+0000","last_fullsized":"2023-10-09T06:27:39.646812+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:58.551898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":298,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646950+0000","last_change":"2023-10-09T06:27:36.623708+0000","last_active":"2023-10-09T06:27:39.646950+0000","last_peered":"2023-10-09T06:27:39.646950+0000","last_clean":"2023-10-09T06:27:39.646950+0000","last_became_active":"2023-10-09T06:27:36.623414+0000","last_became_peered":"2023-10-09T06:27:36.623414+0000","last_unstale":"2023-10-09T06:27:39.646950+0000","last_undegraded":"2023-10-09T06:27:39.646950+0000","last_fullsized":"2023-10-09T06:27:39.646950+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:16.126637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.656007+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:27:39.656007+0000","last_peered":"2023-10-09T06:27:39.656007+0000","last_clean":"2023-10-09T06:27:39.656007+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:27:39.656007+0000","last_undegraded":"2023-10-09T06:27:39.656007+0000","last_fullsized":"2023-10-09T06:27:39.656007+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":266,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650967+0000","last_change":"2023-10-09T06:27:36.633548+0000","last_active":"2023-10-09T06:27:39.650967+0000","last_peered":"2023-10-09T06:27:39.650967+0000","last_clean":"2023-10-09T06:27:39.650967+0000","last_became_active":"2023-10-09T06:27:36.633232+0000","last_became_peered":"2023-10-09T06:27:36.633232+0000","last_unstale":"2023-10-09T06:27:39.650967+0000","last_undegraded":"2023-10-09T06:27:39.650967+0000","last_fullsized":"2023-10-09T06:27:39.650967+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:33.046286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657730+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:27:39.657730+0000","last_peered":"2023-10-09T06:27:39.657730+0000","last_clean":"2023-10-09T06:27:39.657730+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:27:39.657730+0000","last_undegraded":"2023-10-09T06:27:39.657730+0000","last_fullsized":"2023-10-09T06:27:39.657730+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":199,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650948+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:27:39.650948+0000","last_peered":"2023-10-09T06:27:39.650948+0000","last_clean":"2023-10-09T06:27:39.650948+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:27:39.650948+0000","last_undegraded":"2023-10-09T06:27:39.650948+0000","last_fullsized":"2023-10-09T06:27:39.650948+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":250,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649727+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:27:39.649727+0000","last_peered":"2023-10-09T06:27:39.649727+0000","last_clean":"2023-10-09T06:27:39.649727+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:27:39.649727+0000","last_undegraded":"2023-10-09T06:27:39.649727+0000","last_fullsized":"2023-10-09T06:27:39.649727+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":324,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645755+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:27:39.645755+0000","last_peered":"2023-10-09T06:27:39.645755+0000","last_clean":"2023-10-09T06:27:39.645755+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:27:39.645755+0000","last_undegraded":"2023-10-09T06:27:39.645755+0000","last_fullsized":"2023-10-09T06:27:39.645755+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":288,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645383+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:27:39.645383+0000","last_peered":"2023-10-09T06:27:39.645383+0000","last_clean":"2023-10-09T06:27:39.645383+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:27:39.645383+0000","last_undegraded":"2023-10-09T06:27:39.645383+0000","last_fullsized":"2023-10-09T06:27:39.645383+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":167,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650704+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:27:39.650704+0000","last_peered":"2023-10-09T06:27:39.650704+0000","last_clean":"2023-10-09T06:27:39.650704+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:27:39.650704+0000","last_undegraded":"2023-10-09T06:27:39.650704+0000","last_fullsized":"2023-10-09T06:27:39.650704+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.656584+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:27:39.656584+0000","last_peered":"2023-10-09T06:27:39.656584+0000","last_clean":"2023-10-09T06:27:39.656584+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:27:39.656584+0000","last_undegraded":"2023-10-09T06:27:39.656584+0000","last_fullsized":"2023-10-09T06:27:39.656584+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645676+0000","last_change":"2023-10-09T06:27:36.622901+0000","last_active":"2023-10-09T06:27:39.645676+0000","last_peered":"2023-10-09T06:27:39.645676+0000","last_clean":"2023-10-09T06:27:39.645676+0000","last_became_active":"2023-10-09T06:27:36.622538+0000","last_became_peered":"2023-10-09T06:27:36.622538+0000","last_unstale":"2023-10-09T06:27:39.645676+0000","last_undegraded":"2023-10-09T06:27:39.645676+0000","last_fullsized":"2023-10-09T06:27:39.645676+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:19.065248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":265,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650844+0000","last_change":"2023-10-09T06:27:36.634307+0000","last_active":"2023-10-09T06:27:39.650844+0000","last_peered":"2023-10-09T06:27:39.650844+0000","last_clean":"2023-10-09T06:27:39.650844+0000","last_became_active":"2023-10-09T06:27:36.633829+0000","last_became_peered":"2023-10-09T06:27:36.633829+0000","last_unstale":"2023-10-09T06:27:39.650844+0000","last_undegraded":"2023-10-09T06:27:39.650844+0000","last_fullsized":"2023-10-09T06:27:39.650844+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:55.688236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650499+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:27:39.650499+0000","last_peered":"2023-10-09T06:27:39.650499+0000","last_clean":"2023-10-09T06:27:39.650499+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:27:39.650499+0000","last_undegraded":"2023-10-09T06:27:39.650499+0000","last_fullsized":"2023-10-09T06:27:39.650499+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":307,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645430+0000","last_change":"2023-10-09T06:27:36.641396+0000","last_active":"2023-10-09T06:27:39.645430+0000","last_peered":"2023-10-09T06:27:39.645430+0000","last_clean":"2023-10-09T06:27:39.645430+0000","last_became_active":"2023-10-09T06:27:36.637663+0000","last_became_peered":"2023-10-09T06:27:36.637663+0000","last_unstale":"2023-10-09T06:27:39.645430+0000","last_undegraded":"2023-10-09T06:27:39.645430+0000","last_fullsized":"2023-10-09T06:27:39.645430+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:50.870043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":346,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646358+0000","last_change":"2023-10-09T06:27:36.694050+0000","last_active":"2023-10-09T06:27:39.646358+0000","last_peered":"2023-10-09T06:27:39.646358+0000","last_clean":"2023-10-09T06:27:39.646358+0000","last_became_active":"2023-10-09T06:27:36.635223+0000","last_became_peered":"2023-10-09T06:27:36.635223+0000","last_unstale":"2023-10-09T06:27:39.646358+0000","last_undegraded":"2023-10-09T06:27:39.646358+0000","last_fullsized":"2023-10-09T06:27:39.646358+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:57.410954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657369+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:27:39.657369+0000","last_peered":"2023-10-09T06:27:39.657369+0000","last_clean":"2023-10-09T06:27:39.657369+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:27:39.657369+0000","last_undegraded":"2023-10-09T06:27:39.657369+0000","last_fullsized":"2023-10-09T06:27:39.657369+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"318'4665","reported_seq":4695,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:47.118140+0000","last_change":"2023-10-09T06:26:28.774987+0000","last_active":"2023-10-09T06:27:47.118140+0000","last_peered":"2023-10-09T06:27:47.118140+0000","last_clean":"2023-10-09T06:27:47.118140+0000","last_became_active":"2023-10-09T06:26:28.774691+0000","last_became_peered":"2023-10-09T06:26:28.774691+0000","last_unstale":"2023-10-09T06:27:47.118140+0000","last_undegraded":"2023-10-09T06:27:47.118140+0000","last_fullsized":"2023-10-09T06:27:47.118140+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4665,"log_dups_size":0,"ondisk_log_size":4665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:17.484383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305725440,"num_objects":4665,"num_object_clones":0,"num_object_copies":13995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4665,"num_write_kb":298560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":179,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650851+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:27:39.650851+0000","last_peered":"2023-10-09T06:27:39.650851+0000","last_clean":"2023-10-09T06:27:39.650851+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:27:39.650851+0000","last_undegraded":"2023-10-09T06:27:39.650851+0000","last_fullsized":"2023-10-09T06:27:39.650851+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":228,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649827+0000","last_change":"2023-10-09T06:21:45.413136+0000","last_active":"2023-10-09T06:27:39.649827+0000","last_peered":"2023-10-09T06:27:39.649827+0000","last_clean":"2023-10-09T06:27:39.649827+0000","last_became_active":"2023-10-09T06:21:45.412492+0000","last_became_peered":"2023-10-09T06:21:45.412492+0000","last_unstale":"2023-10-09T06:27:39.649827+0000","last_undegraded":"2023-10-09T06:27:39.649827+0000","last_fullsized":"2023-10-09T06:27:39.649827+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:36.021036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"318'4673","reported_seq":4701,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:45.380631+0000","last_change":"2023-10-09T06:26:28.769277+0000","last_active":"2023-10-09T06:27:45.380631+0000","last_peered":"2023-10-09T06:27:45.380631+0000","last_clean":"2023-10-09T06:27:45.380631+0000","last_became_active":"2023-10-09T06:26:28.769053+0000","last_became_peered":"2023-10-09T06:26:28.769053+0000","last_unstale":"2023-10-09T06:27:45.380631+0000","last_undegraded":"2023-10-09T06:27:45.380631+0000","last_fullsized":"2023-10-09T06:27:45.380631+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4673,"log_dups_size":0,"ondisk_log_size":4673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:00.039037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306249728,"num_objects":4673,"num_object_clones":0,"num_object_copies":14019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4673,"num_write_kb":299072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":165,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650926+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:27:39.650926+0000","last_peered":"2023-10-09T06:27:39.650926+0000","last_clean":"2023-10-09T06:27:39.650926+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:27:39.650926+0000","last_undegraded":"2023-10-09T06:27:39.650926+0000","last_fullsized":"2023-10-09T06:27:39.650926+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":315,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645096+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:27:39.645096+0000","last_peered":"2023-10-09T06:27:39.645096+0000","last_clean":"2023-10-09T06:27:39.645096+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:27:39.645096+0000","last_undegraded":"2023-10-09T06:27:39.645096+0000","last_fullsized":"2023-10-09T06:27:39.645096+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"318'4623","reported_seq":6034,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.217662+0000","last_change":"2023-10-09T06:27:37.711678+0000","last_active":"2023-10-09T06:27:43.217662+0000","last_peered":"2023-10-09T06:27:43.217662+0000","last_clean":"2023-10-09T06:27:43.217662+0000","last_became_active":"2023-10-09T06:27:37.637084+0000","last_became_peered":"2023-10-09T06:27:37.637084+0000","last_unstale":"2023-10-09T06:27:43.217662+0000","last_undegraded":"2023-10-09T06:27:43.217662+0000","last_fullsized":"2023-10-09T06:27:43.217662+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4623,"log_dups_size":0,"ondisk_log_size":4623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:01.855653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302972928,"num_objects":4623,"num_object_clones":0,"num_object_copies":13869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4623,"num_write_kb":295872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":680,"num_bytes_recovered":44564480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":207,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650825+0000","last_change":"2023-10-09T06:27:36.696253+0000","last_active":"2023-10-09T06:27:39.650825+0000","last_peered":"2023-10-09T06:27:39.650825+0000","last_clean":"2023-10-09T06:27:39.650825+0000","last_became_active":"2023-10-09T06:27:36.634276+0000","last_became_peered":"2023-10-09T06:27:36.634276+0000","last_unstale":"2023-10-09T06:27:39.650825+0000","last_undegraded":"2023-10-09T06:27:39.650825+0000","last_fullsized":"2023-10-09T06:27:39.650825+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:45.085348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":294,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646652+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:27:39.646652+0000","last_peered":"2023-10-09T06:27:39.646652+0000","last_clean":"2023-10-09T06:27:39.646652+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:27:39.646652+0000","last_undegraded":"2023-10-09T06:27:39.646652+0000","last_fullsized":"2023-10-09T06:27:39.646652+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"318'4431","reported_seq":4460,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.223582+0000","last_change":"2023-10-09T06:26:28.775173+0000","last_active":"2023-10-09T06:27:43.223582+0000","last_peered":"2023-10-09T06:27:43.223582+0000","last_clean":"2023-10-09T06:27:43.223582+0000","last_became_active":"2023-10-09T06:26:28.774891+0000","last_became_peered":"2023-10-09T06:26:28.774891+0000","last_unstale":"2023-10-09T06:27:43.223582+0000","last_undegraded":"2023-10-09T06:27:43.223582+0000","last_fullsized":"2023-10-09T06:27:43.223582+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:37.713365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":283584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657227+0000","last_change":"2023-10-09T06:27:36.635812+0000","last_active":"2023-10-09T06:27:39.657227+0000","last_peered":"2023-10-09T06:27:39.657227+0000","last_clean":"2023-10-09T06:27:39.657227+0000","last_became_active":"2023-10-09T06:27:36.633937+0000","last_became_peered":"2023-10-09T06:27:36.633937+0000","last_unstale":"2023-10-09T06:27:39.657227+0000","last_undegraded":"2023-10-09T06:27:39.657227+0000","last_fullsized":"2023-10-09T06:27:39.657227+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:47:33.149738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":271,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.647575+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:27:39.647575+0000","last_peered":"2023-10-09T06:27:39.647575+0000","last_clean":"2023-10-09T06:27:39.647575+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:27:39.647575+0000","last_undegraded":"2023-10-09T06:27:39.647575+0000","last_fullsized":"2023-10-09T06:27:39.647575+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"318'4722","reported_seq":4751,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:46.101494+0000","last_change":"2023-10-09T06:26:28.776488+0000","last_active":"2023-10-09T06:27:46.101494+0000","last_peered":"2023-10-09T06:27:46.101494+0000","last_clean":"2023-10-09T06:27:46.101494+0000","last_became_active":"2023-10-09T06:26:28.776212+0000","last_became_peered":"2023-10-09T06:26:28.776212+0000","last_unstale":"2023-10-09T06:27:46.101494+0000","last_undegraded":"2023-10-09T06:27:46.101494+0000","last_fullsized":"2023-10-09T06:27:46.101494+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:12.868016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309460992,"num_objects":4722,"num_object_clones":0,"num_object_copies":14166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4722,"num_write_kb":302208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":650,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.662393+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:27:39.662393+0000","last_peered":"2023-10-09T06:27:39.662393+0000","last_clean":"2023-10-09T06:27:39.662393+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:27:39.662393+0000","last_undegraded":"2023-10-09T06:27:39.662393+0000","last_fullsized":"2023-10-09T06:27:39.662393+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":19,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.656400+0000","last_change":"2023-10-09T06:27:36.699482+0000","last_active":"2023-10-09T06:27:39.656400+0000","last_peered":"2023-10-09T06:27:39.656400+0000","last_clean":"2023-10-09T06:27:39.656400+0000","last_became_active":"2023-10-09T06:27:36.654947+0000","last_became_peered":"2023-10-09T06:27:36.654947+0000","last_unstale":"2023-10-09T06:27:39.656400+0000","last_undegraded":"2023-10-09T06:27:39.656400+0000","last_fullsized":"2023-10-09T06:27:39.656400+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:22.654599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":298,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645230+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:27:39.645230+0000","last_peered":"2023-10-09T06:27:39.645230+0000","last_clean":"2023-10-09T06:27:39.645230+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:27:39.645230+0000","last_undegraded":"2023-10-09T06:27:39.645230+0000","last_fullsized":"2023-10-09T06:27:39.645230+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"318'4672","reported_seq":4752,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:46.735021+0000","last_change":"2023-10-09T06:27:38.226939+0000","last_active":"2023-10-09T06:27:46.735021+0000","last_peered":"2023-10-09T06:27:46.735021+0000","last_clean":"2023-10-09T06:27:46.735021+0000","last_became_active":"2023-10-09T06:27:37.639569+0000","last_became_peered":"2023-10-09T06:27:37.639569+0000","last_unstale":"2023-10-09T06:27:46.735021+0000","last_undegraded":"2023-10-09T06:27:46.735021+0000","last_fullsized":"2023-10-09T06:27:46.735021+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4672,"log_dups_size":0,"ondisk_log_size":4672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:12.660411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4672,"num_write_kb":299008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":620,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.651908+0000","last_change":"2023-10-09T06:27:37.639571+0000","last_active":"2023-10-09T06:27:39.651908+0000","last_peered":"2023-10-09T06:27:39.651908+0000","last_clean":"2023-10-09T06:27:39.651908+0000","last_became_active":"2023-10-09T06:27:37.639192+0000","last_became_peered":"2023-10-09T06:27:37.639192+0000","last_unstale":"2023-10-09T06:27:39.651908+0000","last_undegraded":"2023-10-09T06:27:39.651908+0000","last_fullsized":"2023-10-09T06:27:39.651908+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:48.561106+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":275,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646619+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:27:39.646619+0000","last_peered":"2023-10-09T06:27:39.646619+0000","last_clean":"2023-10-09T06:27:39.646619+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:27:39.646619+0000","last_undegraded":"2023-10-09T06:27:39.646619+0000","last_fullsized":"2023-10-09T06:27:39.646619+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645411+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:27:39.645411+0000","last_peered":"2023-10-09T06:27:39.645411+0000","last_clean":"2023-10-09T06:27:39.645411+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:27:39.645411+0000","last_undegraded":"2023-10-09T06:27:39.645411+0000","last_fullsized":"2023-10-09T06:27:39.645411+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"318'4617","reported_seq":4705,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:46.085605+0000","last_change":"2023-10-09T06:27:38.682846+0000","last_active":"2023-10-09T06:27:46.085605+0000","last_peered":"2023-10-09T06:27:46.085605+0000","last_clean":"2023-10-09T06:27:46.085605+0000","last_became_active":"2023-10-09T06:27:37.670912+0000","last_became_peered":"2023-10-09T06:27:37.670912+0000","last_unstale":"2023-10-09T06:27:46.085605+0000","last_undegraded":"2023-10-09T06:27:46.085605+0000","last_fullsized":"2023-10-09T06:27:46.085605+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4617,"log_dups_size":0,"ondisk_log_size":4617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:29.767749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302579712,"num_objects":4617,"num_object_clones":0,"num_object_copies":13851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4617,"num_write_kb":295488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":299,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645007+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:27:39.645007+0000","last_peered":"2023-10-09T06:27:39.645007+0000","last_clean":"2023-10-09T06:27:39.645007+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:27:39.645007+0000","last_undegraded":"2023-10-09T06:27:39.645007+0000","last_fullsized":"2023-10-09T06:27:39.645007+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":278,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645652+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:27:39.645652+0000","last_peered":"2023-10-09T06:27:39.645652+0000","last_clean":"2023-10-09T06:27:39.645652+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:27:39.645652+0000","last_undegraded":"2023-10-09T06:27:39.645652+0000","last_fullsized":"2023-10-09T06:27:39.645652+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"318'4705","reported_seq":4806,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:47.133212+0000","last_change":"2023-10-09T06:27:38.032793+0000","last_active":"2023-10-09T06:27:47.133212+0000","last_peered":"2023-10-09T06:27:47.133212+0000","last_clean":"2023-10-09T06:27:47.133212+0000","last_became_active":"2023-10-09T06:27:37.718582+0000","last_became_peered":"2023-10-09T06:27:37.718582+0000","last_unstale":"2023-10-09T06:27:47.133212+0000","last_undegraded":"2023-10-09T06:27:47.133212+0000","last_fullsized":"2023-10-09T06:27:47.133212+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:29.012069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308346880,"num_objects":4705,"num_object_clones":0,"num_object_copies":14115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4705,"num_write_kb":301120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1703936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":598,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.651972+0000","last_change":"2023-10-09T06:22:06.743942+0000","last_active":"2023-10-09T06:27:39.651972+0000","last_peered":"2023-10-09T06:27:39.651972+0000","last_clean":"2023-10-09T06:27:39.651972+0000","last_became_active":"2023-10-09T06:21:48.109415+0000","last_became_peered":"2023-10-09T06:21:48.109415+0000","last_unstale":"2023-10-09T06:27:39.651972+0000","last_undegraded":"2023-10-09T06:27:39.651972+0000","last_fullsized":"2023-10-09T06:27:39.651972+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:39.893423+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.655947+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:27:39.655947+0000","last_peered":"2023-10-09T06:27:39.655947+0000","last_clean":"2023-10-09T06:27:39.655947+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:27:39.655947+0000","last_undegraded":"2023-10-09T06:27:39.655947+0000","last_fullsized":"2023-10-09T06:27:39.655947+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":268,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650091+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:27:39.650091+0000","last_peered":"2023-10-09T06:27:39.650091+0000","last_clean":"2023-10-09T06:27:39.650091+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:27:39.650091+0000","last_undegraded":"2023-10-09T06:27:39.650091+0000","last_fullsized":"2023-10-09T06:27:39.650091+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"318'4686","reported_seq":4716,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:46.725345+0000","last_change":"2023-10-09T06:26:28.775845+0000","last_active":"2023-10-09T06:27:46.725345+0000","last_peered":"2023-10-09T06:27:46.725345+0000","last_clean":"2023-10-09T06:27:46.725345+0000","last_became_active":"2023-10-09T06:26:28.775581+0000","last_became_peered":"2023-10-09T06:26:28.775581+0000","last_unstale":"2023-10-09T06:27:46.725345+0000","last_undegraded":"2023-10-09T06:27:46.725345+0000","last_fullsized":"2023-10-09T06:27:46.725345+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4686,"log_dups_size":0,"ondisk_log_size":4686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:22.188138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307101696,"num_objects":4686,"num_object_clones":0,"num_object_copies":14058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":299904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":580,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.658295+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:27:39.658295+0000","last_peered":"2023-10-09T06:27:39.658295+0000","last_clean":"2023-10-09T06:27:39.658295+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:27:39.658295+0000","last_undegraded":"2023-10-09T06:27:39.658295+0000","last_fullsized":"2023-10-09T06:27:39.658295+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":302,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649897+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:27:39.649897+0000","last_peered":"2023-10-09T06:27:39.649897+0000","last_clean":"2023-10-09T06:27:39.649897+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:27:39.649897+0000","last_undegraded":"2023-10-09T06:27:39.649897+0000","last_fullsized":"2023-10-09T06:27:39.649897+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657543+0000","last_change":"2023-10-09T06:27:36.673228+0000","last_active":"2023-10-09T06:27:39.657543+0000","last_peered":"2023-10-09T06:27:39.657543+0000","last_clean":"2023-10-09T06:27:39.657543+0000","last_became_active":"2023-10-09T06:27:36.672886+0000","last_became_peered":"2023-10-09T06:27:36.672886+0000","last_unstale":"2023-10-09T06:27:39.657543+0000","last_undegraded":"2023-10-09T06:27:39.657543+0000","last_fullsized":"2023-10-09T06:27:39.657543+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.495353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":245,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649983+0000","last_change":"2023-10-09T06:21:45.406448+0000","last_active":"2023-10-09T06:27:39.649983+0000","last_peered":"2023-10-09T06:27:39.649983+0000","last_clean":"2023-10-09T06:27:39.649983+0000","last_became_active":"2023-10-09T06:21:45.405542+0000","last_became_peered":"2023-10-09T06:21:45.405542+0000","last_unstale":"2023-10-09T06:27:39.649983+0000","last_undegraded":"2023-10-09T06:27:39.649983+0000","last_fullsized":"2023-10-09T06:27:39.649983+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:17.955408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"318'4657","reported_seq":4701,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.841341+0000","last_change":"2023-10-09T06:27:38.421674+0000","last_active":"2023-10-09T06:27:43.841341+0000","last_peered":"2023-10-09T06:27:43.841341+0000","last_clean":"2023-10-09T06:27:43.841341+0000","last_became_active":"2023-10-09T06:27:37.645839+0000","last_became_peered":"2023-10-09T06:27:37.645839+0000","last_unstale":"2023-10-09T06:27:43.841341+0000","last_undegraded":"2023-10-09T06:27:43.841341+0000","last_fullsized":"2023-10-09T06:27:43.841341+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4657,"log_dups_size":0,"ondisk_log_size":4657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:08.252652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305201152,"num_objects":4657,"num_object_clones":0,"num_object_copies":13971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4657,"num_write_kb":298048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":604,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.644742+0000","last_change":"2023-10-09T06:22:02.875782+0000","last_active":"2023-10-09T06:27:39.644742+0000","last_peered":"2023-10-09T06:27:39.644742+0000","last_clean":"2023-10-09T06:27:39.644742+0000","last_became_active":"2023-10-09T06:21:48.114269+0000","last_became_peered":"2023-10-09T06:21:48.114269+0000","last_unstale":"2023-10-09T06:27:39.644742+0000","last_undegraded":"2023-10-09T06:27:39.644742+0000","last_fullsized":"2023-10-09T06:27:39.644742+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:26.968539+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":175,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649548+0000","last_change":"2023-10-09T06:21:45.408005+0000","last_active":"2023-10-09T06:27:39.649548+0000","last_peered":"2023-10-09T06:27:39.649548+0000","last_clean":"2023-10-09T06:27:39.649548+0000","last_became_active":"2023-10-09T06:21:45.406721+0000","last_became_peered":"2023-10-09T06:21:45.406721+0000","last_unstale":"2023-10-09T06:27:39.649548+0000","last_undegraded":"2023-10-09T06:27:39.649548+0000","last_fullsized":"2023-10-09T06:27:39.649548+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:23.786166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":279,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645272+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:27:39.645272+0000","last_peered":"2023-10-09T06:27:39.645272+0000","last_clean":"2023-10-09T06:27:39.645272+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:27:39.645272+0000","last_undegraded":"2023-10-09T06:27:39.645272+0000","last_fullsized":"2023-10-09T06:27:39.645272+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":284,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649830+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:27:39.649830+0000","last_peered":"2023-10-09T06:27:39.649830+0000","last_clean":"2023-10-09T06:27:39.649830+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:27:39.649830+0000","last_undegraded":"2023-10-09T06:27:39.649830+0000","last_fullsized":"2023-10-09T06:27:39.649830+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"318'4572","reported_seq":4618,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.812589+0000","last_change":"2023-10-09T06:27:39.119712+0000","last_active":"2023-10-09T06:27:43.812589+0000","last_peered":"2023-10-09T06:27:43.812589+0000","last_clean":"2023-10-09T06:27:43.812589+0000","last_became_active":"2023-10-09T06:27:37.639796+0000","last_became_peered":"2023-10-09T06:27:37.639796+0000","last_unstale":"2023-10-09T06:27:43.812589+0000","last_undegraded":"2023-10-09T06:27:43.812589+0000","last_fullsized":"2023-10-09T06:27:43.812589+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4572,"log_dups_size":0,"ondisk_log_size":4572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:54.114977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299630592,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4572,"num_write_kb":292608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":610,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650535+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:27:39.650535+0000","last_peered":"2023-10-09T06:27:39.650535+0000","last_clean":"2023-10-09T06:27:39.650535+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:27:39.650535+0000","last_undegraded":"2023-10-09T06:27:39.650535+0000","last_fullsized":"2023-10-09T06:27:39.650535+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":303,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.652162+0000","last_change":"2023-10-09T06:27:36.674413+0000","last_active":"2023-10-09T06:27:39.652162+0000","last_peered":"2023-10-09T06:27:39.652162+0000","last_clean":"2023-10-09T06:27:39.652162+0000","last_became_active":"2023-10-09T06:27:36.674092+0000","last_became_peered":"2023-10-09T06:27:36.674092+0000","last_unstale":"2023-10-09T06:27:39.652162+0000","last_undegraded":"2023-10-09T06:27:39.652162+0000","last_fullsized":"2023-10-09T06:27:39.652162+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:35.418640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":292,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649086+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:27:39.649086+0000","last_peered":"2023-10-09T06:27:39.649086+0000","last_clean":"2023-10-09T06:27:39.649086+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:27:39.649086+0000","last_undegraded":"2023-10-09T06:27:39.649086+0000","last_fullsized":"2023-10-09T06:27:39.649086+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":287,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650157+0000","last_change":"2023-10-09T06:21:45.411569+0000","last_active":"2023-10-09T06:27:39.650157+0000","last_peered":"2023-10-09T06:27:39.650157+0000","last_clean":"2023-10-09T06:27:39.650157+0000","last_became_active":"2023-10-09T06:21:45.411094+0000","last_became_peered":"2023-10-09T06:21:45.411094+0000","last_unstale":"2023-10-09T06:27:39.650157+0000","last_undegraded":"2023-10-09T06:27:39.650157+0000","last_fullsized":"2023-10-09T06:27:39.650157+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:37.839275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"318'4519","reported_seq":4556,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.813442+0000","last_change":"2023-10-09T06:27:38.948354+0000","last_active":"2023-10-09T06:27:43.813442+0000","last_peered":"2023-10-09T06:27:43.813442+0000","last_clean":"2023-10-09T06:27:43.813442+0000","last_became_active":"2023-10-09T06:27:37.645904+0000","last_became_peered":"2023-10-09T06:27:37.645904+0000","last_unstale":"2023-10-09T06:27:43.813442+0000","last_undegraded":"2023-10-09T06:27:43.813442+0000","last_fullsized":"2023-10-09T06:27:43.813442+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:28.724829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296157184,"num_objects":4519,"num_object_clones":0,"num_object_copies":13557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4519,"num_write_kb":289216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":269,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.651816+0000","last_change":"2023-10-09T06:27:36.635873+0000","last_active":"2023-10-09T06:27:39.651816+0000","last_peered":"2023-10-09T06:27:39.651816+0000","last_clean":"2023-10-09T06:27:39.651816+0000","last_became_active":"2023-10-09T06:27:36.635175+0000","last_became_peered":"2023-10-09T06:27:36.635175+0000","last_unstale":"2023-10-09T06:27:39.651816+0000","last_undegraded":"2023-10-09T06:27:39.651816+0000","last_fullsized":"2023-10-09T06:27:39.651816+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:22.828032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":591,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649862+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:27:39.649862+0000","last_peered":"2023-10-09T06:27:39.649862+0000","last_clean":"2023-10-09T06:27:39.649862+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:27:39.649862+0000","last_undegraded":"2023-10-09T06:27:39.649862+0000","last_fullsized":"2023-10-09T06:27:39.649862+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.656043+0000","last_change":"2023-10-09T06:27:36.651809+0000","last_active":"2023-10-09T06:27:39.656043+0000","last_peered":"2023-10-09T06:27:39.656043+0000","last_clean":"2023-10-09T06:27:39.656043+0000","last_became_active":"2023-10-09T06:27:36.651554+0000","last_became_peered":"2023-10-09T06:27:36.651554+0000","last_unstale":"2023-10-09T06:27:39.656043+0000","last_undegraded":"2023-10-09T06:27:39.656043+0000","last_fullsized":"2023-10-09T06:27:39.656043+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:58:28.151780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":280,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646812+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:27:39.646812+0000","last_peered":"2023-10-09T06:27:39.646812+0000","last_clean":"2023-10-09T06:27:39.646812+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:27:39.646812+0000","last_undegraded":"2023-10-09T06:27:39.646812+0000","last_fullsized":"2023-10-09T06:27:39.646812+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"318'4551","reported_seq":4604,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.812991+0000","last_change":"2023-10-09T06:27:38.936723+0000","last_active":"2023-10-09T06:27:43.812991+0000","last_peered":"2023-10-09T06:27:43.812991+0000","last_clean":"2023-10-09T06:27:43.812991+0000","last_became_active":"2023-10-09T06:27:37.645689+0000","last_became_peered":"2023-10-09T06:27:37.645689+0000","last_unstale":"2023-10-09T06:27:43.812991+0000","last_undegraded":"2023-10-09T06:27:43.812991+0000","last_fullsized":"2023-10-09T06:27:43.812991+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4551,"log_dups_size":0,"ondisk_log_size":4551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:01.547318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298254336,"num_objects":4551,"num_object_clones":0,"num_object_copies":13653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4551,"num_write_kb":291264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1441792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":602,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650402+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:27:39.650402+0000","last_peered":"2023-10-09T06:27:39.650402+0000","last_clean":"2023-10-09T06:27:39.650402+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:27:39.650402+0000","last_undegraded":"2023-10-09T06:27:39.650402+0000","last_fullsized":"2023-10-09T06:27:39.650402+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649733+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:27:39.649733+0000","last_peered":"2023-10-09T06:27:39.649733+0000","last_clean":"2023-10-09T06:27:39.649733+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:27:39.649733+0000","last_undegraded":"2023-10-09T06:27:39.649733+0000","last_fullsized":"2023-10-09T06:27:39.649733+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":290,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.644597+0000","last_change":"2023-10-09T06:27:36.683415+0000","last_active":"2023-10-09T06:27:39.644597+0000","last_peered":"2023-10-09T06:27:39.644597+0000","last_clean":"2023-10-09T06:27:39.644597+0000","last_became_active":"2023-10-09T06:27:36.678284+0000","last_became_peered":"2023-10-09T06:27:36.678284+0000","last_unstale":"2023-10-09T06:27:39.644597+0000","last_undegraded":"2023-10-09T06:27:39.644597+0000","last_fullsized":"2023-10-09T06:27:39.644597+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:01.115078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:41.768403+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:27:41.768403+0000","last_peered":"2023-10-09T06:27:41.768403+0000","last_clean":"2023-10-09T06:27:41.768403+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:27:41.768403+0000","last_undegraded":"2023-10-09T06:27:41.768403+0000","last_fullsized":"2023-10-09T06:27:41.768403+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"318'4698","reported_seq":4728,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:47.075192+0000","last_change":"2023-10-09T06:26:28.775392+0000","last_active":"2023-10-09T06:27:47.075192+0000","last_peered":"2023-10-09T06:27:47.075192+0000","last_clean":"2023-10-09T06:27:47.075192+0000","last_became_active":"2023-10-09T06:26:28.774522+0000","last_became_peered":"2023-10-09T06:26:28.774522+0000","last_unstale":"2023-10-09T06:27:47.075192+0000","last_undegraded":"2023-10-09T06:27:47.075192+0000","last_fullsized":"2023-10-09T06:27:47.075192+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4698,"log_dups_size":0,"ondisk_log_size":4698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.490573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307888128,"num_objects":4698,"num_object_clones":0,"num_object_copies":14094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4698,"num_write_kb":300672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":580,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.658308+0000","last_change":"2023-10-09T06:27:39.657859+0000","last_active":"2023-10-09T06:27:39.658308+0000","last_peered":"2023-10-09T06:27:39.658308+0000","last_clean":"2023-10-09T06:27:39.658308+0000","last_became_active":"2023-10-09T06:27:39.657516+0000","last_became_peered":"2023-10-09T06:27:39.657516+0000","last_unstale":"2023-10-09T06:27:39.658308+0000","last_undegraded":"2023-10-09T06:27:39.658308+0000","last_fullsized":"2023-10-09T06:27:39.658308+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:43.689046+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650903+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:27:39.650903+0000","last_peered":"2023-10-09T06:27:39.650903+0000","last_clean":"2023-10-09T06:27:39.650903+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:27:39.650903+0000","last_undegraded":"2023-10-09T06:27:39.650903+0000","last_fullsized":"2023-10-09T06:27:39.650903+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":301,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.649762+0000","last_change":"2023-10-09T06:21:45.404630+0000","last_active":"2023-10-09T06:27:39.649762+0000","last_peered":"2023-10-09T06:27:39.649762+0000","last_clean":"2023-10-09T06:27:39.649762+0000","last_became_active":"2023-10-09T06:21:45.404088+0000","last_became_peered":"2023-10-09T06:21:45.404088+0000","last_unstale":"2023-10-09T06:27:39.649762+0000","last_undegraded":"2023-10-09T06:27:39.649762+0000","last_fullsized":"2023-10-09T06:27:39.649762+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:20.274852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":284,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.647844+0000","last_change":"2023-10-09T06:21:45.412916+0000","last_active":"2023-10-09T06:27:39.647844+0000","last_peered":"2023-10-09T06:27:39.647844+0000","last_clean":"2023-10-09T06:27:39.647844+0000","last_became_active":"2023-10-09T06:21:45.412599+0000","last_became_peered":"2023-10-09T06:21:45.412599+0000","last_unstale":"2023-10-09T06:27:39.647844+0000","last_undegraded":"2023-10-09T06:27:39.647844+0000","last_fullsized":"2023-10-09T06:27:39.647844+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:50.860506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"318'4556","reported_seq":4585,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:43.205859+0000","last_change":"2023-10-09T06:26:28.777254+0000","last_active":"2023-10-09T06:27:43.205859+0000","last_peered":"2023-10-09T06:27:43.205859+0000","last_clean":"2023-10-09T06:27:43.205859+0000","last_became_active":"2023-10-09T06:26:28.776915+0000","last_became_peered":"2023-10-09T06:26:28.776915+0000","last_unstale":"2023-10-09T06:27:43.205859+0000","last_undegraded":"2023-10-09T06:27:43.205859+0000","last_fullsized":"2023-10-09T06:27:43.205859+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:04.121493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298582016,"num_objects":4556,"num_object_clones":0,"num_object_copies":13668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4556,"num_write_kb":291584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":601,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645357+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:27:39.645357+0000","last_peered":"2023-10-09T06:27:39.645357+0000","last_clean":"2023-10-09T06:27:39.645357+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:27:39.645357+0000","last_undegraded":"2023-10-09T06:27:39.645357+0000","last_fullsized":"2023-10-09T06:27:39.645357+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":19,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657509+0000","last_change":"2023-10-09T06:27:36.696219+0000","last_active":"2023-10-09T06:27:39.657509+0000","last_peered":"2023-10-09T06:27:39.657509+0000","last_clean":"2023-10-09T06:27:39.657509+0000","last_became_active":"2023-10-09T06:27:36.653183+0000","last_became_peered":"2023-10-09T06:27:36.653183+0000","last_unstale":"2023-10-09T06:27:39.657509+0000","last_undegraded":"2023-10-09T06:27:39.657509+0000","last_fullsized":"2023-10-09T06:27:39.657509+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:16.945012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":277,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.650502+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:27:39.650502+0000","last_peered":"2023-10-09T06:27:39.650502+0000","last_clean":"2023-10-09T06:27:39.650502+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:27:39.650502+0000","last_undegraded":"2023-10-09T06:27:39.650502+0000","last_fullsized":"2023-10-09T06:27:39.650502+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.655816+0000","last_change":"2023-10-09T06:27:36.661487+0000","last_active":"2023-10-09T06:27:39.655816+0000","last_peered":"2023-10-09T06:27:39.655816+0000","last_clean":"2023-10-09T06:27:39.655816+0000","last_became_active":"2023-10-09T06:27:36.660598+0000","last_became_peered":"2023-10-09T06:27:36.660598+0000","last_unstale":"2023-10-09T06:27:39.655816+0000","last_undegraded":"2023-10-09T06:27:39.655816+0000","last_fullsized":"2023-10-09T06:27:39.655816+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:52.336583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"318'4600","reported_seq":4629,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:42.933575+0000","last_change":"2023-10-09T06:26:28.775931+0000","last_active":"2023-10-09T06:27:42.933575+0000","last_peered":"2023-10-09T06:27:42.933575+0000","last_clean":"2023-10-09T06:27:42.933575+0000","last_became_active":"2023-10-09T06:26:28.775652+0000","last_became_peered":"2023-10-09T06:26:28.775652+0000","last_unstale":"2023-10-09T06:27:42.933575+0000","last_undegraded":"2023-10-09T06:27:42.933575+0000","last_fullsized":"2023-10-09T06:27:42.933575+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":4600,"log_dups_size":0,"ondisk_log_size":4600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:39.386224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4600,"num_write_kb":294400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":625,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657249+0000","last_change":"2023-10-09T06:27:37.639111+0000","last_active":"2023-10-09T06:27:39.657249+0000","last_peered":"2023-10-09T06:27:39.657249+0000","last_clean":"2023-10-09T06:27:39.657249+0000","last_became_active":"2023-10-09T06:27:37.638712+0000","last_became_peered":"2023-10-09T06:27:37.638712+0000","last_unstale":"2023-10-09T06:27:39.657249+0000","last_undegraded":"2023-10-09T06:27:39.657249+0000","last_fullsized":"2023-10-09T06:27:39.657249+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:11.923206+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":302,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.647331+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:27:39.647331+0000","last_peered":"2023-10-09T06:27:39.647331+0000","last_clean":"2023-10-09T06:27:39.647331+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:27:39.647331+0000","last_undegraded":"2023-10-09T06:27:39.647331+0000","last_fullsized":"2023-10-09T06:27:39.647331+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":276,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:41.768341+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:27:41.768341+0000","last_peered":"2023-10-09T06:27:41.768341+0000","last_clean":"2023-10-09T06:27:41.768341+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:27:41.768341+0000","last_undegraded":"2023-10-09T06:27:41.768341+0000","last_fullsized":"2023-10-09T06:27:41.768341+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":271,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645076+0000","last_change":"2023-10-09T06:21:45.410364+0000","last_active":"2023-10-09T06:27:39.645076+0000","last_peered":"2023-10-09T06:27:39.645076+0000","last_clean":"2023-10-09T06:27:39.645076+0000","last_became_active":"2023-10-09T06:21:45.408793+0000","last_became_peered":"2023-10-09T06:21:45.408793+0000","last_unstale":"2023-10-09T06:27:39.645076+0000","last_undegraded":"2023-10-09T06:27:39.645076+0000","last_fullsized":"2023-10-09T06:27:39.645076+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:54:29.683789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":277,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.647768+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:27:39.647768+0000","last_peered":"2023-10-09T06:27:39.647768+0000","last_clean":"2023-10-09T06:27:39.647768+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:27:39.647768+0000","last_undegraded":"2023-10-09T06:27:39.647768+0000","last_fullsized":"2023-10-09T06:27:39.647768+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":319,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-10-09T06:27:41.769828+0000","last_change":"2023-10-09T06:27:36.635337+0000","last_active":"2023-10-09T06:27:41.769828+0000","last_peered":"2023-10-09T06:27:41.769828+0000","last_clean":"2023-10-09T06:27:41.769828+0000","last_became_active":"2023-10-09T06:27:36.634979+0000","last_became_peered":"2023-10-09T06:27:36.634979+0000","last_unstale":"2023-10-09T06:27:41.769828+0000","last_undegraded":"2023-10-09T06:27:41.769828+0000","last_fullsized":"2023-10-09T06:27:41.769828+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:33.257699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":123,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645272+0000","last_change":"2023-10-09T06:21:45.420252+0000","last_active":"2023-10-09T06:27:39.645272+0000","last_peered":"2023-10-09T06:27:39.645272+0000","last_clean":"2023-10-09T06:27:39.645272+0000","last_became_active":"2023-10-09T06:21:45.409139+0000","last_became_peered":"2023-10-09T06:21:45.409139+0000","last_unstale":"2023-10-09T06:27:39.645272+0000","last_undegraded":"2023-10-09T06:27:39.645272+0000","last_fullsized":"2023-10-09T06:27:39.645272+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.129521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":325,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645270+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:27:39.645270+0000","last_peered":"2023-10-09T06:27:39.645270+0000","last_clean":"2023-10-09T06:27:39.645270+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:27:39.645270+0000","last_undegraded":"2023-10-09T06:27:39.645270+0000","last_fullsized":"2023-10-09T06:27:39.645270+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":123,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645424+0000","last_change":"2023-10-09T06:21:45.426973+0000","last_active":"2023-10-09T06:27:39.645424+0000","last_peered":"2023-10-09T06:27:39.645424+0000","last_clean":"2023-10-09T06:27:39.645424+0000","last_became_active":"2023-10-09T06:21:45.409422+0000","last_became_peered":"2023-10-09T06:21:45.409422+0000","last_unstale":"2023-10-09T06:27:39.645424+0000","last_undegraded":"2023-10-09T06:27:39.645424+0000","last_fullsized":"2023-10-09T06:27:39.645424+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:38.348239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":315,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.645108+0000","last_change":"2023-10-09T06:27:36.634086+0000","last_active":"2023-10-09T06:27:39.645108+0000","last_peered":"2023-10-09T06:27:39.645108+0000","last_clean":"2023-10-09T06:27:39.645108+0000","last_became_active":"2023-10-09T06:27:36.633802+0000","last_became_peered":"2023-10-09T06:27:36.633802+0000","last_unstale":"2023-10-09T06:27:39.645108+0000","last_undegraded":"2023-10-09T06:27:39.645108+0000","last_fullsized":"2023-10-09T06:27:39.645108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:04.293924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.659878+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:27:39.659878+0000","last_peered":"2023-10-09T06:27:39.659878+0000","last_clean":"2023-10-09T06:27:39.659878+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:27:39.659878+0000","last_undegraded":"2023-10-09T06:27:39.659878+0000","last_fullsized":"2023-10-09T06:27:39.659878+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.657677+0000","last_change":"2023-10-09T06:27:36.666424+0000","last_active":"2023-10-09T06:27:39.657677+0000","last_peered":"2023-10-09T06:27:39.657677+0000","last_clean":"2023-10-09T06:27:39.657677+0000","last_became_active":"2023-10-09T06:27:36.665914+0000","last_became_peered":"2023-10-09T06:27:36.665914+0000","last_unstale":"2023-10-09T06:27:39.657677+0000","last_undegraded":"2023-10-09T06:27:39.657677+0000","last_fullsized":"2023-10-09T06:27:39.657677+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:50.118755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":291,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.646876+0000","last_change":"2023-10-09T06:21:45.423074+0000","last_active":"2023-10-09T06:27:39.646876+0000","last_peered":"2023-10-09T06:27:39.646876+0000","last_clean":"2023-10-09T06:27:39.646876+0000","last_became_active":"2023-10-09T06:21:45.410391+0000","last_became_peered":"2023-10-09T06:21:45.410391+0000","last_unstale":"2023-10-09T06:27:39.646876+0000","last_undegraded":"2023-10-09T06:27:39.646876+0000","last_fullsized":"2023-10-09T06:27:39.646876+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:04:21.247701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":278,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-10-09T06:27:39.644924+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:27:39.644924+0000","last_peered":"2023-10-09T06:27:39.644924+0000","last_clean":"2023-10-09T06:27:39.644924+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:27:39.644924+0000","last_undegraded":"2023-10-09T06:27:39.644924+0000","last_fullsized":"2023-10-09T06:27:39.644924+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4846190592,"num_objects":73947,"num_object_clones":0,"num_object_copies":221841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73947,"num_write_kb":4732608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14619770880,"data_stored":14619770880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73947,"ondisk_log_size":73947,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":10650,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234365,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2646992,"kb_used_data":1742664,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":91098160,"statfs":{"total":95995035648,"available":93284515840,"internally_reserved":0,"allocated":1784487936,"data_stored":1783673228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":8.5980000000000008,"15min":3.7850000000000001},"min":{"1min":0.22600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.2919999999999998,"5min":219.41900000000001,"15min":219.41900000000001},"last":3.464},{"interface":"front","average":{"1min":0.875,"5min":8.7629999999999999,"15min":3.8140000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":4.5640000000000001,"5min":219.399,"15min":219.399},"last":3.4399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:27:28 2023","interfaces":[{"interface":"back","average":{"1min":13.733000000000001,"5min":5.8890000000000002,"15min":3.6659999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20599999999999999},"max":{"1min":215.61699999999999,"5min":215.61699999999999,"15min":215.61699999999999},"last":4.8760000000000003},{"interface":"front","average":{"1min":13.404999999999999,"5min":6.1360000000000001,"15min":3.468},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":216.22200000000001,"5min":216.22200000000001,"15min":216.22200000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":14.24,"5min":12.477,"15min":5.9989999999999997},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":216.34299999999999,"5min":216.34299999999999,"15min":216.34299999999999},"last":2.2349999999999999},{"interface":"front","average":{"1min":14.183,"5min":12.965999999999999,"15min":6.2060000000000004},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":216.25200000000001,"5min":216.25200000000001,"15min":216.25200000000001},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:27:09 2023","interfaces":[{"interface":"back","average":{"1min":14.988,"5min":4.282,"15min":2.5870000000000002},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":215.922,"5min":215.922,"15min":215.922},"last":2.6459999999999999},{"interface":"front","average":{"1min":15.254,"5min":4.3220000000000001,"15min":2.645},"min":{"1min":0.35899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":215.94900000000001,"5min":215.94900000000001,"15min":215.94900000000001},"last":0.76000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:27:09 2023","interfaces":[{"interface":"back","average":{"1min":15.096,"5min":4.7789999999999999,"15min":2.629},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":218.446,"5min":218.446,"15min":218.446},"last":0.33000000000000002},{"interface":"front","average":{"1min":15.068,"5min":4.5899999999999999,"15min":2.3340000000000001},"min":{"1min":0.313,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":216.172,"5min":216.172,"15min":216.172},"last":0.34200000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:27:09 2023","interfaces":[{"interface":"back","average":{"1min":14.989000000000001,"5min":3.673,"15min":2.6549999999999998},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":216.37200000000001,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":15.209,"5min":3.8849999999999998,"15min":2.395},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":218.15899999999999,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:27:09 2023","interfaces":[{"interface":"back","average":{"1min":15.343,"5min":4.0880000000000001,"15min":2.3050000000000002},"min":{"1min":0.35399999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":218.19999999999999,"5min":218.19999999999999,"15min":218.19999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":15.121,"5min":3.8010000000000002,"15min":2.1440000000000001},"min":{"1min":0.371,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":218.05500000000001,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.38500000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496001,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3293124,"kb_used_data":2385916,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":90452028,"statfs":{"total":95995035648,"available":92622876672,"internally_reserved":0,"allocated":2443177984,"data_stored":2442375928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":2.3460000000000001,"15min":2.9609999999999999},"min":{"1min":0.29199999999999998,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":1.0009999999999999,"5min":58.270000000000003,"15min":110.926},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":2.1379999999999999,"15min":3.1720000000000002},"min":{"1min":0.36099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.1579999999999999,"5min":58.253999999999998,"15min":110.911},"last":0.33500000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":18.265000000000001,"5min":5.9909999999999997,"15min":4.5300000000000002},"min":{"1min":0.23999999999999999,"5min":0.215,"15min":0.215},"max":{"1min":292.92200000000003,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":18.207999999999998,"5min":5.9169999999999998,"15min":4.3399999999999999},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":292.904,"5min":292.904,"15min":292.904},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 06:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":3.464,"15min":2.601},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.9780000000000002,"5min":75.191999999999993,"15min":75.191999999999993},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.80300000000000005,"5min":3.6099999999999999,"15min":2.4689999999999999},"min":{"1min":0.36899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.21,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.36099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:27:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.726,"15min":2.0249999999999999},"min":{"1min":0.318,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":1.4390000000000001,"5min":11.148,"15min":110.358},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":1.6599999999999999,"15min":2.0739999999999998},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":1.1100000000000001,"5min":11.180999999999999,"15min":110.43899999999999},"last":11.488}]},{"osd":4,"last update":"Mon Oct 9 06:27:43 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.93899999999999995,"15min":1.2949999999999999},"min":{"1min":0.29299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.4300000000000002,"5min":4.1890000000000001,"15min":17.643999999999998},"last":0.435},{"interface":"front","average":{"1min":0.625,"5min":1.127,"15min":1.2490000000000001},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.464,"5min":6.8780000000000001,"15min":10.563000000000001},"last":0.34999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:27:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0009999999999999,"15min":1.4019999999999999},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":2.508,"5min":5.2460000000000004,"15min":22.379999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.044,"15min":1.369},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":2.512,"5min":4.8920000000000003,"15min":24.401},"last":0.41199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:27:06 2023","interfaces":[{"interface":"back","average":{"1min":15.117000000000001,"5min":3.9900000000000002,"15min":2.4569999999999999},"min":{"1min":0.497,"5min":0.26600000000000001,"15min":0.26200000000000001},"max":{"1min":122.937,"5min":122.937,"15min":122.937},"last":0.40200000000000002},{"interface":"front","average":{"1min":15.058,"5min":3.927,"15min":2.6429999999999998},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":122.92100000000001,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.44600000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402739,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3290684,"kb_used_data":2737140,"kb_used_omap":4,"kb_used_meta":553531,"kb_avail":90454468,"statfs":{"total":95995035648,"available":92625375232,"internally_reserved":0,"allocated":2802831360,"data_stored":2801449285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":566816654},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":6.7539999999999996,"15min":3.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":3.6720000000000002,"5min":145.96600000000001,"15min":148.98400000000001},"last":1.679},{"interface":"front","average":{"1min":1.359,"5min":7.5810000000000004,"15min":4.3479999999999999},"min":{"1min":0.24199999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":8.7669999999999995,"5min":145.982,"15min":149.02699999999999},"last":0.53500000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:27:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":5.4420000000000002,"15min":2.9900000000000002},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.193},"max":{"1min":2.6800000000000002,"5min":139.75399999999999,"15min":139.75399999999999},"last":0.219},{"interface":"front","average":{"1min":0.82299999999999995,"5min":6.4370000000000003,"15min":3.7109999999999999},"min":{"1min":0.26000000000000001,"5min":0.223,"15min":0.219},"max":{"1min":3.8199999999999998,"5min":140.29900000000001,"15min":140.29900000000001},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Oct 9 06:27:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":3.3330000000000002,"15min":2.6080000000000001},"min":{"1min":0.25800000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.8149999999999999,"5min":77.331999999999994,"15min":153.18199999999999},"last":2.488},{"interface":"front","average":{"1min":0.96699999999999997,"5min":2.3420000000000001,"15min":1.9690000000000001},"min":{"1min":0.249,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.7450000000000001,"5min":77.197000000000003,"15min":152.52000000000001},"last":0.27900000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":4.5069999999999997,"15min":3.4820000000000002},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.0259999999999998,"5min":99.391000000000005,"15min":199.43799999999999},"last":2.5550000000000002},{"interface":"front","average":{"1min":0.88,"5min":4.4870000000000001,"15min":3.4209999999999998},"min":{"1min":0.42999999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.71,"5min":99.462000000000003,"15min":199.32900000000001},"last":0.624}]},{"osd":5,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.306,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":2.6059999999999999,"5min":77.436999999999998,"15min":77.436999999999998},"last":0.497},{"interface":"front","average":{"1min":0.89800000000000002,"5min":2.9089999999999998,"15min":2.6150000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.6560000000000001,"5min":77.388000000000005,"15min":77.388000000000005},"last":0.70099999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":3.1989999999999998,"15min":2.7149999999999999},"min":{"1min":0.40500000000000003,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":3.7909999999999999,"5min":77.292000000000002,"15min":77.292000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.359,"5min":3.0499999999999998,"15min":2.5649999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.6589999999999998,"5min":77.378,"15min":77.378},"last":2.617}]},{"osd":7,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":6.157,"5min":3.871,"15min":2.8170000000000002},"min":{"1min":0.36499999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":55.231000000000002,"5min":77.456999999999994,"15min":77.456999999999994},"last":2.5859999999999999},{"interface":"front","average":{"1min":5.7199999999999998,"5min":3.3980000000000001,"15min":2.3719999999999999},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":55.253999999999998,"5min":77.359999999999999,"15min":77.359999999999999},"last":0.60899999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024172,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2860484,"kb_used_data":2118396,"kb_used_omap":5,"kb_used_meta":742074,"kb_avail":90884668,"statfs":{"total":95995035648,"available":93065900032,"internally_reserved":0,"allocated":2169237504,"data_stored":2167617252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":759884410},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:27:21 2023","interfaces":[{"interface":"back","average":{"1min":9.7379999999999995,"5min":3.6320000000000001,"15min":3.3540000000000001},"min":{"1min":0.20300000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":191.446,"5min":191.446,"15min":191.446},"last":0.58899999999999997},{"interface":"front","average":{"1min":9.4390000000000001,"5min":3.8279999999999998,"15min":3.8149999999999999},"min":{"1min":0.24399999999999999,"5min":0.23699999999999999,"15min":0.17499999999999999},"max":{"1min":191.393,"5min":191.393,"15min":191.393},"last":0.66900000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:27:39 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":3.129,"15min":2.4239999999999999},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":51.886000000000003,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.28399999999999997},{"interface":"front","average":{"1min":3.8929999999999998,"5min":3.609,"15min":2.6339999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":52.302999999999997,"5min":57.585000000000001,"15min":57.585000000000001},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:27:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.2189999999999999,"15min":2.9569999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":6.3819999999999997,"5min":12.269,"15min":185.89400000000001},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":2.0960000000000001,"15min":3.0630000000000002},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.219},"max":{"1min":5.2199999999999998,"5min":13.138999999999999,"15min":186.72900000000001},"last":5.3230000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":1.4059999999999999,"15min":1.417},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":5.7229999999999999,"5min":5.9850000000000003,"15min":13.244},"last":0.376},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.52,"15min":1.8140000000000001},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":5.1349999999999998,"5min":5.9889999999999999,"15min":12.752000000000001},"last":0.45300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.3280000000000001,"15min":1.8220000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.27800000000000002},"max":{"1min":5.7939999999999996,"5min":5.7939999999999996,"15min":62.984999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.873,"5min":1.782,"15min":2.1819999999999999},"min":{"1min":0.40100000000000002,"5min":0.35199999999999998,"15min":0.27100000000000002},"max":{"1min":2.806,"5min":11.898,"15min":64.983999999999995},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.9550000000000001,"15min":2.048},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":7.2080000000000002,"5min":11.928000000000001,"15min":21.088999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.153,"5min":1.516,"15min":1.72},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.7649999999999997,"5min":6.819,"15min":13.315},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.659,"15min":2.077},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":5.7140000000000004,"5min":7.5010000000000003,"15min":62.262},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.0179999999999998,"15min":2.2509999999999999},"min":{"1min":0.42699999999999999,"5min":0.377,"15min":0.314},"max":{"1min":2.8359999999999999,"5min":14.167,"15min":62.229999999999997},"last":0.50900000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814041,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2260256,"kb_used_data":1797656,"kb_used_omap":7,"kb_used_meta":462584,"kb_avail":91484896,"statfs":{"total":95995035648,"available":93680533504,"internally_reserved":0,"allocated":1840799744,"data_stored":1839441142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":473686097},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":4.7220000000000004,"15min":3.887},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":9.9139999999999997,"5min":150.32599999999999,"15min":163.40100000000001},"last":2.23},{"interface":"front","average":{"1min":1.498,"5min":3.4300000000000002,"15min":3.7160000000000002},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.182},"max":{"1min":5.8380000000000001,"5min":110.66500000000001,"15min":163.363},"last":1.669}]},{"osd":1,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":4.2560000000000002,"15min":3.6360000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":8.2409999999999997,"5min":205.148,"15min":222.50299999999999},"last":2.3029999999999999},{"interface":"front","average":{"1min":2.266,"5min":4.0099999999999998,"15min":3.887},"min":{"1min":0.28699999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.4019999999999992,"5min":205.267,"15min":222.19399999999999},"last":2.5339999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":1.8380000000000001,"15min":1.8440000000000001},"min":{"1min":0.26500000000000001,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":5.9180000000000001,"5min":16.331,"15min":81.183999999999997},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.997,"5min":1.9159999999999999,"15min":1.9079999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.9379999999999997,"5min":16.364000000000001,"15min":81.061999999999998},"last":1.641}]},{"osd":4,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":2.8730000000000002,"15min":2.0800000000000001},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.9329999999999998,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.339,"5min":2.948,"15min":2.347},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.890000000000001,"5min":90.950000000000003,"15min":90.950000000000003},"last":1.1779999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.4970000000000001,"15min":1.464},"min":{"1min":0.318,"5min":0.317,"15min":0.309},"max":{"1min":6.0069999999999997,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.4299999999999999,"15min":1.9359999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.859,"5min":10.859,"15min":10.859},"last":1.5980000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":3.5,"15min":2.4460000000000002},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.82,"5min":138.291,"15min":138.291},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.103,"5min":3.4990000000000001,"15min":2.1160000000000001},"min":{"1min":0.42299999999999999,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":5.3769999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":2.3559999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.3149999999999999,"15min":1.798},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":5.4050000000000002,"5min":55.851999999999997,"15min":55.851999999999997},"last":2.2719999999999998},{"interface":"front","average":{"1min":1.2969999999999999,"5min":2.0859999999999999,"15min":2.1280000000000001},"min":{"1min":0.40600000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.9749999999999996,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.437}]}]},{"osd":3,"up_from":149,"seq":639950127238,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2764240,"kb_used_data":2331848,"kb_used_omap":2,"kb_used_meta":432381,"kb_avail":90980912,"statfs":{"total":95995035648,"available":93164453888,"internally_reserved":0,"allocated":2387812352,"data_stored":2386527670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":442758197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":5.423,"15min":3.2999999999999998},"min":{"1min":0.19900000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.7800000000000002,"5min":105.252,"15min":187.852},"last":2.516},{"interface":"front","average":{"1min":1.5760000000000001,"5min":5.133,"15min":3.762},"min":{"1min":0.23400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":7.1420000000000003,"5min":105.331,"15min":187.923},"last":0.34399999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":2.552,"15min":2.952},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.335,"5min":59.646999999999998,"15min":187.89099999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.1359999999999999,"5min":2.8679999999999999,"15min":1.9730000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.3659999999999997,"5min":60.085999999999999,"15min":67.028000000000006},"last":1.355}]},{"osd":2,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.5369999999999999,"15min":2.6760000000000002},"min":{"1min":0.27500000000000002,"5min":0.246,"15min":0.23999999999999999},"max":{"1min":7.0919999999999996,"5min":62.904000000000003,"15min":187.76400000000001},"last":0.312},{"interface":"front","average":{"1min":0.94299999999999995,"5min":2.9729999999999999,"15min":2.1699999999999999},"min":{"1min":0.34499999999999997,"5min":0.23400000000000001,"15min":0.22500000000000001},"max":{"1min":3.0510000000000002,"5min":62.719000000000001,"15min":187.79499999999999},"last":1.173}]},{"osd":4,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.371,"15min":1.1160000000000001},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":2.9319999999999999,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.296,"5min":1.3440000000000001,"15min":1.9710000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.9089999999999998,"5min":15.942,"15min":187.67400000000001},"last":4.9539999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.351,"15min":2.1190000000000002},"min":{"1min":0.46700000000000003,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":7.9429999999999996,"5min":16.001999999999999,"15min":187.833},"last":4.9779999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0029999999999999,"15min":2.0209999999999999},"min":{"1min":0.34799999999999998,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":2.6549999999999998,"5min":16.167999999999999,"15min":187.952},"last":0.57399999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.7070000000000001,"15min":2.1659999999999999},"min":{"1min":0.42999999999999999,"5min":0.34899999999999998,"15min":0.33700000000000002},"max":{"1min":7.8179999999999996,"5min":15.976000000000001,"15min":187.78200000000001},"last":4.8799999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.1259999999999999,"15min":1.9750000000000001},"min":{"1min":0.38400000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.0550000000000002,"5min":16.134,"15min":187.81800000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.6060000000000001,"15min":1.869},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":7.3230000000000004,"5min":22.533000000000001,"15min":192.20400000000001},"last":0.438},{"interface":"front","average":{"1min":1.458,"5min":1.5840000000000001,"15min":2.109},"min":{"1min":0.40899999999999997,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":7.8730000000000002,"5min":15.901999999999999,"15min":187.86500000000001},"last":4.9199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019274,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2043204,"kb_used_data":1152316,"kb_used_omap":0,"kb_used_meta":890880,"kb_avail":91701948,"statfs":{"total":95995035648,"available":93902794752,"internally_reserved":0,"allocated":1179971584,"data_stored":1179168807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912261120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":3.6909999999999998,"15min":2.2240000000000002},"min":{"1min":0.255,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":37.485999999999997,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.5419999999999998,"5min":3.9470000000000001,"15min":2.6360000000000001},"min":{"1min":0.33500000000000002,"5min":0.28999999999999998,"15min":0.23999999999999999},"max":{"1min":37.502000000000002,"5min":76.988,"15min":76.988},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.1180000000000001,"15min":1.996},"min":{"1min":0.26700000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.4480000000000004,"5min":9.4480000000000004,"15min":91.849999999999994},"last":1.877},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.341,"15min":2.101},"min":{"1min":0.28199999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":3.1640000000000001,"5min":9.625,"15min":91.885000000000005},"last":1.2769999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.569,"15min":2.3039999999999998},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":6.5179999999999998,"5min":69.640000000000001,"15min":69.640000000000001},"last":2.2730000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":3.7789999999999999,"15min":2.4670000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9209999999999998,"5min":69.676000000000002,"15min":69.676000000000002},"last":0.83099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":1.99,"15min":2.2970000000000002},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.0089999999999999,"5min":45.591000000000001,"15min":91.697000000000003},"last":5.6900000000000004},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.591,"15min":2.1419999999999999},"min":{"1min":0.38400000000000001,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.0529999999999999,"5min":45.548999999999999,"15min":94.356999999999999},"last":0.45900000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.93100000000000005,"15min":1.1639999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.24299999999999999},"max":{"1min":3.5270000000000001,"5min":6.2670000000000003,"15min":10.673},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.85499999999999998,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.0259999999999998,"5min":5.5,"15min":12.552},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:26:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.60999999999999999,"15min":0.84899999999999998},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.26500000000000001},"max":{"1min":0.83799999999999997,"5min":2.1179999999999999,"15min":9.7680000000000007},"last":1.51},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.875,"15min":1.2889999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2050000000000001,"5min":8.859,"15min":15.321999999999999},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:27:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.84299999999999997,"15min":1.484},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":2.2200000000000002,"5min":5.4480000000000004,"15min":62.658000000000001},"last":0.503},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.748,"15min":1.504},"min":{"1min":0.42699999999999999,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":1.903,"5min":3.8450000000000002,"15min":62.573},"last":0.49099999999999999}]}]},{"osd":5,"up_from":0,"seq":163208757639,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2158428160,"data_stored":2158428160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2803433472,"data_stored":2803433472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1844314112,"data_stored":1844314112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2393178112,"data_stored":2393178112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1186332672,"data_stored":1186332672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2445934592,"data_stored":2445934592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1788149760,"data_stored":1788149760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:27:49.928 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_6 pgp_num 16 2023-10-09T06:27:49.958 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:49.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:49.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:50.068 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:50.111 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:50.113 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:50 smithi107 ceph-mon[212750]: pgmap v980: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 496 KiB/s, 8 objects/s recovering 2023-10-09T06:27:50.115 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:50 smithi107 ceph-mon[220921]: pgmap v980: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 496 KiB/s, 8 objects/s recovering 2023-10-09T06:27:50.172 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:50.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:50.278 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:50.311 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:50.356 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:50.359 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:50 smithi150 ceph-mon[194178]: pgmap v980: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 848 op/s; 496 KiB/s, 8 objects/s recovering 2023-10-09T06:27:50.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:50.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:50.463 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:50.489 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:50.541 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:50.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:50.647 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:50.718 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:50.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 78615 78599 65.4559 44.125 0.0104353 0.0152718 2023-10-09T06:27:50.895 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:50.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:51.003 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:27:51.026 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:51.093 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:51.229 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:27:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:51.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:27:51.294 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:51.349 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:51.429 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:51.642 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:51.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:51.748 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:51.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 79390 79374 65.2321 48.4375 0.0323079 0.0153246 2023-10-09T06:27:51.821 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:52.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:27:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:27:52] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2023-10-09T06:27:52.216 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:27:52.216 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:27:52.274 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:27:52.368 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:52.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:52.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:52.485 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:52.555 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[212750]: pgmap v981: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s; 365 KiB/s, 5 objects/s recovering 2023-10-09T06:27:52.555 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3870836539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.555 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.555 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[220921]: pgmap v981: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s; 365 KiB/s, 5 objects/s recovering 2023-10-09T06:27:52.556 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3870836539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.556 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:52 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.564 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:52 smithi150 ceph-mon[194178]: pgmap v981: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s; 365 KiB/s, 5 objects/s recovering 2023-10-09T06:27:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3870836539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:52 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:27:52.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 80246 80230 65.0797 53.5 0.0119273 0.0153428 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[220921]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:53 smithi150 ceph-mon[194178]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: osdmap e319: 8 total, 8 up, 7 in 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:53 smithi107 ceph-mon[212750]: failed to encode map e319 with expected crc 2023-10-09T06:27:53.812 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 81030 81014 64.8736 49 0.0128643 0.0154101 2023-10-09T06:27:54.512 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:54.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:54.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:54.617 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:54 smithi107 ceph-mon[212750]: pgmap v983: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 122 KiB/s, 1 objects/s recovering 2023-10-09T06:27:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:54 smithi107 ceph-mon[220921]: pgmap v983: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 122 KiB/s, 1 objects/s recovering 2023-10-09T06:27:54.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:54 smithi150 ceph-mon[194178]: pgmap v983: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 122 KiB/s, 1 objects/s recovering 2023-10-09T06:27:54.718 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:54.772 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:27:54.813 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 81935 81919 64.7679 56.5625 0.0126449 0.0154346 2023-10-09T06:27:55.184 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:55.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:55.292 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:55.323 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:55.379 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:55.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:55.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:55.488 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:27:55.518 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:55.570 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:27:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:55.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:27:55.667 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:27:55.677 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:55.714 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:27:55.749 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:55.757 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:27:55.779 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:55.805 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:27:55.813 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:27:55.814248+0000 min lat: 0.00108834 max lat: 12.865 avg lat: 0.015479 2023-10-09T06:27:55.813 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:27:55.813 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 82716 82700 64.5685 48.8125 0.00583932 0.015479 2023-10-09T06:27:55.840 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:27:55.859 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:55.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:55.876 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:27:55.916 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:27:55.945 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:27:55.963 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:27:55.974 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:27:56.044 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:56.308 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 11 pgp_num to 16 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[212750]: pgmap v984: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/605532992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/638121528' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[220921]: pgmap v984: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/605532992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/638121528' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:56 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:56 smithi150 ceph-mon[194178]: pgmap v984: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T06:27:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/605532992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:27:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/638121528' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:56 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:27:56.813 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 83575 83559 64.4342 53.6875 0.0166984 0.0155156 2023-10-09T06:27:57.093 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:27:57.323 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:27:57.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:27:57.325 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T06:27:57.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[212750]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:57.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:27:57.431 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:57.476 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:57.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:57 smithi107 ceph-mon[220921]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: osdmap e320: 8 total, 8 up, 7 in 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:57 smithi150 ceph-mon[194178]: failed to encode map e320 with expected crc 2023-10-09T06:27:57.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 84333 84317 64.2256 47.375 0.019846 0.0155646 2023-10-09T06:27:58.038 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:58.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:58.145 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:27:58.177 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:58.248 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:27:58.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:58.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:27:58.357 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:27:58.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:27:58 smithi107 ceph-mon[220921]: pgmap v986: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T06:27:58.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:27:58 smithi107 ceph-mon[212750]: pgmap v986: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T06:27:58.430 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:27:58 smithi150 ceph-mon[194178]: pgmap v986: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T06:27:58.765 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:27:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:58.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:27:58.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 85283 85267 64.1672 59.375 0.013329 0.0155785 2023-10-09T06:27:58.883 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:58.959 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:58.963 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:27:59.042 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:59.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:59.153 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:27:59.185 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:59.242 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:27:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:59.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:27:59.350 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:59.473 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:59.684 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:27:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:27:59.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:86042 11.d 11:b23378d4:::benchmark_data_smithi107_418363_object86041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:59.733430+0000", 2023-10-09T06:27:59.740 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.001703144, 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0017379229999999999, 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 86042 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:27:59.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:27:59.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733430+0000", 2023-10-09T06:27:59.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:27:59.744 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733430+0000", 2023-10-09T06:27:59.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.745 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:27:59.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733427+0000", 2023-10-09T06:27:59.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:27:59.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:27:59.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733473+0000", 2023-10-09T06:27:59.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5918000000000003e-05 2023-10-09T06:27:59.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:27:59.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733497+0000", 2023-10-09T06:27:59.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3799999999999999e-05 2023-10-09T06:27:59.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:27:59.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733508+0000", 2023-10-09T06:27:59.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0764000000000001e-05 2023-10-09T06:27:59.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:27:59.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733809+0000", 2023-10-09T06:27:59.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030131100000000002 2023-10-09T06:27:59.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:27:59.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733940+0000", 2023-10-09T06:27:59.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000130388 2023-10-09T06:27:59.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:27:59.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.734112+0000", 2023-10-09T06:27:59.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172244 2023-10-09T06:27:59.754 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.754 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:27:59.754 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.755 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:86046 11.2 11:48b10996:::benchmark_data_smithi107_418363_object86045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-10-09T06:27:59.755 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:59.733526+0000", 2023-10-09T06:27:59.756 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0016068079999999999, 2023-10-09T06:27:59.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0017342849999999999, 2023-10-09T06:27:59.756 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:27:59.756 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:27:59.756 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:27:59.757 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:27:59.757 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:27:59.757 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 86046 2023-10-09T06:27:59.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733526+0000", 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733526+0000", 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:27:59.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733526+0000", 2023-10-09T06:27:59.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:59.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733546+0000", 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0401999999999999e-05 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733557+0000", 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1309e-05 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:27:59.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733564+0000", 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7290000000000003e-06 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733621+0000", 2023-10-09T06:27:59.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7185000000000002e-05 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733773+0000", 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015133400000000001 2023-10-09T06:27:59.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.733937+0000", 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016405 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.765 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45989.0:86038 11.6 e320/297 11:6d166246:::benchmark_data_smithi107_418363_object86037:head v 320'5444, mlcod=320'5441)", 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:59.727367+0000", 2023-10-09T06:27:59.766 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0077661850000000001, 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0079696339999999997, 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727367+0000", 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:27:59.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727367+0000", 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727366+0000", 2023-10-09T06:27:59.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:27:59.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:27:59.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727394+0000", 2023-10-09T06:27:59.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7393e-05 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727405+0000", 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:27:59.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727411+0000", 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7530000000000004e-06 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:27:59.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727454+0000", 2023-10-09T06:27:59.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2295000000000001e-05 2023-10-09T06:27:59.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727474+0000", 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0486999999999999e-05 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:27:59.774 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.775 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.775 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45989.0:86039 11.6 e320/297 11:6e4b6216:::benchmark_data_smithi107_418363_object86038:head v 320'5445, mlcod=320'5441)", 2023-10-09T06:27:59.775 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:59.727427+0000", 2023-10-09T06:27:59.775 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0077062119999999996, 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0079694970000000007, 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727427+0000", 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:27:59.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727427+0000", 2023-10-09T06:27:59.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:27:59.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:27:59.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727426+0000", 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727449+0000", 2023-10-09T06:27:59.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2325999999999999e-05 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727455+0000", 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9089999999999997e-06 2023-10-09T06:27:59.780 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.727459+0000", 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5850000000000001e-06 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.730243+0000", 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027836100000000002 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:27:59.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:27:59.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:59.730263+0000", 2023-10-09T06:27:59.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0183999999999999e-05 2023-10-09T06:27:59.783 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.783 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:27:59.783 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.784 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:27:59.784 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:27:59.784 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 4 2023-10-09T06:27:59.784 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:27:59.794 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:27:59.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 86118 86102 64.0246 52.1875 0.0203991 0.0156134 2023-10-09T06:27:59.848 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:27:59.848 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:28:00.280 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:00.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:00.388 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:00.414 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:00.472 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:00.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:00.546 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:00 smithi107 ceph-mon[212750]: pgmap v987: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T06:28:00.546 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:00 smithi107 ceph-mon[220921]: pgmap v987: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T06:28:00.578 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:00.613 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:00.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:00 smithi150 ceph-mon[194178]: pgmap v987: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-09T06:28:00.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 86769 86753 63.7501 40.6875 0.0268362 0.015682 2023-10-09T06:28:01.139 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:01.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:01.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:01.244 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:01.284 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:28:01.332 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:01.394 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:01.395 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:01.428 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:01.518 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:01.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:01.623 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:01.651 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:28:01.662 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:01.772 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:01.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:01.814 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 87543 87527 63.5714 48.375 0.0208416 0.015725 2023-10-09T06:28:01.881 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:01.908 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:01.973 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:01.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:01.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:02.077 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:02.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:02] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:28:02.160 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:02.550 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:02.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:02.815 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 88315 88299 63.3953 48.25 0.0165713 0.0157687 2023-10-09T06:28:02.852 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:02.893 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:02 smithi150 ceph-mon[194178]: pgmap v988: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T06:28:02.893 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:02.893 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:02.893 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1266795737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:02.893 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:02.897 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:02.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[212750]: pgmap v988: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T06:28:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1266795737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[220921]: pgmap v988: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s 2023-10-09T06:28:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1266795737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:03.042 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:03.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:03.142 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:03.142 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:03.142 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-09T06:28:03.143 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-09T06:28:03.156 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:03.262 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:03.270 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:28:03.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:03.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:28:03.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:03.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[212750]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T06:28:03.615 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:03.615 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[220921]: failed to encode map e321 with expected crc 2023-10-09T06:28:03.615 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:03.615 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:03 smithi107 ceph-mon[220921]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T06:28:03.643 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:03.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:03.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:03.748 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:03.790 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:03.815 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 88965 88949 63.1367 40.625 0.0307512 0.0158339 2023-10-09T06:28:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:03 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:03 smithi150 ceph-mon[194178]: failed to encode map e321 with expected crc 2023-10-09T06:28:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:03 smithi150 ceph-mon[194178]: osdmap e321: 8 total, 8 up, 7 in 2023-10-09T06:28:04.311 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:04.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:04.406 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:04.421 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:04.450 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:04.507 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:04.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:04.615 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:04.644 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:04.705 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:04.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:04 smithi150 ceph-mon[194178]: pgmap v989: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:28:04.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:04 smithi150 ceph-mon[194178]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:04 smithi150 ceph-mon[194178]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:04.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:04.742 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[212750]: pgmap v989: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[212750]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[220921]: pgmap v989: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[220921]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[220921]: failed to encode map e321 with expected crc 2023-10-09T06:28:04.743 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.744 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:04.811 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:04.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 89664 89648 62.9164 43.6875 0.0281424 0.0158893 2023-10-09T06:28:04.871 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:05.082 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:05.086 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:28:05.102 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:05.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:05.126 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:28:05.196 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:28:05.211 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:05.241 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:05.249 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:28:05.298 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:28:05.300 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:05.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:05.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:05.330 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:28:05.377 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:28:05.408 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:05.428 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.449 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.450 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.450 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:28:05.486 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:05.630 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1790014546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:05.817 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0111 2023-10-09T06:28:05.817 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 90433 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 62.7929 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 38.4842 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 172.25 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1004 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 615.76 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 2756 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0159212 2023-10-09T06:28:05.818 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.146353 2023-10-09T06:28:05.819 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 12.865 2023-10-09T06:28:05.819 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00108834 2023-10-09T06:28:05.823 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:28:05.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1790014546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:05.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1790014546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:06.095 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:06.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:06.199 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:06.230 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:06.284 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:06.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:06.391 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:06.421 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:06.448 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:28:06.467 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:06.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:06.572 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:06.598 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:06.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:06.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:06.747 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:06.754 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:06 smithi107 ceph-mon[212750]: pgmap v991: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:28:06.755 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:06 smithi107 ceph-mon[220921]: pgmap v991: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:28:06.868 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:06 smithi150 ceph-mon[194178]: pgmap v991: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-09T06:28:07.001 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:07.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:07.127 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:07.204 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:07.494 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:07.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:07.614 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:07.690 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:08.229 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:08.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:08.334 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:08.431 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:08.595 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:08.733 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:08.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:08.824 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:08.824 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:08.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:08.867 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:08 smithi107 ceph-mon[220921]: pgmap v992: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 783 op/s 2023-10-09T06:28:08.871 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:08 smithi107 ceph-mon[212750]: pgmap v992: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 783 op/s 2023-10-09T06:28:08.903 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:28:08.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:08 smithi150 ceph-mon[194178]: pgmap v992: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 783 op/s 2023-10-09T06:28:08.933 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:09.411 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:09.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:09.522 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:09.624 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:10.057 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:28:10.065 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:10.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:10.173 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:10.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:10 smithi107 conmon[212706]: 2023-10-09T06:28:10.226+0000 7f7c794f9700 -1 mon.a@0(leader).osd e321 definitely_dead 0 2023-10-09T06:28:10.332 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:10.704 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.5. 2023-10-09T06:28:11.131 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:11.132 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[212750]: pgmap v993: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 757 op/s 2023-10-09T06:28:11.132 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3771704291' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.133 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[220921]: pgmap v993: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 757 op/s 2023-10-09T06:28:11.133 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3771704291' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.133 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:10 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:11.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:10 smithi150 ceph-mon[194178]: pgmap v993: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 39 MiB/s wr, 757 op/s 2023-10-09T06:28:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3771704291' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:10 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-09T06:28:11.235 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:11.264 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:11.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:11.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:11.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:11.416 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:11.442 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:11.484 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:11.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:11.591 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:11.617 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:11.659 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:11.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:11.766 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:11.792 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:11.835 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:11.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:11.887 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:11.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[220921]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:28:11.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T06:28:11.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:11 smithi107 ceph-mon[212750]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: failed to encode map e322 with expected crc 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: osdmap e322: 8 total, 8 up, 8 in 2023-10-09T06:28:11.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:11 smithi150 ceph-mon[194178]: osdmap e323: 8 total, 8 up, 8 in 2023-10-09T06:28:11.943 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:11.972 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:12.021 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-09T06:28:12.206 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:12.234 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:12.236 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:12] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:28:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:12.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:12.339 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:12.371 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:12.584 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:12.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:12.687 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:12.718 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:28:12.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: pgmap v995: 99 pgs: 2 peering, 97 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 31 MiB/s wr, 743 op/s 2023-10-09T06:28:12.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.751 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[212750]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T06:28:12.752 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: pgmap v995: 99 pgs: 2 peering, 97 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 31 MiB/s wr, 743 op/s 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: failed to encode map e323 with expected crc 2023-10-09T06:28:12.753 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:12.754 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:12.754 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:12.754 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:12 smithi107 ceph-mon[220921]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T06:28:12.758 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: pgmap v995: 99 pgs: 2 peering, 97 active+clean; 5.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 31 MiB/s wr, 743 op/s 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: failed to encode map e323 with expected crc 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2352918520' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:13.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:12 smithi150 ceph-mon[194178]: osdmap e324: 8 total, 8 up, 8 in 2023-10-09T06:28:13.747 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:28:13.748 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[220921]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:13 smithi107 ceph-mon[212750]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T06:28:14.153 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: failed to encode map e324 with expected crc 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:13 smithi150 ceph-mon[194178]: osdmap e325: 8 total, 8 up, 8 in 2023-10-09T06:28:14.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:14.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:14.264 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:14.332 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:14.423 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:14.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:14.530 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:14.609 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:14.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: pgmap v998: 99 pgs: 2 remapped+peering, 2 peering, 95 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.7 MiB/s wr, 559 op/s 2023-10-09T06:28:14.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[220921]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: pgmap v998: 99 pgs: 2 remapped+peering, 2 peering, 95 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.7 MiB/s wr, 559 op/s 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: failed to encode map e325 with expected crc 2023-10-09T06:28:14.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:14.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:14.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:14.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:14 smithi107 ceph-mon[212750]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T06:28:15.091 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:15.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:15.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: pgmap v998: 99 pgs: 2 remapped+peering, 2 peering, 95 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 8.7 MiB/s wr, 559 op/s 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: failed to encode map e325 with expected crc 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: failed to encode map e325 with expected crc 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: failed to encode map e325 with expected crc 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: failed to encode map e325 with expected crc 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:14 smithi150 ceph-mon[194178]: osdmap e326: 8 total, 8 up, 8 in 2023-10-09T06:28:15.195 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:15.242 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:15.330 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:15.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:15.437 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:15.512 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:15.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:28:15.700 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:15.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:15.805 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:15.847 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:15.929 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:15.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:15.999 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:15.999 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:15.999 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[212750]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.000 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.001 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:15 smithi107 ceph-mon[220921]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.035 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:16.123 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:15 smithi150 ceph-mon[194178]: failed to encode map e326 with expected crc 2023-10-09T06:28:16.176 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:16.176 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:16.383 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:28:16.441 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:28:16.442 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:16.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:16.481 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:28:16.518 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:28:16.549 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.569 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:28:16.612 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:28:16.621 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:16.661 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:28:16.707 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:28:16.707 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.707 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.707 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.707 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.708 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.708 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.708 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.708 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:28:16.757 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:16.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:16.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:16.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[212750]: pgmap v1001: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 3 peering, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 16974/260010 objects degraded (6.528%); 80 KiB/s, 0 objects/s recovering 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 16974/260010 objects degraded (6.528%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2067047530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[220921]: pgmap v1001: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 3 peering, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 16974/260010 objects degraded (6.528%); 80 KiB/s, 0 objects/s recovering 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 16974/260010 objects degraded (6.528%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2067047530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:16.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:16.888 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:16.929 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:16.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:17.022 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:17.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:17.023 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_6 2023-10-09T06:28:17.023 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:28:17.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:17.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:16 smithi150 ceph-mon[194178]: pgmap v1001: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 3 peering, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 16974/260010 objects degraded (6.528%); 80 KiB/s, 0 objects/s recovering 2023-10-09T06:28:17.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:16 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 16974/260010 objects degraded (6.528%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:17.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2067047530' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:17.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:17.073 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:17.078 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:17.141 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:17.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:17.247 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:17.303 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:17.375 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:17.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:17.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:17.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:17.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:17.637 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:17.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:17.754 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:17.813 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:17.975 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:28:18.117 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:18.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:18.228 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:18.267 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:18.296 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:18.334 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:18.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:18.409 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:18.439 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:18.513 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:18.624 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:18.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:18.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:18.737 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:18.781 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:18.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:18 smithi107 ceph-mon[212750]: pgmap v1002: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 peering, 92 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 16974/255816 objects degraded (6.635%); 5276/255816 objects misplaced (2.062%); 918 KiB/s, 14 objects/s recovering 2023-10-09T06:28:18.844 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:18.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:18.949 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:19.038 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:18 smithi107 ceph-mon[220921]: pgmap v1002: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 peering, 92 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 16974/255816 objects degraded (6.635%); 5276/255816 objects misplaced (2.062%); 918 KiB/s, 14 objects/s recovering 2023-10-09T06:28:19.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:18 smithi150 ceph-mon[194178]: pgmap v1002: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 peering, 92 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 16974/255816 objects degraded (6.635%); 5276/255816 objects misplaced (2.062%); 918 KiB/s, 14 objects/s recovering 2023-10-09T06:28:19.421 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:19.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:19.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:19.526 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:19.594 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:19.751 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:19.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:19.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:19.930 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:19.992 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:19.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:20.098 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:20.125 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:20.168 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:20.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:20.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:20.225 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:28:20.281 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:20.306 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:20.353 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:20.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:20.468 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:20.494 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:20.546 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:20.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:20.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:20.652 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:20.723 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:20.771 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:20.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:20.877 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:20.943 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:21.012 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:21.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:21.126 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:20 smithi107 ceph-mon[220921]: pgmap v1003: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 16974/250587 objects degraded (6.774%); 5276/250587 objects misplaced (2.105%); 816 KiB/s, 12 objects/s recovering 2023-10-09T06:28:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:20 smithi107 ceph-mon[212750]: pgmap v1003: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 16974/250587 objects degraded (6.774%); 5276/250587 objects misplaced (2.105%); 816 KiB/s, 12 objects/s recovering 2023-10-09T06:28:21.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:20 smithi150 ceph-mon[194178]: pgmap v1003: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 16974/250587 objects degraded (6.774%); 5276/250587 objects misplaced (2.105%); 816 KiB/s, 12 objects/s recovering 2023-10-09T06:28:21.180 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:21.353 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:21.406 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:21.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:21.519 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:21.562 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:21.622 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:21.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:21.730 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:21.784 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:21.921 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:21.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:22.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:21 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/248217 objects degraded (6.838%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:22.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:21 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/248217 objects degraded (6.838%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:22.036 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:22.142 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:22.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:21 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/248217 objects degraded (6.838%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:22] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T06:28:22.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:22 smithi107 ceph-mon[212750]: pgmap v1004: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 16974/248217 objects degraded (6.838%); 5125/248217 objects misplaced (2.065%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:22.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:22.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:22 smithi107 ceph-mon[220921]: pgmap v1004: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 16974/248217 objects degraded (6.838%); 5125/248217 objects misplaced (2.065%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:22.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:23.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:22 smithi150 ceph-mon[194178]: pgmap v1004: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 16974/248217 objects degraded (6.838%); 5125/248217 objects misplaced (2.065%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:23.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:23.219 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:23.485 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:23.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:23.604 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:23.657 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:28:23.659 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:23.670 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:28:23.717 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:23.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:23.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:23.822 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:23.867 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:23.933 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:23.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:23.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:24.042 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:24.143 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:24.919 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:28:25.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:25.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:25.134 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[220921]: pgmap v1005: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16974/245250 objects degraded (6.921%); 5125/245250 objects misplaced (2.090%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:28:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[220921]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3071089798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[212750]: pgmap v1005: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16974/245250 objects degraded (6.921%); 5125/245250 objects misplaced (2.090%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[212750]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3071089798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:24 smithi150 ceph-mon[194178]: pgmap v1005: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16974/245250 objects degraded (6.921%); 5125/245250 objects misplaced (2.090%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:28:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:24 smithi150 ceph-mon[194178]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:28:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3071089798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:28:25.188 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:25.283 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:25.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:25.393 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:25.440 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:25.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:25.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:25.646 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:25.681 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:25.783 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:25.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:25.894 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:25.935 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:25 smithi150 ceph-mon[194178]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[212750]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: osdmap e327: 8 total, 8 up, 8 in 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:25 smithi107 ceph-mon[220921]: failed to encode map e327 with expected crc 2023-10-09T06:28:26.220 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1005,"stamp":"2023-10-09T06:28:23.198483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5356525496,"num_objects":81750,"num_object_clones":0,"num_object_copies":245250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16974,"num_objects_misplaced":5125,"num_objects_unfound":0,"num_objects_dirty":81750,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99203,"num_write_kb":5787742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1062,"num_bytes_recovered":66858538,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100968,"ondisk_log_size":100968,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21676612,"kb_used_data":15801924,"kb_used_omap":20,"kb_used_meta":5874283,"kb_avail":728284604,"statfs":{"total":767960285184,"available":745763434496,"internally_reserved":0,"allocated":16181170176,"data_stored":16172062499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6015266446},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,4,6,5,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-357433344,"num_objects":-5454,"num_object_clones":0,"num_object_copies":-16362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11209,"num_objects_misplaced":5125,"num_objects_unfound":0,"num_objects_dirty":-5454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5454,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":233,"num_bytes_recovered":15336522,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.494060"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711259+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:28:13.711259+0000","last_peered":"2023-10-09T06:28:13.711259+0000","last_clean":"2023-10-09T06:28:13.711259+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:28:13.711259+0000","last_undegraded":"2023-10-09T06:28:13.711259+0000","last_fullsized":"2023-10-09T06:28:13.711259+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711033+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:28:13.711033+0000","last_peered":"2023-10-09T06:28:13.711033+0000","last_clean":"2023-10-09T06:28:13.711033+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:28:13.711033+0000","last_undegraded":"2023-10-09T06:28:13.711033+0000","last_fullsized":"2023-10-09T06:28:13.711033+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":317,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723319+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:28:13.723319+0000","last_peered":"2023-10-09T06:28:13.723319+0000","last_clean":"2023-10-09T06:28:13.723319+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:28:13.723319+0000","last_undegraded":"2023-10-09T06:28:13.723319+0000","last_fullsized":"2023-10-09T06:28:13.723319+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":317,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.710761+0000","last_change":"2023-10-09T06:27:36.639854+0000","last_active":"2023-10-09T06:28:14.710761+0000","last_peered":"2023-10-09T06:28:14.710761+0000","last_clean":"2023-10-09T06:28:14.710761+0000","last_became_active":"2023-10-09T06:27:36.639562+0000","last_became_peered":"2023-10-09T06:27:36.639562+0000","last_unstale":"2023-10-09T06:28:14.710761+0000","last_undegraded":"2023-10-09T06:28:14.710761+0000","last_fullsized":"2023-10-09T06:28:14.710761+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:12.869880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712066+0000","last_change":"2023-10-09T06:27:36.661666+0000","last_active":"2023-10-09T06:28:13.712066+0000","last_peered":"2023-10-09T06:28:13.712066+0000","last_clean":"2023-10-09T06:28:13.712066+0000","last_became_active":"2023-10-09T06:27:36.660973+0000","last_became_peered":"2023-10-09T06:27:36.660973+0000","last_unstale":"2023-10-09T06:28:13.712066+0000","last_undegraded":"2023-10-09T06:28:13.712066+0000","last_fullsized":"2023-10-09T06:28:13.712066+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:06.391336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.717143+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:28:13.717143+0000","last_peered":"2023-10-09T06:28:13.717143+0000","last_clean":"2023-10-09T06:28:13.717143+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:28:13.717143+0000","last_undegraded":"2023-10-09T06:28:13.717143+0000","last_fullsized":"2023-10-09T06:28:13.717143+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711485+0000","last_change":"2023-10-09T06:28:11.700811+0000","last_active":"2023-10-09T06:28:13.711485+0000","last_peered":"2023-10-09T06:28:13.711485+0000","last_clean":"2023-10-09T06:28:13.711485+0000","last_became_active":"2023-10-09T06:28:11.700501+0000","last_became_peered":"2023-10-09T06:28:11.700501+0000","last_unstale":"2023-10-09T06:28:13.711485+0000","last_undegraded":"2023-10-09T06:28:13.711485+0000","last_fullsized":"2023-10-09T06:28:13.711485+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:01.584981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.717412+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:28:13.717412+0000","last_peered":"2023-10-09T06:28:13.717412+0000","last_clean":"2023-10-09T06:28:13.717412+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:28:13.717412+0000","last_undegraded":"2023-10-09T06:28:13.717412+0000","last_fullsized":"2023-10-09T06:28:13.717412+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715585+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:28:13.715585+0000","last_peered":"2023-10-09T06:28:13.715585+0000","last_clean":"2023-10-09T06:28:13.715585+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:28:13.715585+0000","last_undegraded":"2023-10-09T06:28:13.715585+0000","last_fullsized":"2023-10-09T06:28:13.715585+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.710867+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:28:13.710867+0000","last_peered":"2023-10-09T06:28:13.710867+0000","last_clean":"2023-10-09T06:28:13.710867+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:28:13.710867+0000","last_undegraded":"2023-10-09T06:28:13.710867+0000","last_fullsized":"2023-10-09T06:28:13.710867+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":207,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.719589+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:28:13.719589+0000","last_peered":"2023-10-09T06:28:13.719589+0000","last_clean":"2023-10-09T06:28:13.719589+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:28:13.719589+0000","last_undegraded":"2023-10-09T06:28:13.719589+0000","last_fullsized":"2023-10-09T06:28:13.719589+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":258,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723599+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:28:13.723599+0000","last_peered":"2023-10-09T06:28:13.723599+0000","last_clean":"2023-10-09T06:28:13.723599+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:28:13.723599+0000","last_undegraded":"2023-10-09T06:28:13.723599+0000","last_fullsized":"2023-10-09T06:28:13.723599+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":332,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711261+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:28:13.711261+0000","last_peered":"2023-10-09T06:28:13.711261+0000","last_clean":"2023-10-09T06:28:13.711261+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:28:13.711261+0000","last_undegraded":"2023-10-09T06:28:13.711261+0000","last_fullsized":"2023-10-09T06:28:13.711261+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":296,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711444+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:28:13.711444+0000","last_peered":"2023-10-09T06:28:13.711444+0000","last_clean":"2023-10-09T06:28:13.711444+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:28:13.711444+0000","last_undegraded":"2023-10-09T06:28:13.711444+0000","last_fullsized":"2023-10-09T06:28:13.711444+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":175,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718671+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:28:13.718671+0000","last_peered":"2023-10-09T06:28:13.718671+0000","last_clean":"2023-10-09T06:28:13.718671+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:28:13.718671+0000","last_undegraded":"2023-10-09T06:28:13.718671+0000","last_fullsized":"2023-10-09T06:28:13.718671+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.710888+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:28:13.710888+0000","last_peered":"2023-10-09T06:28:13.710888+0000","last_clean":"2023-10-09T06:28:13.710888+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:28:13.710888+0000","last_undegraded":"2023-10-09T06:28:13.710888+0000","last_fullsized":"2023-10-09T06:28:13.710888+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.708953+0000","last_change":"2023-10-09T06:27:36.622901+0000","last_active":"2023-10-09T06:28:13.708953+0000","last_peered":"2023-10-09T06:28:13.708953+0000","last_clean":"2023-10-09T06:28:13.708953+0000","last_became_active":"2023-10-09T06:27:36.622538+0000","last_became_peered":"2023-10-09T06:27:36.622538+0000","last_unstale":"2023-10-09T06:28:13.708953+0000","last_undegraded":"2023-10-09T06:28:13.708953+0000","last_fullsized":"2023-10-09T06:28:13.708953+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:19.065248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.713022+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:28:13.713022+0000","last_peered":"2023-10-09T06:28:13.713022+0000","last_clean":"2023-10-09T06:28:13.713022+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:28:13.713022+0000","last_undegraded":"2023-10-09T06:28:13.713022+0000","last_fullsized":"2023-10-09T06:28:13.713022+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":169,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718325+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:28:13.718325+0000","last_peered":"2023-10-09T06:28:13.718325+0000","last_clean":"2023-10-09T06:28:13.718325+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:28:13.718325+0000","last_undegraded":"2023-10-09T06:28:13.718325+0000","last_fullsized":"2023-10-09T06:28:13.718325+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.707533+0000","last_change":"2023-10-09T06:27:36.641396+0000","last_active":"2023-10-09T06:28:13.707533+0000","last_peered":"2023-10-09T06:28:13.707533+0000","last_clean":"2023-10-09T06:28:13.707533+0000","last_became_active":"2023-10-09T06:27:36.637663+0000","last_became_peered":"2023-10-09T06:27:36.637663+0000","last_unstale":"2023-10-09T06:28:13.707533+0000","last_undegraded":"2023-10-09T06:28:13.707533+0000","last_fullsized":"2023-10-09T06:28:13.707533+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:25:50.870043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":374,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715519+0000","last_change":"2023-10-09T06:28:11.711541+0000","last_active":"2023-10-09T06:28:13.715519+0000","last_peered":"2023-10-09T06:28:13.715519+0000","last_clean":"2023-10-09T06:28:13.715519+0000","last_became_active":"2023-10-09T06:28:11.698668+0000","last_became_peered":"2023-10-09T06:28:11.698668+0000","last_unstale":"2023-10-09T06:28:13.715519+0000","last_undegraded":"2023-10-09T06:28:13.715519+0000","last_fullsized":"2023-10-09T06:28:13.715519+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:33:49.963750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711282+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:28:13.711282+0000","last_peered":"2023-10-09T06:28:13.711282+0000","last_clean":"2023-10-09T06:28:13.711282+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:28:13.711282+0000","last_undegraded":"2023-10-09T06:28:13.711282+0000","last_fullsized":"2023-10-09T06:28:13.711282+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"326'6084","reported_seq":6578,"reported_epoch":326,"state":"active+recovering+remapped","last_fresh":"2023-10-09T06:28:20.437913+0000","last_change":"2023-10-09T06:28:12.968211+0000","last_active":"2023-10-09T06:28:20.437913+0000","last_peered":"2023-10-09T06:28:20.437913+0000","last_clean":"2023-10-09T06:28:10.664487+0000","last_became_active":"2023-10-09T06:28:12.914831+0000","last_became_peered":"2023-10-09T06:28:12.914831+0000","last_unstale":"2023-10-09T06:28:20.437913+0000","last_undegraded":"2023-10-09T06:28:20.437913+0000","last_fullsized":"2023-10-09T06:28:20.437913+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:13:44.550183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5125,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6084,"num_write_kb":356416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":14745600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":-71}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":187,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718553+0000","last_change":"2023-10-09T06:23:17.771558+0000","last_active":"2023-10-09T06:28:13.718553+0000","last_peered":"2023-10-09T06:28:13.718553+0000","last_clean":"2023-10-09T06:28:13.718553+0000","last_became_active":"2023-10-09T06:23:17.744834+0000","last_became_peered":"2023-10-09T06:23:17.744834+0000","last_unstale":"2023-10-09T06:28:13.718553+0000","last_undegraded":"2023-10-09T06:28:13.718553+0000","last_fullsized":"2023-10-09T06:28:13.718553+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:43:28.441205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712711+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:28:13.712711+0000","last_peered":"2023-10-09T06:28:13.712711+0000","last_clean":"2023-10-09T06:28:13.712711+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:28:13.712711+0000","last_undegraded":"2023-10-09T06:28:13.712711+0000","last_fullsized":"2023-10-09T06:28:13.712711+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"326'6307","reported_seq":6345,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:20.384479+0000","last_change":"2023-10-09T06:26:28.769277+0000","last_active":"2023-10-09T06:28:20.384479+0000","last_peered":"2023-10-09T06:28:20.384479+0000","last_clean":"2023-10-09T06:28:20.384479+0000","last_became_active":"2023-10-09T06:26:28.769053+0000","last_became_peered":"2023-10-09T06:26:28.769053+0000","last_unstale":"2023-10-09T06:28:20.384479+0000","last_undegraded":"2023-10-09T06:28:20.384479+0000","last_fullsized":"2023-10-09T06:28:20.384479+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:07:00.039037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6307,"num_write_kb":368064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":173,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718606+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:28:13.718606+0000","last_peered":"2023-10-09T06:28:13.718606+0000","last_clean":"2023-10-09T06:28:13.718606+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:28:13.718606+0000","last_undegraded":"2023-10-09T06:28:13.718606+0000","last_fullsized":"2023-10-09T06:28:13.718606+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":323,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711120+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:28:13.711120+0000","last_peered":"2023-10-09T06:28:13.711120+0000","last_clean":"2023-10-09T06:28:13.711120+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:28:13.711120+0000","last_undegraded":"2023-10-09T06:28:13.711120+0000","last_fullsized":"2023-10-09T06:28:13.711120+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"326'6263","reported_seq":7692,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:28:18.198186+0000","last_change":"2023-10-09T06:28:12.993423+0000","last_active":"2023-10-09T06:28:18.198186+0000","last_peered":"2023-10-09T06:28:18.198186+0000","last_clean":"2023-10-09T06:28:10.696199+0000","last_became_active":"2023-10-09T06:28:12.961354+0000","last_became_peered":"2023-10-09T06:28:12.961354+0000","last_unstale":"2023-10-09T06:28:18.198186+0000","last_undegraded":"2023-10-09T06:28:12.725711+0000","last_fullsized":"2023-10-09T06:28:12.693643+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6263,"log_dups_size":0,"ondisk_log_size":6263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:45:05.966853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345178112,"num_objects":5267,"num_object_clones":0,"num_object_copies":15801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5765,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6263,"num_write_kb":368960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":680,"num_bytes_recovered":44564480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,6],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":5267},{"shards":"6,7","objects":-250}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":229,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.719464+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:28:13.719464+0000","last_peered":"2023-10-09T06:28:13.719464+0000","last_clean":"2023-10-09T06:28:13.719464+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:28:13.719464+0000","last_undegraded":"2023-10-09T06:28:13.719464+0000","last_fullsized":"2023-10-09T06:28:13.719464+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715637+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:28:13.715637+0000","last_peered":"2023-10-09T06:28:13.715637+0000","last_clean":"2023-10-09T06:28:13.715637+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:28:13.715637+0000","last_undegraded":"2023-10-09T06:28:13.715637+0000","last_fullsized":"2023-10-09T06:28:13.715637+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"326'5946","reported_seq":5983,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:18.141345+0000","last_change":"2023-10-09T06:26:28.775173+0000","last_active":"2023-10-09T06:28:18.141345+0000","last_peered":"2023-10-09T06:28:18.141345+0000","last_clean":"2023-10-09T06:28:18.141345+0000","last_became_active":"2023-10-09T06:26:28.774891+0000","last_became_peered":"2023-10-09T06:26:28.774891+0000","last_unstale":"2023-10-09T06:28:18.141345+0000","last_undegraded":"2023-10-09T06:28:18.141345+0000","last_fullsized":"2023-10-09T06:28:18.141345+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:37.713365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334364672,"num_objects":5102,"num_object_clones":0,"num_object_copies":15306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5946,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.713706+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:28:13.713706+0000","last_peered":"2023-10-09T06:28:13.713706+0000","last_clean":"2023-10-09T06:28:13.713706+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:28:13.713706+0000","last_undegraded":"2023-10-09T06:28:13.713706+0000","last_fullsized":"2023-10-09T06:28:13.713706+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":279,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.707610+0000","last_change":"2023-10-09T06:13:02.837461+0000","last_active":"2023-10-09T06:28:13.707610+0000","last_peered":"2023-10-09T06:28:13.707610+0000","last_clean":"2023-10-09T06:28:13.707610+0000","last_became_active":"2023-10-09T06:13:02.837155+0000","last_became_peered":"2023-10-09T06:13:02.837155+0000","last_unstale":"2023-10-09T06:28:13.707610+0000","last_undegraded":"2023-10-09T06:28:13.707610+0000","last_fullsized":"2023-10-09T06:28:13.707610+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:17.426256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"326'6295","reported_seq":6332,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:21.013991+0000","last_change":"2023-10-09T06:26:28.776488+0000","last_active":"2023-10-09T06:28:21.013991+0000","last_peered":"2023-10-09T06:28:21.013991+0000","last_clean":"2023-10-09T06:28:21.013991+0000","last_became_active":"2023-10-09T06:26:28.776212+0000","last_became_peered":"2023-10-09T06:26:28.776212+0000","last_unstale":"2023-10-09T06:28:21.013991+0000","last_undegraded":"2023-10-09T06:28:21.013991+0000","last_fullsized":"2023-10-09T06:28:21.013991+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6295,"log_dups_size":0,"ondisk_log_size":6295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:12.868016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334430208,"num_objects":5103,"num_object_clones":0,"num_object_copies":15309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6295,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":658,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718235+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:28:13.718235+0000","last_peered":"2023-10-09T06:28:13.718235+0000","last_clean":"2023-10-09T06:28:13.718235+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:28:13.718235+0000","last_undegraded":"2023-10-09T06:28:13.718235+0000","last_fullsized":"2023-10-09T06:28:13.718235+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":27,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711550+0000","last_change":"2023-10-09T06:27:36.699482+0000","last_active":"2023-10-09T06:28:13.711550+0000","last_peered":"2023-10-09T06:28:13.711550+0000","last_clean":"2023-10-09T06:28:13.711550+0000","last_became_active":"2023-10-09T06:27:36.654947+0000","last_became_peered":"2023-10-09T06:27:36.654947+0000","last_unstale":"2023-10-09T06:28:13.711550+0000","last_undegraded":"2023-10-09T06:28:13.711550+0000","last_fullsized":"2023-10-09T06:28:13.711550+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:22.654599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":306,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711082+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:28:13.711082+0000","last_peered":"2023-10-09T06:28:13.711082+0000","last_clean":"2023-10-09T06:28:13.711082+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:28:13.711082+0000","last_undegraded":"2023-10-09T06:28:13.711082+0000","last_fullsized":"2023-10-09T06:28:13.711082+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"326'6240","reported_seq":6328,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:21.709963+0000","last_change":"2023-10-09T06:27:38.226939+0000","last_active":"2023-10-09T06:28:21.709963+0000","last_peered":"2023-10-09T06:28:21.709963+0000","last_clean":"2023-10-09T06:28:21.709963+0000","last_became_active":"2023-10-09T06:27:37.639569+0000","last_became_peered":"2023-10-09T06:27:37.639569+0000","last_unstale":"2023-10-09T06:28:21.709963+0000","last_undegraded":"2023-10-09T06:28:21.709963+0000","last_fullsized":"2023-10-09T06:28:21.709963+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6240,"log_dups_size":0,"ondisk_log_size":6240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:25:12.660411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330039296,"num_objects":5036,"num_object_clones":0,"num_object_copies":15108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":638,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.733366+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:28:14.733366+0000","last_peered":"2023-10-09T06:28:14.733366+0000","last_clean":"2023-10-09T06:28:14.733366+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:28:14.733366+0000","last_undegraded":"2023-10-09T06:28:14.733366+0000","last_fullsized":"2023-10-09T06:28:14.733366+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":283,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715517+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:28:13.715517+0000","last_peered":"2023-10-09T06:28:13.715517+0000","last_clean":"2023-10-09T06:28:13.715517+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:28:13.715517+0000","last_undegraded":"2023-10-09T06:28:13.715517+0000","last_fullsized":"2023-10-09T06:28:13.715517+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":303,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712039+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:28:13.712039+0000","last_peered":"2023-10-09T06:28:13.712039+0000","last_clean":"2023-10-09T06:28:13.712039+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:28:13.712039+0000","last_undegraded":"2023-10-09T06:28:13.712039+0000","last_fullsized":"2023-10-09T06:28:13.712039+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"326'6127","reported_seq":6223,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:21.030661+0000","last_change":"2023-10-09T06:27:38.682846+0000","last_active":"2023-10-09T06:28:21.030661+0000","last_peered":"2023-10-09T06:28:21.030661+0000","last_clean":"2023-10-09T06:28:21.030661+0000","last_became_active":"2023-10-09T06:27:37.670912+0000","last_became_peered":"2023-10-09T06:27:37.670912+0000","last_unstale":"2023-10-09T06:28:21.030661+0000","last_undegraded":"2023-10-09T06:28:21.030661+0000","last_fullsized":"2023-10-09T06:28:21.030661+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:15:29.767749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323289088,"num_objects":4933,"num_object_clones":0,"num_object_copies":14799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6127,"num_write_kb":353920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":307,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.710902+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:28:13.710902+0000","last_peered":"2023-10-09T06:28:13.710902+0000","last_clean":"2023-10-09T06:28:13.710902+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:28:13.710902+0000","last_undegraded":"2023-10-09T06:28:13.710902+0000","last_fullsized":"2023-10-09T06:28:13.710902+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":286,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712353+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:28:13.712353+0000","last_peered":"2023-10-09T06:28:13.712353+0000","last_clean":"2023-10-09T06:28:13.712353+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:28:13.712353+0000","last_undegraded":"2023-10-09T06:28:13.712353+0000","last_fullsized":"2023-10-09T06:28:13.712353+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"326'6260","reported_seq":6369,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:22.141941+0000","last_change":"2023-10-09T06:27:38.032793+0000","last_active":"2023-10-09T06:28:22.141941+0000","last_peered":"2023-10-09T06:28:22.141941+0000","last_clean":"2023-10-09T06:28:22.141941+0000","last_became_active":"2023-10-09T06:27:37.718582+0000","last_became_peered":"2023-10-09T06:27:37.718582+0000","last_unstale":"2023-10-09T06:28:22.141941+0000","last_undegraded":"2023-10-09T06:28:22.141941+0000","last_fullsized":"2023-10-09T06:28:22.141941+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:29.012069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331743232,"num_objects":5062,"num_object_clones":0,"num_object_copies":15186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6260,"num_write_kb":362304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1703936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":616,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.737320+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:28:14.737320+0000","last_peered":"2023-10-09T06:28:14.737320+0000","last_clean":"2023-10-09T06:28:14.737320+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:28:14.737320+0000","last_undegraded":"2023-10-09T06:28:14.737320+0000","last_fullsized":"2023-10-09T06:28:14.737320+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712222+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:28:13.712222+0000","last_peered":"2023-10-09T06:28:13.712222+0000","last_clean":"2023-10-09T06:28:13.712222+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:28:13.712222+0000","last_undegraded":"2023-10-09T06:28:13.712222+0000","last_fullsized":"2023-10-09T06:28:13.712222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":276,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.725925+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:28:13.725925+0000","last_peered":"2023-10-09T06:28:13.725925+0000","last_clean":"2023-10-09T06:28:13.725925+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:28:13.725925+0000","last_undegraded":"2023-10-09T06:28:13.725925+0000","last_fullsized":"2023-10-09T06:28:13.725925+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"326'6223","reported_seq":6262,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:21.749734+0000","last_change":"2023-10-09T06:26:28.775845+0000","last_active":"2023-10-09T06:28:21.749734+0000","last_peered":"2023-10-09T06:28:21.749734+0000","last_clean":"2023-10-09T06:28:21.749734+0000","last_became_active":"2023-10-09T06:26:28.775581+0000","last_became_peered":"2023-10-09T06:26:28.775581+0000","last_unstale":"2023-10-09T06:28:21.749734+0000","last_undegraded":"2023-10-09T06:28:21.749734+0000","last_fullsized":"2023-10-09T06:28:21.749734+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6223,"log_dups_size":0,"ondisk_log_size":6223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:29:22.188138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329383960,"num_objects":5027,"num_object_clones":0,"num_object_copies":15081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5027,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6223,"num_write_kb":359937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":588,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.717229+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:28:13.717229+0000","last_peered":"2023-10-09T06:28:13.717229+0000","last_clean":"2023-10-09T06:28:13.717229+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:28:13.717229+0000","last_undegraded":"2023-10-09T06:28:13.717229+0000","last_fullsized":"2023-10-09T06:28:13.717229+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":310,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723888+0000","last_change":"2023-10-09T06:19:06.671630+0000","last_active":"2023-10-09T06:28:13.723888+0000","last_peered":"2023-10-09T06:28:13.723888+0000","last_clean":"2023-10-09T06:28:13.723888+0000","last_became_active":"2023-10-09T06:19:06.671400+0000","last_became_peered":"2023-10-09T06:19:06.671400+0000","last_unstale":"2023-10-09T06:28:13.723888+0000","last_undegraded":"2023-10-09T06:28:13.723888+0000","last_fullsized":"2023-10-09T06:28:13.723888+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.065638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712910+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:28:13.712910+0000","last_peered":"2023-10-09T06:28:13.712910+0000","last_clean":"2023-10-09T06:28:13.712910+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:28:13.712910+0000","last_undegraded":"2023-10-09T06:28:13.712910+0000","last_fullsized":"2023-10-09T06:28:13.712910+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":262,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723449+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:28:13.723449+0000","last_peered":"2023-10-09T06:28:13.723449+0000","last_clean":"2023-10-09T06:28:13.723449+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:28:13.723449+0000","last_undegraded":"2023-10-09T06:28:13.723449+0000","last_fullsized":"2023-10-09T06:28:13.723449+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"326'6256","reported_seq":6308,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:18.873409+0000","last_change":"2023-10-09T06:27:38.421674+0000","last_active":"2023-10-09T06:28:18.873409+0000","last_peered":"2023-10-09T06:28:18.873409+0000","last_clean":"2023-10-09T06:28:18.873409+0000","last_became_active":"2023-10-09T06:27:37.645839+0000","last_became_peered":"2023-10-09T06:27:37.645839+0000","last_unstale":"2023-10-09T06:28:18.873409+0000","last_undegraded":"2023-10-09T06:28:18.873409+0000","last_fullsized":"2023-10-09T06:28:18.873409+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:08.252652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341442560,"num_objects":5210,"num_object_clones":0,"num_object_copies":15630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":366912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":622,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.734953+0000","last_change":"2023-10-09T06:28:14.734953+0000","last_active":"2023-10-09T06:28:14.734953+0000","last_peered":"2023-10-09T06:28:14.734953+0000","last_clean":"2023-10-09T06:28:14.734953+0000","last_became_active":"2023-10-09T06:28:14.734606+0000","last_became_peered":"2023-10-09T06:28:14.734606+0000","last_unstale":"2023-10-09T06:28:14.734953+0000","last_undegraded":"2023-10-09T06:28:14.734953+0000","last_fullsized":"2023-10-09T06:28:14.734953+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:07.167587+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.713703+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:28:13.713703+0000","last_peered":"2023-10-09T06:28:13.713703+0000","last_clean":"2023-10-09T06:28:13.713703+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:28:13.713703+0000","last_undegraded":"2023-10-09T06:28:13.713703+0000","last_fullsized":"2023-10-09T06:28:13.713703+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":287,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711320+0000","last_change":"2023-10-09T06:08:16.097356+0000","last_active":"2023-10-09T06:28:13.711320+0000","last_peered":"2023-10-09T06:28:13.711320+0000","last_clean":"2023-10-09T06:28:13.711320+0000","last_became_active":"2023-10-09T06:08:16.097035+0000","last_became_peered":"2023-10-09T06:08:16.097035+0000","last_unstale":"2023-10-09T06:28:13.711320+0000","last_undegraded":"2023-10-09T06:28:13.711320+0000","last_fullsized":"2023-10-09T06:28:13.711320+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:33.656884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":292,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723853+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:28:13.723853+0000","last_peered":"2023-10-09T06:28:13.723853+0000","last_clean":"2023-10-09T06:28:13.723853+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:28:13.723853+0000","last_undegraded":"2023-10-09T06:28:13.723853+0000","last_fullsized":"2023-10-09T06:28:13.723853+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"326'6113","reported_seq":6175,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:28:18.887937+0000","last_change":"2023-10-09T06:28:12.952558+0000","last_active":"2023-10-09T06:28:18.887937+0000","last_peered":"2023-10-09T06:28:18.887937+0000","last_clean":"2023-10-09T06:28:10.671880+0000","last_became_active":"2023-10-09T06:28:12.941599+0000","last_became_peered":"2023-10-09T06:28:12.941599+0000","last_unstale":"2023-10-09T06:28:18.887937+0000","last_undegraded":"2023-10-09T06:28:12.742169+0000","last_fullsized":"2023-10-09T06:28:12.689455+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:48.360791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337969152,"num_objects":5157,"num_object_clones":0,"num_object_copies":15471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6113,"num_write_kb":360640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6],"avail_no_missing":["3","0","6"],"object_location_counts":[{"shards":"0,3,6","objects":5157},{"shards":"3,6","objects":-254}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":618,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718396+0000","last_change":"2023-10-09T06:21:29.742324+0000","last_active":"2023-10-09T06:28:13.718396+0000","last_peered":"2023-10-09T06:28:13.718396+0000","last_clean":"2023-10-09T06:28:13.718396+0000","last_became_active":"2023-10-09T06:21:17.840943+0000","last_became_peered":"2023-10-09T06:21:17.840943+0000","last_unstale":"2023-10-09T06:28:13.718396+0000","last_undegraded":"2023-10-09T06:28:13.718396+0000","last_fullsized":"2023-10-09T06:28:13.718396+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:58.221775+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":311,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715536+0000","last_change":"2023-10-09T06:27:36.674413+0000","last_active":"2023-10-09T06:28:13.715536+0000","last_peered":"2023-10-09T06:28:13.715536+0000","last_clean":"2023-10-09T06:28:13.715536+0000","last_became_active":"2023-10-09T06:27:36.674092+0000","last_became_peered":"2023-10-09T06:27:36.674092+0000","last_unstale":"2023-10-09T06:28:13.715536+0000","last_undegraded":"2023-10-09T06:28:13.715536+0000","last_fullsized":"2023-10-09T06:28:13.715536+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:35.418640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723415+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:28:13.723415+0000","last_peered":"2023-10-09T06:28:13.723415+0000","last_clean":"2023-10-09T06:28:13.723415+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:28:13.723415+0000","last_undegraded":"2023-10-09T06:28:13.723415+0000","last_fullsized":"2023-10-09T06:28:13.723415+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.725981+0000","last_change":"2023-10-09T06:28:11.701778+0000","last_active":"2023-10-09T06:28:13.725981+0000","last_peered":"2023-10-09T06:28:13.725981+0000","last_clean":"2023-10-09T06:28:13.725981+0000","last_became_active":"2023-10-09T06:28:11.701459+0000","last_became_peered":"2023-10-09T06:28:11.701459+0000","last_unstale":"2023-10-09T06:28:13.725981+0000","last_undegraded":"2023-10-09T06:28:13.725981+0000","last_fullsized":"2023-10-09T06:28:13.725981+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:58.853301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"326'6080","reported_seq":6133,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:28:18.900533+0000","last_change":"2023-10-09T06:28:12.977126+0000","last_active":"2023-10-09T06:28:18.900533+0000","last_peered":"2023-10-09T06:28:18.900533+0000","last_clean":"2023-10-09T06:28:10.661033+0000","last_became_active":"2023-10-09T06:28:12.961378+0000","last_became_peered":"2023-10-09T06:28:12.961378+0000","last_unstale":"2023-10-09T06:28:18.900533+0000","last_undegraded":"2023-10-09T06:28:12.731449+0000","last_fullsized":"2023-10-09T06:28:12.692894+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:15.023695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332136448,"num_objects":5068,"num_object_clones":0,"num_object_copies":15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5574,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6080,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":5068},{"shards":"3,6","objects":-261}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":13,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.714010+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:28:13.714010+0000","last_peered":"2023-10-09T06:28:13.714010+0000","last_clean":"2023-10-09T06:28:13.714010+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:28:13.714010+0000","last_undegraded":"2023-10-09T06:28:13.714010+0000","last_fullsized":"2023-10-09T06:28:13.714010+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":599,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718281+0000","last_change":"2023-10-09T06:21:27.787513+0000","last_active":"2023-10-09T06:28:13.718281+0000","last_peered":"2023-10-09T06:28:13.718281+0000","last_clean":"2023-10-09T06:28:13.718281+0000","last_became_active":"2023-10-09T06:21:12.762593+0000","last_became_peered":"2023-10-09T06:21:12.762593+0000","last_unstale":"2023-10-09T06:28:13.718281+0000","last_undegraded":"2023-10-09T06:28:13.718281+0000","last_fullsized":"2023-10-09T06:28:13.718281+0000","mapping_epoch":192,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:07.848234+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712067+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:28:13.712067+0000","last_peered":"2023-10-09T06:28:13.712067+0000","last_clean":"2023-10-09T06:28:13.712067+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:28:13.712067+0000","last_undegraded":"2023-10-09T06:28:13.712067+0000","last_fullsized":"2023-10-09T06:28:13.712067+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.717243+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:28:13.717243+0000","last_peered":"2023-10-09T06:28:13.717243+0000","last_clean":"2023-10-09T06:28:13.717243+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:28:13.717243+0000","last_undegraded":"2023-10-09T06:28:13.717243+0000","last_fullsized":"2023-10-09T06:28:13.717243+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"326'6081","reported_seq":6141,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:18.894437+0000","last_change":"2023-10-09T06:27:38.936723+0000","last_active":"2023-10-09T06:28:18.894437+0000","last_peered":"2023-10-09T06:28:18.894437+0000","last_clean":"2023-10-09T06:28:18.894437+0000","last_became_active":"2023-10-09T06:27:37.645689+0000","last_became_peered":"2023-10-09T06:27:37.645689+0000","last_unstale":"2023-10-09T06:28:18.894437+0000","last_undegraded":"2023-10-09T06:28:18.894437+0000","last_fullsized":"2023-10-09T06:28:18.894437+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6081,"log_dups_size":0,"ondisk_log_size":6081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:01.547318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335740928,"num_objects":5123,"num_object_clones":0,"num_object_copies":15369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6081,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":1441792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":610,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723965+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:28:13.723965+0000","last_peered":"2023-10-09T06:28:13.723965+0000","last_clean":"2023-10-09T06:28:13.723965+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:28:13.723965+0000","last_undegraded":"2023-10-09T06:28:13.723965+0000","last_fullsized":"2023-10-09T06:28:13.723965+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":169,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.718199+0000","last_change":"2023-10-09T06:19:06.688983+0000","last_active":"2023-10-09T06:28:13.718199+0000","last_peered":"2023-10-09T06:28:13.718199+0000","last_clean":"2023-10-09T06:28:13.718199+0000","last_became_active":"2023-10-09T06:19:06.688508+0000","last_became_peered":"2023-10-09T06:19:06.688508+0000","last_unstale":"2023-10-09T06:28:13.718199+0000","last_undegraded":"2023-10-09T06:28:13.718199+0000","last_fullsized":"2023-10-09T06:28:13.718199+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:36.994094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":298,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715916+0000","last_change":"2023-10-09T06:27:36.683415+0000","last_active":"2023-10-09T06:28:13.715916+0000","last_peered":"2023-10-09T06:28:13.715916+0000","last_clean":"2023-10-09T06:28:13.715916+0000","last_became_active":"2023-10-09T06:27:36.678284+0000","last_became_peered":"2023-10-09T06:27:36.678284+0000","last_unstale":"2023-10-09T06:28:13.715916+0000","last_undegraded":"2023-10-09T06:28:13.715916+0000","last_fullsized":"2023-10-09T06:28:13.715916+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:01.115078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":303,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.707470+0000","last_change":"2023-10-09T06:21:45.400880+0000","last_active":"2023-10-09T06:28:13.707470+0000","last_peered":"2023-10-09T06:28:13.707470+0000","last_clean":"2023-10-09T06:28:13.707470+0000","last_became_active":"2023-10-09T06:21:45.400592+0000","last_became_peered":"2023-10-09T06:21:45.400592+0000","last_unstale":"2023-10-09T06:28:13.707470+0000","last_undegraded":"2023-10-09T06:28:13.707470+0000","last_fullsized":"2023-10-09T06:28:13.707470+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:08:07.445238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"326'6234","reported_seq":6272,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:22.167674+0000","last_change":"2023-10-09T06:26:28.775392+0000","last_active":"2023-10-09T06:28:22.167674+0000","last_peered":"2023-10-09T06:28:22.167674+0000","last_clean":"2023-10-09T06:28:22.167674+0000","last_became_active":"2023-10-09T06:26:28.774522+0000","last_became_peered":"2023-10-09T06:26:28.774522+0000","last_unstale":"2023-10-09T06:28:22.167674+0000","last_undegraded":"2023-10-09T06:28:22.167674+0000","last_fullsized":"2023-10-09T06:28:22.167674+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6234,"log_dups_size":0,"ondisk_log_size":6234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:02.490573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6234,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":588,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.715317+0000","last_change":"2023-10-09T06:27:39.657859+0000","last_active":"2023-10-09T06:28:13.715317+0000","last_peered":"2023-10-09T06:28:13.715317+0000","last_clean":"2023-10-09T06:28:13.715317+0000","last_became_active":"2023-10-09T06:27:39.657516+0000","last_became_peered":"2023-10-09T06:27:39.657516+0000","last_unstale":"2023-10-09T06:28:13.715317+0000","last_undegraded":"2023-10-09T06:28:13.715317+0000","last_fullsized":"2023-10-09T06:28:13.715317+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:43.689046+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":169,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.719541+0000","last_change":"2023-10-09T06:19:06.758054+0000","last_active":"2023-10-09T06:28:13.719541+0000","last_peered":"2023-10-09T06:28:13.719541+0000","last_clean":"2023-10-09T06:28:13.719541+0000","last_became_active":"2023-10-09T06:19:06.757104+0000","last_became_peered":"2023-10-09T06:19:06.757104+0000","last_unstale":"2023-10-09T06:28:13.719541+0000","last_undegraded":"2023-10-09T06:28:13.719541+0000","last_fullsized":"2023-10-09T06:28:13.719541+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:52.920078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":318,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.723816+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:28:13.723816+0000","last_peered":"2023-10-09T06:28:13.723816+0000","last_clean":"2023-10-09T06:28:13.723816+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:28:13.723816+0000","last_undegraded":"2023-10-09T06:28:13.723816+0000","last_fullsized":"2023-10-09T06:28:13.723816+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":301,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.707888+0000","last_change":"2023-10-09T06:28:11.701561+0000","last_active":"2023-10-09T06:28:13.707888+0000","last_peered":"2023-10-09T06:28:13.707888+0000","last_clean":"2023-10-09T06:28:13.707888+0000","last_became_active":"2023-10-09T06:28:11.701201+0000","last_became_peered":"2023-10-09T06:28:11.701201+0000","last_unstale":"2023-10-09T06:28:13.707888+0000","last_undegraded":"2023-10-09T06:28:13.707888+0000","last_fullsized":"2023-10-09T06:28:13.707888+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:01.807960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"326'6200","reported_seq":6237,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:18.191957+0000","last_change":"2023-10-09T06:26:28.777254+0000","last_active":"2023-10-09T06:28:18.191957+0000","last_peered":"2023-10-09T06:28:18.191957+0000","last_clean":"2023-10-09T06:28:18.191957+0000","last_became_active":"2023-10-09T06:26:28.776915+0000","last_became_peered":"2023-10-09T06:26:28.776915+0000","last_unstale":"2023-10-09T06:28:18.191957+0000","last_undegraded":"2023-10-09T06:28:18.191957+0000","last_fullsized":"2023-10-09T06:28:18.191957+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:00:04.121493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344719360,"num_objects":5260,"num_object_clones":0,"num_object_copies":15780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":366720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":609,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711918+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:28:13.711918+0000","last_peered":"2023-10-09T06:28:13.711918+0000","last_clean":"2023-10-09T06:28:13.711918+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:28:13.711918+0000","last_undegraded":"2023-10-09T06:28:13.711918+0000","last_fullsized":"2023-10-09T06:28:13.711918+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":27,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711431+0000","last_change":"2023-10-09T06:27:36.696219+0000","last_active":"2023-10-09T06:28:13.711431+0000","last_peered":"2023-10-09T06:28:13.711431+0000","last_clean":"2023-10-09T06:28:13.711431+0000","last_became_active":"2023-10-09T06:27:36.653183+0000","last_became_peered":"2023-10-09T06:27:36.653183+0000","last_unstale":"2023-10-09T06:28:13.711431+0000","last_undegraded":"2023-10-09T06:28:13.711431+0000","last_fullsized":"2023-10-09T06:28:13.711431+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:16.945012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":286,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.724202+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:28:13.724202+0000","last_peered":"2023-10-09T06:28:13.724202+0000","last_clean":"2023-10-09T06:28:13.724202+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:28:13.724202+0000","last_undegraded":"2023-10-09T06:28:13.724202+0000","last_fullsized":"2023-10-09T06:28:13.724202+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712152+0000","last_change":"2023-10-09T06:27:36.661487+0000","last_active":"2023-10-09T06:28:13.712152+0000","last_peered":"2023-10-09T06:28:13.712152+0000","last_clean":"2023-10-09T06:28:13.712152+0000","last_became_active":"2023-10-09T06:27:36.660598+0000","last_became_peered":"2023-10-09T06:27:36.660598+0000","last_unstale":"2023-10-09T06:28:13.712152+0000","last_undegraded":"2023-10-09T06:28:13.712152+0000","last_fullsized":"2023-10-09T06:28:13.712152+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:52.336583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"326'6433","reported_seq":6470,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:23.061097+0000","last_change":"2023-10-09T06:26:28.775931+0000","last_active":"2023-10-09T06:28:23.061097+0000","last_peered":"2023-10-09T06:28:23.061097+0000","last_clean":"2023-10-09T06:28:23.061097+0000","last_became_active":"2023-10-09T06:26:28.775652+0000","last_became_peered":"2023-10-09T06:26:28.775652+0000","last_unstale":"2023-10-09T06:28:23.061097+0000","last_undegraded":"2023-10-09T06:28:23.061097+0000","last_fullsized":"2023-10-09T06:28:23.061097+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":297,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:26:27.751676+0000","last_clean_scrub_stamp":"2023-10-09T06:26:27.751676+0000","objects_scrubbed":0,"log_size":6433,"log_dups_size":0,"ondisk_log_size":6433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:39.386224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333643776,"num_objects":5091,"num_object_clones":0,"num_object_copies":15273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6433,"num_write_kb":368768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":654,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.734086+0000","last_change":"2023-10-09T06:28:14.734086+0000","last_active":"2023-10-09T06:28:14.734086+0000","last_peered":"2023-10-09T06:28:14.734086+0000","last_clean":"2023-10-09T06:28:14.734086+0000","last_became_active":"2023-10-09T06:28:14.733702+0000","last_became_peered":"2023-10-09T06:28:14.733702+0000","last_unstale":"2023-10-09T06:28:14.734086+0000","last_undegraded":"2023-10-09T06:28:14.734086+0000","last_fullsized":"2023-10-09T06:28:14.734086+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.831911+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":310,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.717305+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:28:13.717305+0000","last_peered":"2023-10-09T06:28:13.717305+0000","last_clean":"2023-10-09T06:28:13.717305+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:28:13.717305+0000","last_undegraded":"2023-10-09T06:28:13.717305+0000","last_fullsized":"2023-10-09T06:28:13.717305+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":284,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.707341+0000","last_change":"2023-10-09T06:13:02.837520+0000","last_active":"2023-10-09T06:28:13.707341+0000","last_peered":"2023-10-09T06:28:13.707341+0000","last_clean":"2023-10-09T06:28:13.707341+0000","last_became_active":"2023-10-09T06:13:02.837209+0000","last_became_peered":"2023-10-09T06:13:02.837209+0000","last_unstale":"2023-10-09T06:28:13.707341+0000","last_undegraded":"2023-10-09T06:28:13.707341+0000","last_fullsized":"2023-10-09T06:28:13.707341+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:52.648434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":289,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711046+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:28:13.711046+0000","last_peered":"2023-10-09T06:28:13.711046+0000","last_clean":"2023-10-09T06:28:13.711046+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:28:13.711046+0000","last_undegraded":"2023-10-09T06:28:13.711046+0000","last_fullsized":"2023-10-09T06:28:13.711046+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":286,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T06:28:14.709899+0000","last_change":"2023-10-09T06:19:06.667310+0000","last_active":"2023-10-09T06:28:14.709899+0000","last_peered":"2023-10-09T06:28:14.709899+0000","last_clean":"2023-10-09T06:28:14.709899+0000","last_became_active":"2023-10-09T06:19:06.666642+0000","last_became_peered":"2023-10-09T06:19:06.666642+0000","last_unstale":"2023-10-09T06:28:14.709899+0000","last_undegraded":"2023-10-09T06:28:14.709899+0000","last_fullsized":"2023-10-09T06:28:14.709899+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:47.396042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":327,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.716414+0000","last_change":"2023-10-09T06:27:36.635337+0000","last_active":"2023-10-09T06:28:13.716414+0000","last_peered":"2023-10-09T06:28:13.716414+0000","last_clean":"2023-10-09T06:28:13.716414+0000","last_became_active":"2023-10-09T06:27:36.634979+0000","last_became_peered":"2023-10-09T06:27:36.634979+0000","last_unstale":"2023-10-09T06:28:13.716414+0000","last_undegraded":"2023-10-09T06:28:13.716414+0000","last_fullsized":"2023-10-09T06:28:13.716414+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:33.257699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":17,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.713632+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:28:13.713632+0000","last_peered":"2023-10-09T06:28:13.713632+0000","last_clean":"2023-10-09T06:28:13.713632+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:28:13.713632+0000","last_undegraded":"2023-10-09T06:28:13.713632+0000","last_fullsized":"2023-10-09T06:28:13.713632+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":333,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711240+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:28:13.711240+0000","last_peered":"2023-10-09T06:28:13.711240+0000","last_clean":"2023-10-09T06:28:13.711240+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:28:13.711240+0000","last_undegraded":"2023-10-09T06:28:13.711240+0000","last_fullsized":"2023-10-09T06:28:13.711240+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":17,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.712833+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:28:13.712833+0000","last_peered":"2023-10-09T06:28:13.712833+0000","last_clean":"2023-10-09T06:28:13.712833+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:28:13.712833+0000","last_undegraded":"2023-10-09T06:28:13.712833+0000","last_fullsized":"2023-10-09T06:28:13.712833+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":323,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711665+0000","last_change":"2023-10-09T06:27:36.634086+0000","last_active":"2023-10-09T06:28:13.711665+0000","last_peered":"2023-10-09T06:28:13.711665+0000","last_clean":"2023-10-09T06:28:13.711665+0000","last_became_active":"2023-10-09T06:27:36.633802+0000","last_became_peered":"2023-10-09T06:27:36.633802+0000","last_unstale":"2023-10-09T06:28:13.711665+0000","last_undegraded":"2023-10-09T06:28:13.711665+0000","last_fullsized":"2023-10-09T06:28:13.711665+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:04.293924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711279+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:28:13.711279+0000","last_peered":"2023-10-09T06:28:13.711279+0000","last_clean":"2023-10-09T06:28:13.711279+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:28:13.711279+0000","last_undegraded":"2023-10-09T06:28:13.711279+0000","last_fullsized":"2023-10-09T06:28:13.711279+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.710792+0000","last_change":"2023-10-09T06:27:36.666424+0000","last_active":"2023-10-09T06:28:13.710792+0000","last_peered":"2023-10-09T06:28:13.710792+0000","last_clean":"2023-10-09T06:28:13.710792+0000","last_became_active":"2023-10-09T06:27:36.665914+0000","last_became_peered":"2023-10-09T06:27:36.665914+0000","last_unstale":"2023-10-09T06:28:13.710792+0000","last_undegraded":"2023-10-09T06:28:13.710792+0000","last_fullsized":"2023-10-09T06:28:13.710792+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:05:50.118755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":314,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.711369+0000","last_change":"2023-10-09T06:28:11.714379+0000","last_active":"2023-10-09T06:28:13.711369+0000","last_peered":"2023-10-09T06:28:13.711369+0000","last_clean":"2023-10-09T06:28:13.711369+0000","last_became_active":"2023-10-09T06:28:11.700535+0000","last_became_peered":"2023-10-09T06:28:11.700535+0000","last_unstale":"2023-10-09T06:28:13.711369+0000","last_undegraded":"2023-10-09T06:28:13.711369+0000","last_fullsized":"2023-10-09T06:28:13.711369+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:54.368541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":286,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T06:28:13.710811+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:28:13.710811+0000","last_peered":"2023-10-09T06:28:13.710811+0000","last_clean":"2023-10-09T06:28:13.710811+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:28:13.710811+0000","last_undegraded":"2023-10-09T06:28:13.710811+0000","last_fullsized":"2023-10-09T06:28:13.710811+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5355929624,"num_objects":81726,"num_object_clones":0,"num_object_copies":245178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16974,"num_objects_misplaced":5125,"num_objects_unfound":0,"num_objects_dirty":81726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99142,"num_write_kb":5787713,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":66256896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16074092544,"data_stored":16074080328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99142,"ondisk_log_size":99142,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":11274,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234372,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2922848,"kb_used_data":2014424,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":90822304,"statfs":{"total":95995035648,"available":93002039296,"internally_reserved":0,"allocated":2062770176,"data_stored":2061939308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:54 2023","interfaces":[{"interface":"back","average":{"1min":12.352,"5min":7.7060000000000004,"15min":4.5469999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":215.97800000000001,"5min":215.97800000000001,"15min":219.41900000000001},"last":34.536999999999999},{"interface":"front","average":{"1min":12.484999999999999,"5min":7.899,"15min":4.5780000000000003},"min":{"1min":0.27200000000000002,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":215.577,"5min":215.577,"15min":219.399},"last":34.466000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:27:28 2023","interfaces":[{"interface":"back","average":{"1min":13.733000000000001,"5min":5.8890000000000002,"15min":3.6659999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.20599999999999999},"max":{"1min":215.61699999999999,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":13.404999999999999,"5min":6.1360000000000001,"15min":3.468},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":216.22200000000001,"5min":216.22200000000001,"15min":216.22200000000001},"last":0.64600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":14.24,"5min":12.477,"15min":5.9989999999999997},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":216.34299999999999,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":14.183,"5min":12.965999999999999,"15min":6.2060000000000004},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":216.25200000000001,"5min":216.25200000000001,"15min":216.25200000000001},"last":0.68100000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":4.4770000000000003,"15min":2.673},"min":{"1min":0.37,"5min":0.33000000000000002,"15min":0.26100000000000001},"max":{"1min":11.84,"5min":215.922,"15min":215.922},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.8220000000000001,"5min":4.4390000000000001,"15min":2.7240000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":8.6950000000000003,"5min":215.94900000000001,"15min":215.94900000000001},"last":0.66700000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":4.7910000000000004,"15min":2.629},"min":{"1min":0.253,"5min":0.245,"15min":0.23300000000000001},"max":{"1min":4.0110000000000001,"5min":218.446,"15min":218.446},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.74199999999999999,"5min":4.6310000000000002,"15min":2.3260000000000001},"min":{"1min":0.31,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.4690000000000003,"5min":216.172,"15min":216.172},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:28:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":3.8119999999999998,"15min":2.6840000000000002},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":12.324999999999999,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.621},{"interface":"front","average":{"1min":0.63200000000000001,"5min":3.9039999999999999,"15min":2.387},"min":{"1min":0.35999999999999999,"5min":0.28999999999999998,"15min":0.28299999999999997},"max":{"1min":3.2669999999999999,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:28:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":4.0910000000000002,"15min":2.2930000000000001},"min":{"1min":0.317,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6559999999999999,"5min":218.19999999999999,"15min":218.19999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":3.831,"15min":2.1459999999999999},"min":{"1min":0.38500000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.5469999999999997,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.58899999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496008,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3504592,"kb_used_data":2601480,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":90240560,"statfs":{"total":95995035648,"available":92406333440,"internally_reserved":0,"allocated":2663915520,"data_stored":2663100326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:03 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":2.0910000000000002,"15min":2.9399999999999999},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.222},"max":{"1min":3.7189999999999999,"5min":58.270000000000003,"15min":110.926},"last":0.378},{"interface":"front","average":{"1min":1.694,"5min":2.0139999999999998,"15min":3.2029999999999998},"min":{"1min":0.28999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.895,"5min":58.253999999999998,"15min":110.911},"last":3.383}]},{"osd":1,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":18.265000000000001,"5min":5.9909999999999997,"15min":4.5300000000000002},"min":{"1min":0.23999999999999999,"5min":0.215,"15min":0.215},"max":{"1min":292.92200000000003,"5min":292.92200000000003,"15min":292.92200000000003},"last":6.9189999999999996},{"interface":"front","average":{"1min":18.207999999999998,"5min":5.9169999999999998,"15min":4.3399999999999999},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.246},"max":{"1min":292.904,"5min":292.904,"15min":292.904},"last":0.311}]},{"osd":2,"last update":"Mon Oct 9 06:27:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":3.464,"15min":2.601},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.9780000000000002,"5min":75.191999999999993,"15min":75.191999999999993},"last":3.3490000000000002},{"interface":"front","average":{"1min":0.80300000000000005,"5min":3.6099999999999999,"15min":2.4689999999999999},"min":{"1min":0.36899999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.21,"5min":75.120000000000005,"15min":75.120000000000005},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:28:03 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":1.9770000000000001,"15min":2.125},"min":{"1min":0.42399999999999999,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":9.4740000000000002,"5min":11.148,"15min":110.358},"last":2.9020000000000001},{"interface":"front","average":{"1min":2.988,"5min":2.0670000000000002,"15min":2.2360000000000002},"min":{"1min":0.318,"5min":0.311,"15min":0.311},"max":{"1min":11.488,"5min":11.488,"15min":110.43899999999999},"last":0.439}]},{"osd":4,"last update":"Mon Oct 9 06:27:43 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.93899999999999995,"15min":1.2949999999999999},"min":{"1min":0.29299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.4300000000000002,"5min":4.1890000000000001,"15min":17.643999999999998},"last":3.2570000000000001},{"interface":"front","average":{"1min":0.625,"5min":1.127,"15min":1.2490000000000001},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.26400000000000001},"max":{"1min":2.464,"5min":6.8780000000000001,"15min":10.563000000000001},"last":0.34499999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:27:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.0009999999999999,"15min":1.4019999999999999},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.28499999999999998},"max":{"1min":2.508,"5min":5.2460000000000004,"15min":22.379999999999999},"last":3.3140000000000001},{"interface":"front","average":{"1min":0.67900000000000005,"5min":1.044,"15min":1.369},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":2.512,"5min":4.8920000000000003,"15min":24.401},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":4.0670000000000002,"15min":2.4609999999999999},"min":{"1min":0.35199999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1629999999999998,"5min":122.937,"15min":122.937},"last":3.7450000000000001},{"interface":"front","average":{"1min":0.89100000000000001,"5min":3.9710000000000001,"15min":2.629},"min":{"1min":0.38500000000000001,"5min":0.33900000000000002,"15min":0.32800000000000001},"max":{"1min":2.5630000000000002,"5min":122.92100000000001,"15min":122.92100000000001},"last":3.734}]}]},{"osd":1,"up_from":110,"seq":472446402746,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3572332,"kb_used_data":2958756,"kb_used_omap":4,"kb_used_meta":613435,"kb_avail":90172820,"statfs":{"total":95995035648,"available":92336967680,"internally_reserved":0,"allocated":3029766144,"data_stored":3028339455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":628158350},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":6.7539999999999996,"15min":3.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.189,"15min":0.189},"max":{"1min":3.6720000000000002,"5min":145.96600000000001,"15min":148.98400000000001},"last":6.2720000000000002},{"interface":"front","average":{"1min":1.359,"5min":7.5810000000000004,"15min":4.3479999999999999},"min":{"1min":0.24199999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":8.7669999999999995,"5min":145.982,"15min":149.02699999999999},"last":6.319}]},{"osd":2,"last update":"Mon Oct 9 06:27:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":5.4420000000000002,"15min":2.9900000000000002},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.193},"max":{"1min":2.6800000000000002,"5min":139.75399999999999,"15min":139.75399999999999},"last":2.8620000000000001},{"interface":"front","average":{"1min":0.82299999999999995,"5min":6.4370000000000003,"15min":3.7109999999999999},"min":{"1min":0.26000000000000001,"5min":0.223,"15min":0.219},"max":{"1min":3.8199999999999998,"5min":140.29900000000001,"15min":140.29900000000001},"last":4.3179999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:28:14 2023","interfaces":[{"interface":"back","average":{"1min":4.5220000000000002,"5min":3.9860000000000002,"15min":2.8839999999999999},"min":{"1min":0.27500000000000002,"5min":0.24399999999999999,"15min":0.215},"max":{"1min":31.062000000000001,"5min":77.331999999999994,"15min":153.18199999999999},"last":3.5150000000000001},{"interface":"front","average":{"1min":3.1349999999999998,"5min":2.827,"15min":2.1509999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":27.657,"5min":77.197000000000003,"15min":152.52000000000001},"last":6.0460000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":4.5069999999999997,"15min":3.4820000000000002},"min":{"1min":0.311,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.0259999999999998,"5min":99.391000000000005,"15min":199.43799999999999},"last":6.4950000000000001},{"interface":"front","average":{"1min":0.88,"5min":4.4870000000000001,"15min":3.4209999999999998},"min":{"1min":0.42999999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.71,"5min":99.462000000000003,"15min":199.32900000000001},"last":5.6529999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":2.306,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":2.6059999999999999,"5min":77.436999999999998,"15min":77.436999999999998},"last":6.2290000000000001},{"interface":"front","average":{"1min":0.89800000000000002,"5min":2.9089999999999998,"15min":2.6150000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"max":{"1min":3.6560000000000001,"5min":77.388000000000005,"15min":77.388000000000005},"last":6.1879999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":3.1989999999999998,"15min":2.7149999999999999},"min":{"1min":0.40500000000000003,"5min":0.34699999999999998,"15min":0.28399999999999997},"max":{"1min":3.7909999999999999,"5min":77.292000000000002,"15min":77.292000000000002},"last":6.2169999999999996},{"interface":"front","average":{"1min":1.359,"5min":3.0499999999999998,"15min":2.5649999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.6589999999999998,"5min":77.378,"15min":77.378},"last":6.577}]},{"osd":7,"last update":"Mon Oct 9 06:27:27 2023","interfaces":[{"interface":"back","average":{"1min":6.157,"5min":3.871,"15min":2.8170000000000002},"min":{"1min":0.36499999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":55.231000000000002,"5min":77.456999999999994,"15min":77.456999999999994},"last":14.5},{"interface":"front","average":{"1min":5.7199999999999998,"5min":3.3980000000000001,"15min":2.3719999999999999},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":55.253999999999998,"5min":77.359999999999999,"15min":77.359999999999999},"last":14.462}]}]},{"osd":0,"up_from":91,"seq":390842024179,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3044980,"kb_used_data":2302892,"kb_used_omap":5,"kb_used_meta":742074,"kb_avail":90700172,"statfs":{"total":95995035648,"available":92876976128,"internally_reserved":0,"allocated":2358161408,"data_stored":2356496542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":759884410},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:27:21 2023","interfaces":[{"interface":"back","average":{"1min":9.7379999999999995,"5min":3.6320000000000001,"15min":3.3540000000000001},"min":{"1min":0.20300000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":191.446,"5min":191.446,"15min":191.446},"last":0.82899999999999996},{"interface":"front","average":{"1min":9.4390000000000001,"5min":3.8279999999999998,"15min":3.8149999999999999},"min":{"1min":0.24399999999999999,"5min":0.23699999999999999,"15min":0.17499999999999999},"max":{"1min":191.393,"5min":191.393,"15min":191.393},"last":0.79800000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:27:39 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":3.129,"15min":2.4239999999999999},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":51.886000000000003,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.8929999999999998,"5min":3.609,"15min":2.6339999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":52.302999999999997,"5min":57.585000000000001,"15min":57.585000000000001},"last":1.9079999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:28:04 2023","interfaces":[{"interface":"back","average":{"1min":13.316000000000001,"5min":4.6550000000000002,"15min":3.819},"min":{"1min":0.27000000000000002,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":208.11199999999999,"5min":208.11199999999999,"15min":208.11199999999999},"last":2.0960000000000001},{"interface":"front","average":{"1min":13.005000000000001,"5min":4.4500000000000002,"15min":3.9039999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":208.25899999999999,"5min":208.25899999999999,"15min":208.25899999999999},"last":2.0600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:27:54 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.512,"15min":1.3440000000000001},"min":{"1min":0.29999999999999999,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":6.8579999999999997,"5min":6.8579999999999997,"15min":11.162000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.613,"15min":1.7709999999999999},"min":{"1min":0.33200000000000002,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":10.118,"5min":10.118,"15min":12.752000000000001},"last":5.8289999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:27:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":1.518,"15min":1.754},"min":{"1min":0.34899999999999998,"5min":0.32100000000000001,"15min":0.27800000000000002},"max":{"1min":10.865,"5min":10.865,"15min":62.984999999999999},"last":3.4740000000000002},{"interface":"front","average":{"1min":1.804,"5min":1.9410000000000001,"15min":2.2149999999999999},"min":{"1min":0.377,"5min":0.35199999999999998,"15min":0.29199999999999998},"max":{"1min":8.5530000000000008,"5min":11.898,"15min":64.983999999999995},"last":1.9690000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:27:54 2023","interfaces":[{"interface":"back","average":{"1min":7.5300000000000002,"5min":3.2709999999999999,"15min":2.4590000000000001},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":120.572,"5min":120.572,"15min":120.572},"last":1.9930000000000001},{"interface":"front","average":{"1min":7.2489999999999997,"5min":2.7610000000000001,"15min":2.0019999999999998},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":120.64400000000001,"5min":120.64400000000001,"15min":120.64400000000001},"last":2.9820000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:27:54 2023","interfaces":[{"interface":"back","average":{"1min":7.0300000000000002,"5min":2.919,"15min":2.4169999999999998},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32300000000000001},"max":{"1min":122.018,"5min":122.018,"15min":122.018},"last":5.9000000000000004},{"interface":"front","average":{"1min":7.7709999999999999,"5min":3.3490000000000002,"15min":2.6760000000000002},"min":{"1min":0.45600000000000002,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":122.32299999999999,"5min":122.32299999999999,"15min":122.32299999999999},"last":2.121}]}]},{"osd":2,"up_from":128,"seq":549755814048,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2429588,"kb_used_data":1952844,"kb_used_omap":7,"kb_used_meta":476536,"kb_avail":91315564,"statfs":{"total":95995035648,"available":93507137536,"internally_reserved":0,"allocated":1999712256,"data_stored":1998304968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":487972945},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":4.7220000000000004,"15min":3.887},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":9.9139999999999997,"5min":150.32599999999999,"15min":163.40100000000001},"last":162.21100000000001},{"interface":"front","average":{"1min":1.498,"5min":3.4300000000000002,"15min":3.7160000000000002},"min":{"1min":0.20300000000000001,"5min":0.182,"15min":0.182},"max":{"1min":5.8380000000000001,"5min":110.66500000000001,"15min":163.363},"last":168.49700000000001}]},{"osd":1,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":4.2560000000000002,"15min":3.6360000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":8.2409999999999997,"5min":205.148,"15min":222.50299999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.266,"5min":4.0099999999999998,"15min":3.887},"min":{"1min":0.28699999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":9.4019999999999992,"5min":205.267,"15min":222.19399999999999},"last":0.33100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.948,"15min":1.907},"min":{"1min":0.26900000000000002,"5min":0.26500000000000001,"15min":0.22700000000000001},"max":{"1min":6.0609999999999999,"5min":16.331,"15min":81.183999999999997},"last":0.26800000000000002},{"interface":"front","average":{"1min":3.4129999999999998,"5min":2.4489999999999998,"15min":2.1070000000000002},"min":{"1min":0.318,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.368,"5min":16.364000000000001,"15min":81.061999999999998},"last":0.30199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":2.8730000000000002,"15min":2.0800000000000001},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.9329999999999998,"5min":90.938999999999993,"15min":90.938999999999993},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.339,"5min":2.948,"15min":2.347},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.890000000000001,"5min":90.950000000000003,"15min":90.950000000000003},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.4970000000000001,"15min":1.464},"min":{"1min":0.318,"5min":0.317,"15min":0.309},"max":{"1min":6.0069999999999997,"5min":8.9589999999999996,"15min":8.9589999999999996},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.4299999999999999,"15min":1.9359999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.859,"5min":10.859,"15min":10.859},"last":0.65900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":3.5,"15min":2.4460000000000002},"min":{"1min":0.312,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.82,"5min":138.291,"15min":138.291},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.103,"5min":3.4990000000000001,"15min":2.1160000000000001},"min":{"1min":0.42299999999999999,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":5.3769999999999998,"5min":138.01599999999999,"15min":138.01599999999999},"last":0.624}]},{"osd":7,"last update":"Mon Oct 9 06:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":2.3149999999999999,"15min":1.798},"min":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.36499999999999999},"max":{"1min":5.4050000000000002,"5min":55.851999999999997,"15min":55.851999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.2969999999999999,"5min":2.0859999999999999,"15min":2.1280000000000001},"min":{"1min":0.40600000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.9749999999999996,"5min":55.731000000000002,"15min":55.731000000000002},"last":0.53000000000000003}]}]},{"osd":3,"up_from":149,"seq":639950127245,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3127932,"kb_used_data":2673396,"kb_used_omap":2,"kb_used_meta":454525,"kb_avail":90617220,"statfs":{"total":95995035648,"available":92792033280,"internally_reserved":0,"allocated":2737557504,"data_stored":2736232234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":465433653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":4.2599999999999998,"15min":3.4249999999999998},"min":{"1min":0.26200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.056,"5min":105.252,"15min":187.852},"last":0.248},{"interface":"front","average":{"1min":1.5760000000000001,"5min":3.8450000000000002,"15min":3.8029999999999999},"min":{"1min":0.247,"5min":0.184,"15min":0.184},"max":{"1min":6.6859999999999999,"5min":105.331,"15min":187.923},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2909999999999999,"5min":2.0059999999999998,"15min":2.9750000000000001},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":8.7829999999999995,"5min":22.045999999999999,"15min":187.89099999999999},"last":1.8120000000000001},{"interface":"front","average":{"1min":3.4039999999999999,"5min":2.6859999999999999,"15min":2.1600000000000001},"min":{"1min":0.33100000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.194000000000001,"5min":31.053000000000001,"15min":67.028000000000006},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.9279999999999999,"15min":2.7080000000000002},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.23999999999999999},"max":{"1min":6.7320000000000002,"5min":39.951999999999998,"15min":187.76400000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.998,"5min":2.4729999999999999,"15min":2.2749999999999999},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.22500000000000001},"max":{"1min":9.6219999999999999,"5min":25.530999999999999,"15min":187.79499999999999},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.534,"15min":1.1870000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":7.1109999999999998,"5min":22.585999999999999,"15min":22.585999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.306,"5min":1.466,"15min":1.9790000000000001},"min":{"1min":0.35899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.9539999999999997,"5min":15.942,"15min":187.67400000000001},"last":1.782}]},{"osd":5,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.4470000000000001,"15min":2.1259999999999999},"min":{"1min":0.40799999999999997,"5min":0.378,"15min":0.29899999999999999},"max":{"1min":4.9779999999999998,"5min":16.001999999999999,"15min":187.833},"last":8.6159999999999997},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.0649999999999999,"15min":2.0139999999999998},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":3.1909999999999998,"5min":16.167999999999999,"15min":187.952},"last":0.42299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.8160000000000001,"15min":2.1739999999999999},"min":{"1min":0.39700000000000002,"5min":0.34899999999999998,"15min":0.33700000000000002},"max":{"1min":4.8799999999999999,"5min":15.976000000000001,"15min":187.78200000000001},"last":7.3630000000000004},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.22,"15min":1.9670000000000001},"min":{"1min":0.34399999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.1899999999999999,"5min":16.134,"15min":187.81800000000001},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":1.762,"15min":1.956},"min":{"1min":0.438,"5min":0.37,"15min":0.34100000000000003},"max":{"1min":7.1399999999999997,"5min":22.533000000000001,"15min":192.20400000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.7410000000000001,"15min":2.129},"min":{"1min":0.47099999999999997,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":4.9199999999999999,"5min":15.901999999999999,"15min":187.86500000000001},"last":5.3639999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019282,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181144,"kb_used_data":1294352,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":91564008,"statfs":{"total":95995035648,"available":93761544192,"internally_reserved":0,"allocated":1325416448,"data_stored":1324593439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":3.6890000000000001,"15min":2.153},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.8929999999999998,"5min":76.944999999999993,"15min":76.944999999999993},"last":0.374},{"interface":"front","average":{"1min":0.66300000000000003,"5min":3.915,"15min":2.5680000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.23999999999999999},"max":{"1min":2.907,"5min":76.988,"15min":76.988},"last":0.441}]},{"osd":1,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.1180000000000001,"15min":1.996},"min":{"1min":0.26700000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.4480000000000004,"5min":9.4480000000000004,"15min":91.849999999999994},"last":0.621},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.341,"15min":2.101},"min":{"1min":0.28199999999999997,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":3.1640000000000001,"5min":9.625,"15min":91.885000000000005},"last":2.23}]},{"osd":2,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.569,"15min":2.3039999999999998},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":6.5179999999999998,"5min":69.640000000000001,"15min":69.640000000000001},"last":1.371},{"interface":"front","average":{"1min":0.95299999999999996,"5min":3.7789999999999999,"15min":2.4670000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.9209999999999998,"5min":69.676000000000002,"15min":69.676000000000002},"last":1.9119999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:28:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1400000000000001,"5min":2.3380000000000001,"15min":2.468},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":36.235999999999997,"5min":45.591000000000001,"15min":91.697000000000003},"last":1.5820000000000001},{"interface":"front","average":{"1min":2.8860000000000001,"5min":2.028,"15min":2.2930000000000001},"min":{"1min":0.34799999999999998,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":35.344999999999999,"5min":45.548999999999999,"15min":94.356999999999999},"last":2.0449999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:27:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.93100000000000005,"15min":1.1639999999999999},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.24299999999999999},"max":{"1min":3.5270000000000001,"5min":6.2670000000000003,"15min":10.673},"last":1.252},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.85499999999999998,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.0259999999999998,"5min":5.5,"15min":12.552},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59499999999999997,"15min":0.83299999999999996},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":1.51,"5min":2.1179999999999999,"15min":9.7680000000000007},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.91600000000000004,"15min":1.2929999999999999},"min":{"1min":0.36099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.71,"5min":8.859,"15min":15.321999999999999},"last":1.3380000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.90900000000000003,"15min":1.476},"min":{"1min":0.312,"5min":0.312,"15min":0.308},"max":{"1min":2.8620000000000001,"5min":5.4480000000000004,"15min":62.658000000000001},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.749,"15min":1.496},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":1.8799999999999999,"5min":3.8450000000000002,"15min":62.573},"last":0.46999999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757646,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893196,"kb_used_data":3780,"kb_used_omap":0,"kb_used_meta":889408,"kb_avail":92851956,"statfs":{"total":95995035648,"available":95080402944,"internally_reserved":0,"allocated":3870720,"data_stored":3056227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910753792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:27:53 2023","interfaces":[{"interface":"back","average":{"1min":7.4089999999999998,"5min":5.4269999999999996,"15min":2.9649999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.215},"max":{"1min":131.87200000000001,"5min":131.87200000000001,"15min":131.87200000000001},"last":2.0649999999999999},{"interface":"front","average":{"1min":7.4039999999999999,"5min":5.4630000000000001,"15min":3.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":131.80199999999999,"5min":131.80199999999999,"15min":131.80199999999999},"last":0.41399999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:27:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4860000000000007,"5min":3.145,"15min":2.306},"min":{"1min":0.28699999999999998,"5min":0.23999999999999999,"15min":0.23400000000000001},"max":{"1min":126.71899999999999,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.255},{"interface":"front","average":{"1min":8.3160000000000007,"5min":3.0310000000000001,"15min":2.4350000000000001},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":126.931,"5min":126.931,"15min":126.931},"last":1.7829999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:27:36 2023","interfaces":[{"interface":"back","average":{"1min":3.7639999999999998,"5min":2.613,"15min":2.274},"min":{"1min":0.30399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":51.393000000000001,"5min":81.260000000000005,"15min":81.260000000000005},"last":0.60599999999999998},{"interface":"front","average":{"1min":3.617,"5min":2.9809999999999999,"15min":2.23},"min":{"1min":0.307,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":51.432000000000002,"5min":81.245000000000005,"15min":81.245000000000005},"last":0.65300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:28:00 2023","interfaces":[{"interface":"back","average":{"1min":12.712999999999999,"5min":4.4050000000000002,"15min":3.4910000000000001},"min":{"1min":0.36899999999999999,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":118.274,"5min":118.274,"15min":190.411},"last":1.694},{"interface":"front","average":{"1min":13.329000000000001,"5min":4.3330000000000002,"15min":3.4420000000000002},"min":{"1min":0.371,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":119.22,"5min":119.22,"15min":190.70699999999999},"last":0.90900000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:27:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":1.117,"15min":1.403},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.306,"5min":32.850999999999999,"15min":32.850999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":1.089,"15min":1.1910000000000001},"min":{"1min":0.379,"5min":0.33300000000000002,"15min":0.27600000000000002},"max":{"1min":0.79800000000000004,"5min":32.82,"15min":32.82},"last":0.374}]},{"osd":6,"last update":"Mon Oct 9 06:27:53 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.4279999999999999,"15min":1.496},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68799999999999994,"5min":38.902000000000001,"15min":38.902000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.272,"15min":1.5349999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":0.98299999999999998,"5min":35.774000000000001,"15min":35.774000000000001},"last":1.881}]},{"osd":7,"last update":"Mon Oct 9 06:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.1080000000000001,"15min":1.3740000000000001},"min":{"1min":0.36099999999999999,"5min":0.32400000000000001,"15min":0.315},"max":{"1min":1.7729999999999999,"5min":30.658999999999999,"15min":30.658999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.3440000000000001,"15min":1.431},"min":{"1min":0.41199999999999998,"5min":0.36199999999999999,"15min":0.32300000000000001},"max":{"1min":1.478,"5min":30.683,"15min":30.683},"last":3.1360000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2352349184,"data_stored":2352349184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3009740800,"data_stored":3009740800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1988431872,"data_stored":1988427800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2714697728,"data_stored":2714697728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1313280000,"data_stored":1313275928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":131072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2640842752,"data_stored":2640838680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2054619136,"data_stored":2054619136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:28:26.227 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_6 pgp_num 16 2023-10-09T06:28:27.131 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:26 smithi107 ceph-mon[220921]: pgmap v1007: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16974/240861 objects degraded (7.047%); 5125/240861 objects misplaced (2.128%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:28:27.132 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:26 smithi107 ceph-mon[212750]: pgmap v1007: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16974/240861 objects degraded (7.047%); 5125/240861 objects misplaced (2.128%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:28:27.313 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:27.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:27.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:27.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:26 smithi150 ceph-mon[194178]: pgmap v1007: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 16974/240861 objects degraded (7.047%); 5125/240861 objects misplaced (2.128%); 1.4 MiB/s, 23 objects/s recovering 2023-10-09T06:28:27.423 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:27.494 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:28:27.494 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:28:27.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.494 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:103956 11.a 11:58a49acc:::benchmark_data_smithi107_418363_object13520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-10-09T06:28:27.494 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:27.479037+0000", 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.01244637, 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012478797999999999, 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 103956 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479037+0000", 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479037+0000", 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.496 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479037+0000", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479040+0000", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9569999999999998e-06 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479047+0000", 2023-10-09T06:28:27.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0010000000000004e-06 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.479052+0000", 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5559999999999998e-06 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.484028+0000", 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0049758019999999997 2023-10-09T06:28:27.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.484207+0000", 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000178495 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.484375+0000", 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000168436 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:27.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491173+0000", 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0067977869999999996 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:103960 11.b 11:d6b8778f:::benchmark_data_smithi107_418363_object13524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:27.481802+0000", 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0096815459999999992, 2023-10-09T06:28:27.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0098021250000000001, 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 103960 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481802+0000", 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481802+0000", 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:27.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481801+0000", 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481808+0000", 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.691e-06 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481825+0000", 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7659000000000002e-05 2023-10-09T06:28:27.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481834+0000", 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7280000000000001e-06 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.481876+0000", 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1508999999999998e-05 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.482016+0000", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014088800000000001 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.482130+0000", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011343100000000001 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.486202+0000", 2023-10-09T06:28:27.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0040723859999999999 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.45989.0:103965 11.0 e327/314)", 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:27.491451+0000", 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 3.3209e-05, 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00023115500000000001, 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:27.506 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "queued for pg", 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491451+0000", 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491451+0000", 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.507 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491450+0000", 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491457+0000", 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9700000000000002e-06 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.508 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491470+0000", 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3081000000000001e-05 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:27.491478+0000", 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0870000000000007e-06 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.509 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:27.510 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:28:27.510 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 3 2023-10-09T06:28:27.510 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:28:27.562 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:28.084 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:28.086 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/240861 objects degraded (7.047%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:28.086 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/240861 objects degraded (7.047%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:28.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:28.189 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:28.219 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:28.362 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:28:28.363 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:28:28.363 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:28:28.363 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:28:28.365 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:28:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/240861 objects degraded (7.047%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:28.405 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:28:28.452 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:28:28.468 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:28.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:28.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:28.518 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:28:28.518 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.519 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:28:28.575 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:28.647 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:29.218 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:28:29.253 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:28 smithi107 ceph-mon[212750]: pgmap v1008: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16974/237207 objects degraded (7.156%); 4953/237207 objects misplaced (2.088%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:29.254 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/93445647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:29.254 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:28 smithi107 ceph-mon[220921]: pgmap v1008: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16974/237207 objects degraded (7.156%); 4953/237207 objects misplaced (2.088%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:29.254 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/93445647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:29.302 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:29.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:28 smithi150 ceph-mon[194178]: pgmap v1008: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 16974/237207 objects degraded (7.156%); 4953/237207 objects misplaced (2.088%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/93445647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:29.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:29.464 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:29.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:29.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:29.605 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:28:29.644 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:29.702 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:29.792 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:29.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:29.905 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:29.938 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:29.939 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:30.004 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:30.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:30.112 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:30.188 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:30.214 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:30.410 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:30.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:30.524 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:30.606 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:31.243 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:30 smithi107 ceph-mon[220921]: pgmap v1009: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16974/230790 objects degraded (7.355%); 4953/230790 objects misplaced (2.146%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:31.246 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:30 smithi107 ceph-mon[212750]: pgmap v1009: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16974/230790 objects degraded (7.355%); 4953/230790 objects misplaced (2.146%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:30 smithi150 ceph-mon[194178]: pgmap v1009: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 16974/230790 objects degraded (7.355%); 4953/230790 objects misplaced (2.146%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:31.522 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:31.545 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:31.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:31.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:31.658 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:31.689 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:31.765 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:31.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:31.872 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:31.901 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:31.905 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:28:31.964 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:31.964 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:31.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:32.069 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:32.102 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:32.366 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:32] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2023-10-09T06:28:32.366 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:32.369 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:32.391 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:32.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:32.513 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:32.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:32.599 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:32.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:32.705 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:32.759 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:32.849 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:32.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:32.958 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:33.017 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:33.102 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:33.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:32 smithi150 ceph-mon[194178]: pgmap v1010: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 16974/228396 objects degraded (7.432%); 4787/228396 objects misplaced (2.096%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:33.109 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/228396 objects degraded (7.432%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:33.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:33.210 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:33.265 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:33.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:32 smithi107 ceph-mon[212750]: pgmap v1010: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 16974/228396 objects degraded (7.432%); 4787/228396 objects misplaced (2.096%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:33.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/228396 objects degraded (7.432%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:33.283 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:32 smithi107 ceph-mon[220921]: pgmap v1010: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 16974/228396 objects degraded (7.432%); 4787/228396 objects misplaced (2.096%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:33.283 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/228396 objects degraded (7.432%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:33.360 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:33.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:33.465 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:33.503 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:33.584 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:33.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:33.724 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:33.825 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:34.197 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:34.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:34.306 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:34.405 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:34.501 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:34.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:34.608 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:34.656 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:34.758 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:34.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:34.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:34.924 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:35.009 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 11 pgp_num to 16 2023-10-09T06:28:35.065 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:35.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:35.173 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:35.210 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:35.272 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:34 smithi107 ceph-mon[212750]: pgmap v1011: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 16974/225261 objects degraded (7.535%); 4787/225261 objects misplaced (2.125%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:35.272 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1005325980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.273 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:35 smithi107 ceph-mon[220921]: pgmap v1011: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 16974/225261 objects degraded (7.535%); 4787/225261 objects misplaced (2.125%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:35.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1005325980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.298 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:35.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:35 smithi150 ceph-mon[194178]: pgmap v1011: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 16974/225261 objects degraded (7.535%); 4787/225261 objects misplaced (2.125%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:35.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1005325980' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:28:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:35.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:35.406 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:35.489 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:35.899 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:36.230 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:36.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:36.342 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:36.383 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:28:36.383 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T06:28:36.383 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.383 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.384 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.384 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.384 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:36 smithi150 ceph-mon[194178]: pgmap v1013: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 16974/221121 objects degraded (7.676%); 4787/221121 objects misplaced (2.165%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:36.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:36.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:28:36.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T06:28:36.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[220921]: pgmap v1013: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 16974/221121 objects degraded (7.676%); 4787/221121 objects misplaced (2.165%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:36.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:28:36.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: osdmap e328: 8 total, 8 up, 8 in 2023-10-09T06:28:36.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: failed to encode map e328 with expected crc 2023-10-09T06:28:36.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:36 smithi107 ceph-mon[212750]: pgmap v1013: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 16974/221121 objects degraded (7.676%); 4787/221121 objects misplaced (2.165%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:28:36.472 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:36.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:36.578 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:36.615 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:36.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:36.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:36.770 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:36.771 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:36.795 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:36.851 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:36.934 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:36.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:37.048 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:37.067 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:28:37.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4156697735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:37.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:37.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4156697735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:37.101 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:37.151 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4156697735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: pgmap v1015: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16974/217539 objects degraded (7.803%); 4637/217539 objects misplaced (2.132%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:28:38.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/221121 objects degraded (7.676%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:38.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:38.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:38.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T06:28:38.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: pgmap v1015: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16974/217539 objects degraded (7.803%); 4637/217539 objects misplaced (2.132%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:28:38.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:38 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/221121 objects degraded (7.676%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: osdmap e329: 8 total, 8 up, 8 in 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: failed to encode map e329 with expected crc 2023-10-09T06:28:38.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: pgmap v1015: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16974/217539 objects degraded (7.803%); 4637/217539 objects misplaced (2.132%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:28:38.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:38 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/221121 objects degraded (7.676%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:38.755 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:38.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:38.863 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:38.924 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:39.424 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:28:39.583 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:39.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:39.697 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:39.805 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:39.928 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:39.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:40.037 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:40.078 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:40.162 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:40.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:40.283 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:40.290 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:40 smithi107 ceph-mon[212750]: pgmap v1016: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 16974/211500 objects degraded (8.026%); 4637/211500 objects misplaced (2.192%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:28:40.316 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:40.353 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:40.369 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:28:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:40 smithi150 ceph-mon[194178]: pgmap v1016: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 16974/211500 objects degraded (8.026%); 4637/211500 objects misplaced (2.192%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:28:40.438 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:28:40.477 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:28:40.511 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:40.515 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:28:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:40.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:40.554 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:28:40.594 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:28:40.619 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:40.642 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:28:40.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:40 smithi107 ceph-mon[220921]: pgmap v1016: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 16974/211500 objects degraded (8.026%); 4637/211500 objects misplaced (2.192%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:28:40.663 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:40.683 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.695 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:28:40.744 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:40.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:40.851 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:40.923 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:41.078 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:41.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:41.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:41.189 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:41.220 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:41.220 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:41.221 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:28:41.221 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.106561 2023-10-09T06:28:41.221 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.10656093448644222 2023-10-09T06:28:41.243 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:41.343 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:41.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:41.453 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:41.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1467003836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:41.492 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:41.559 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:41.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1467003836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1467003836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:41.666 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:41.706 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:41.726 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:28:41.776 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:41.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:41.882 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:41.915 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:41.980 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:41.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:42.088 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:42.121 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:42.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:42] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2023-10-09T06:28:42.189 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:42.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:42.298 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:42.332 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:42.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:42 smithi150 ceph-mon[194178]: pgmap v1017: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 16974/208968 objects degraded (8.123%); 4494/208968 objects misplaced (2.151%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:28:42.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/208968 objects degraded (8.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:42.446 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:42.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:42.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:42.551 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:42.577 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:42.630 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:42.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:42 smithi107 ceph-mon[212750]: pgmap v1017: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 16974/208968 objects degraded (8.123%); 4494/208968 objects misplaced (2.151%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:28:42.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/208968 objects degraded (8.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:42.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:42 smithi107 ceph-mon[220921]: pgmap v1017: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 16974/208968 objects degraded (8.123%); 4494/208968 objects misplaced (2.151%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:28:42.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/208968 objects degraded (8.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:42.739 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:42.840 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:42.989 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:43.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:43.100 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:43.131 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:43.192 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:43.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:43.263 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:43.299 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:43.362 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:43.413 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:43.903 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:43.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:64729 9.3 9:cba02ed8:::benchmark_data_smithi107_300016_object64728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:28:43.952 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 595.85761259799995, 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37691757999999997, 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 64729 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081652+0000", 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081651+0000", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081673+0000", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2178999999999999e-05 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081681+0000", 2023-10-09T06:28:43.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9219999999999997e-06 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.081688+0000", 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2469999999999997e-06 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085025+0000", 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033377429999999998 2023-10-09T06:28:43.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085147+0000", 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000122089 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.085280+0000", 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013249099999999999 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444158+0000", 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35887852100000001 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.444204+0000", 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5355999999999997e-05 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:43.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456528+0000", 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01232391 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.456591+0000", 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3146999999999997e-05 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:18:48.458569+0000", 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019784849999999999 2023-10-09T06:28:43.959 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189419 9.8 9:1506b9c6:::benchmark_data_smithi107_300016_object91847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 413.40164970500001, 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99043692000000005, 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:28:43.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189419 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537615+0000", 2023-10-09T06:28:43.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537613+0000", 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537620+0000", 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2089999999999999e-06 2023-10-09T06:28:43.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537637+0000", 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7238e-05 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537645+0000", 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8110000000000003e-06 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537696+0000", 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0652000000000002e-05 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.537708+0000", 2023-10-09T06:28:43.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2262999999999999e-05 2023-10-09T06:28:43.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508194+0000", 2023-10-09T06:28:43.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97048550899999997 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508310+0000", 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116286 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:28:43.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.508418+0000", 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010815999999999999 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.518815+0000", 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010397269000000001 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524555+0000", 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057400180000000004 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.527965+0000", 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034098969999999998 2023-10-09T06:28:43.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528013+0000", 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8173000000000001e-05 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:43.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528052+0000", 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8163e-05 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189440 9.8 9:186cfebb:::benchmark_data_smithi107_300016_object91868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:28:43.971 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 413.35690611500002, 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94578946799999997, 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189440 2023-10-09T06:28:43.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582358+0000", 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582357+0000", 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582364+0000", 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5750000000000003e-06 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582377+0000", 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3012e-05 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582385+0000", 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0059999999999992e-06 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582445+0000", 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0266000000000002e-05 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.582457+0000", 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1567e-05 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511852+0000", 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.92939540200000004 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.511956+0000", 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103732 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.512066+0000", 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109555 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.520344+0000", 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0082784669999999994 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524583+0000", 2023-10-09T06:28:43.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004239306 2023-10-09T06:28:43.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528084+0000", 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035007110000000001 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528117+0000", 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3117999999999997e-05 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528148+0000", 2023-10-09T06:28:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0361e-05 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189461 9.8 9:171409ee:::benchmark_data_smithi107_300016_object91889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 413.31278963099999, 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.90177728999999995, 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.982 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189461 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626475+0000", 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626473+0000", 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:43.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626479+0000", 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8989999999999998e-06 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626490+0000", 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0424999999999999e-05 2023-10-09T06:28:43.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626495+0000", 2023-10-09T06:28:43.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2089999999999996e-06 2023-10-09T06:28:43.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626547+0000", 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1792000000000002e-05 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:50.626557+0000", 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0407e-05 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515348+0000", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.88879119399999995 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515455+0000", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106475 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.515561+0000", 2023-10-09T06:28:43.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010639199999999999 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.524611+0000", 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090500259999999992 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.526227+0000", 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016156779999999999 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528177+0000", 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001950212 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528210+0000", 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2511000000000001e-05 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:43.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:51.528252+0000", 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2354000000000001e-05 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67308 11.a 11:53c8ff68:::benchmark_data_smithi107_418363_object67307:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628526+0000", 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310737814000007, 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1014280839999999, 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.991 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67308 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628526+0000", 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628526+0000", 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628525+0000", 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:43.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628549+0000", 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4008000000000001e-05 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628560+0000", 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1317e-05 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628571+0000", 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1070000000000001e-05 2023-10-09T06:28:43.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629748+0000", 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0011773 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629759+0000", 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0421000000000001e-05 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.650049+0000", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0202905600000001 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.650772+0000", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00072213799999999999 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729894+0000", 2023-10-09T06:28:43.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.079122909000000005 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729955+0000", 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0121000000000003e-05 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67312 11.a 11:52f09ba4:::benchmark_data_smithi107_418363_object67311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:43.997 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628579+0000", 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310685726000003, 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0995466199999999, 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67312 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:43.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628579+0000", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628579+0000", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628578+0000", 2023-10-09T06:28:43.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628599+0000", 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0808000000000001e-05 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628607+0000", 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7810000000000005e-06 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628613+0000", 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1330000000000004e-06 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629782+0000", 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001168835 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629788+0000", 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6150000000000002e-06 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.651011+0000", 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.021222946 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.651131+0000", 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119848 2023-10-09T06:28:44.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.651314+0000", 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000183248 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.662595+0000", 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011281004000000001 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.727917+0000", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.065321987999999997 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.727951+0000", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3933000000000002e-05 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728012+0000", 2023-10-09T06:28:44.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0770000000000003e-05 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728125+0000", 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000113194 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67318 11.a 11:56d0cd63:::benchmark_data_smithi107_418363_object67317:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.005 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628625+0000", 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310639037000001, 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0997339900000001, 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67318 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628625+0000", 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628625+0000", 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628625+0000", 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628644+0000", 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9726999999999999e-05 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628651+0000", 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9619999999999996e-06 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628657+0000", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6400000000000002e-06 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629808+0000", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0011510909999999999 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629814+0000", 2023-10-09T06:28:44.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1890000000000004e-06 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.654409+0000", 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.024594738 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.654583+0000", 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173537 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.654763+0000", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018053399999999999 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.671363+0000", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.016599892000000002 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728173+0000", 2023-10-09T06:28:44.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.056809538 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728213+0000", 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0065999999999999e-05 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728266+0000", 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3689e-05 2023-10-09T06:28:44.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728359+0000", 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2886000000000005e-05 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67327 11.a 11:51bcbd43:::benchmark_data_smithi107_418363_object67326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628662+0000", 2023-10-09T06:28:44.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310602368000005, 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0999477689999999, 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67327 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628662+0000", 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628662+0000", 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628661+0000", 2023-10-09T06:28:44.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628683+0000", 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1245000000000001e-05 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628690+0000", 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8689999999999998e-06 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628695+0000", 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5969999999999999e-06 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629835+0000", 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001139363 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629841+0000", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1029999999999998e-06 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.657276+0000", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.027435844 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.657446+0000", 2023-10-09T06:28:44.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016954200000000001 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.657584+0000", 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013771399999999999 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.678741+0000", 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021157386 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728431+0000", 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.049690362000000002 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728470+0000", 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8670000000000001e-05 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728522+0000", 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1978999999999998e-05 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728610+0000", 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7569000000000001e-05 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67328 11.a 11:50ebc1f3:::benchmark_data_smithi107_418363_object67327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628701+0000", 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310563169000005, 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1001264260000001, 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.022 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67328 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628701+0000", 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628701+0000", 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628701+0000", 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628726+0000", 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5038e-05 2023-10-09T06:28:44.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628734+0000", 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7649999999999993e-06 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628741+0000", 2023-10-09T06:28:44.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5499999999999997e-06 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629859+0000", 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001118378 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629865+0000", 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.896e-06 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.660018+0000", 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0301527610000001 2023-10-09T06:28:44.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.660162+0000", 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014423999999999999 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.660329+0000", 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016704 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.678751+0000", 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.018421571000000001 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728654+0000", 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.049903272999999998 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728686+0000", 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1726000000000001e-05 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728735+0000", 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8782000000000002e-05 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728828+0000", 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2807999999999996e-05 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67353 11.a 11:51a60a06:::benchmark_data_smithi107_418363_object67352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628746+0000", 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310518232000007, 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1003467360000001, 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.032 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67353 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628746+0000", 2023-10-09T06:28:44.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628746+0000", 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628746+0000", 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628766+0000", 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0129999999999999e-05 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628773+0000", 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0040000000000002e-06 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628778+0000", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4790000000000004e-06 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629884+0000", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0011056060000000001 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629890+0000", 2023-10-09T06:28:44.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0510000000000002e-06 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.662640+0000", 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0327499920000001 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.662778+0000", 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000138551 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.662932+0000", 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000153318 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.678760+0000", 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015828025999999999 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728860+0000", 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.050100443000000001 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728886+0000", 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5267999999999999e-05 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.728936+0000", 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0176999999999997e-05 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729093+0000", 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000157093 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67375 11.a 11:587de5a4:::benchmark_data_smithi107_418363_object67374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628783+0000", 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310481112999994, 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.100593132, 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67375 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.041 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628783+0000", 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628783+0000", 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628783+0000", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628802+0000", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9202999999999999e-05 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628809+0000", 2023-10-09T06:28:44.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.238e-06 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628815+0000", 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0299999999999999e-06 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629909+0000", 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001093505 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629914+0000", 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.66e-06 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.665471+0000", 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0355570570000001 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.665625+0000", 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015346500000000001 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.666105+0000", 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00048036500000000001 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.727262+0000", 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.061156290000000002 2023-10-09T06:28:44.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729152+0000", 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018905009999999999 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729195+0000", 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2577000000000002e-05 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729265+0000", 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0685000000000005e-05 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729376+0000", 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110937 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67383 11.a 11:57be5347:::benchmark_data_smithi107_418363_object67382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.628820+0000", 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.310444168999993, 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.100839253, 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.050 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67383 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628820+0000", 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628820+0000", 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628820+0000", 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628839+0000", 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9127000000000002e-05 2023-10-09T06:28:44.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628845+0000", 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5889999999999999e-06 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.628852+0000", 2023-10-09T06:28:44.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4969999999999998e-06 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630547+0000", 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001695142 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630555+0000", 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5399999999999998e-06 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.668619+0000", 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0380649120000001 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.668778+0000", 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015862499999999999 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.668960+0000", 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018189700000000001 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.727272+0000", 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.058311578000000003 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729478+0000", 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022068589999999998 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729520+0000", 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1569000000000001e-05 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729566+0000", 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6471000000000003e-05 2023-10-09T06:28:44.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729659+0000", 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2947000000000003e-05 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67400 11.a 11:5fd23055:::benchmark_data_smithi107_418363_object67399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.629934+0000", 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309330204000005, 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0999218930000001, 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.061 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67400 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629934+0000", 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629934+0000", 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629933+0000", 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629961+0000", 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7504999999999998e-05 2023-10-09T06:28:44.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629970+0000", 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6180000000000001e-06 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629977+0000", 2023-10-09T06:28:44.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1690000000000003e-06 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630573+0000", 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00059590200000000004 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630579+0000", 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0530000000000003e-06 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.671391+0000", 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0408120999999999 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.671534+0000", 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000143005 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.676154+0000", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0046198690000000004 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.727375+0000", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.051220704999999998 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729698+0000", 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002323061 2023-10-09T06:28:44.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729734+0000", 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.625e-05 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729773+0000", 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8333e-05 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729856+0000", 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.331e-05 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67408 11.a 11:5c5caa88:::benchmark_data_smithi107_418363_object67407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.629983+0000", 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309281728000002, 2023-10-09T06:28:44.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1215206390000001, 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67408 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629983+0000", 2023-10-09T06:28:44.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629983+0000", 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.629982+0000", 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630009+0000", 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6733999999999998e-05 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630018+0000", 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5629999999999998e-06 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630024+0000", 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7019999999999998e-06 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630599+0000", 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00057496499999999998 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630605+0000", 2023-10-09T06:28:44.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7570000000000001e-06 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.729977+0000", 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0993723929999999 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.730081+0000", 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103398 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.730209+0000", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000128119 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.737397+0000", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0071878860000000001 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.746905+0000", 2023-10-09T06:28:44.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0095086660000000007 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751326+0000", 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004421136 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751388+0000", 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1291000000000007e-05 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751503+0000", 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115516 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67417 11.a 11:5e70568d:::benchmark_data_smithi107_418363_object67416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630028+0000", 2023-10-09T06:28:44.078 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309235842000007, 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1216377900000001, 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67417 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630028+0000", 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630028+0000", 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630028+0000", 2023-10-09T06:28:44.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630049+0000", 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1229999999999998e-05 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630056+0000", 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6159999999999998e-06 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630061+0000", 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1240000000000004e-06 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630696+0000", 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00063500700000000004 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630704+0000", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9500000000000001e-06 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.732477+0000", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1017725359999999 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.732564+0000", 2023-10-09T06:28:44.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7767000000000002e-05 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.732657+0000", 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2757999999999995e-05 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748313+0000", 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015655954999999999 2023-10-09T06:28:44.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748351+0000", 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7713000000000003e-05 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751540+0000", 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031896030000000001 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751584+0000", 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3682e-05 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751666+0000", 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2225000000000006e-05 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.087 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67437 11.a 11:523cf69c:::benchmark_data_smithi107_418363_object67436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630065+0000", 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309199179999993, 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1217758280000001, 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.088 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67437 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630065+0000", 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630065+0000", 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630065+0000", 2023-10-09T06:28:44.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630086+0000", 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0964000000000001e-05 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630092+0000", 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5250000000000002e-06 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630097+0000", 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2000000000000002e-06 2023-10-09T06:28:44.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630726+0000", 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062886500000000004 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630732+0000", 2023-10-09T06:28:44.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9089999999999997e-06 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.734825+0000", 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1040931279999999 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.734939+0000", 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114048 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.735068+0000", 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012842800000000001 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748356+0000", 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013288614000000001 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751699+0000", 2023-10-09T06:28:44.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033422690000000001 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751725+0000", 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6268999999999999e-05 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751761+0000", 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6146999999999998e-05 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751841+0000", 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9852999999999996e-05 2023-10-09T06:28:44.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67497 11.a 11:5b8a2d0c:::benchmark_data_smithi107_418363_object67496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630101+0000", 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309162908000005, 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1219149319999999, 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.099 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67497 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630101+0000", 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630101+0000", 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630101+0000", 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630123+0000", 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1895e-05 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630130+0000", 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.985e-06 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630135+0000", 2023-10-09T06:28:44.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.609e-06 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630758+0000", 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062260199999999999 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630765+0000", 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0099999999999998e-06 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.737427+0000", 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1066622829999999 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.737537+0000", 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110062 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.737667+0000", 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000129243 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748362+0000", 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010695418999999999 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751873+0000", 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003511157 2023-10-09T06:28:44.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751900+0000", 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6427e-05 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.751936+0000", 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5911999999999998e-05 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752016+0000", 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0711999999999996e-05 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67524 11.a 11:5b7b9a0d:::benchmark_data_smithi107_418363_object67523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630140+0000", 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309124718999996, 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.122061451, 2023-10-09T06:28:44.107 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67524 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630140+0000", 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630140+0000", 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630139+0000", 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630160+0000", 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0928e-05 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630167+0000", 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7100000000000001e-06 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630172+0000", 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9660000000000004e-06 2023-10-09T06:28:44.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632330+0000", 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021586090000000001 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632359+0000", 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8427000000000001e-05 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.739930+0000", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.107571171 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.740072+0000", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014166800000000001 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.740227+0000", 2023-10-09T06:28:44.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015528199999999999 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748369+0000", 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.008141924 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752047+0000", 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036786089999999998 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752073+0000", 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6004999999999999e-05 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752118+0000", 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4450000000000003e-05 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752201+0000", 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3115000000000006e-05 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67531 11.a 11:54a8a947:::benchmark_data_smithi107_418363_object67530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630176+0000", 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309088709999997, 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1229185310000001, 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.115 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67531 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630176+0000", 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630176+0000", 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630175+0000", 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630200+0000", 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4926000000000002e-05 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630207+0000", 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4339999999999997e-06 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630212+0000", 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9869999999999999e-06 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632486+0000", 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022744620000000001 2023-10-09T06:28:44.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632509+0000", 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3439e-05 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.742510+0000", 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1100002419999999 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.742648+0000", 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000137849 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.742804+0000", 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000156398 2023-10-09T06:28:44.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748375+0000", 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055706729999999999 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752244+0000", 2023-10-09T06:28:44.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003869281 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.752958+0000", 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00071389900000000004 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.753004+0000", 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6010000000000002e-05 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.753094+0000", 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0302000000000004e-05 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.124 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67540 11.a 11:52a39f8b:::benchmark_data_smithi107_418363_object67539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e314)", 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:36.630216+0000", 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 67.309048461000003, 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.125461837, 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:28:44.125 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67540 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630216+0000", 2023-10-09T06:28:44.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630216+0000", 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:28:44.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630215+0000", 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630243+0000", 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7118999999999999e-05 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630249+0000", 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6540000000000002e-06 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.630254+0000", 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0610000000000003e-06 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632540+0000", 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022861520000000001 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:36.632546+0000", 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9889999999999997e-06 2023-10-09T06:28:44.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.744783+0000", 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1122368490000001 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.744918+0000", 2023-10-09T06:28:44.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000134308 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,6", 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.745053+0000", 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000135298 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.748380+0000", 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033275000000000002 2023-10-09T06:28:44.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.133 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.754104+0000", 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057239789999999997 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:28:44.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.755520+0000", 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014157760000000001 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.755568+0000", 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7752e-05 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:37.755678+0000", 2023-10-09T06:28:44.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109762 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:28:44.136 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:28:44.184 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:44.276 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:44.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:44.383 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:44 smithi107 ceph-mon[212750]: pgmap v1018: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 16974/206031 objects degraded (8.239%); 4494/206031 objects misplaced (2.181%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:28:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:44 smithi107 ceph-mon[220921]: pgmap v1018: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 16974/206031 objects degraded (8.239%); 4494/206031 objects misplaced (2.181%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:28:44.477 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:44.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:44 smithi150 ceph-mon[194178]: pgmap v1018: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 16974/206031 objects degraded (8.239%); 4494/206031 objects misplaced (2.181%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:28:45.095 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:28:46.295 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:46.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:46.404 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:46.509 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:46.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:46 smithi107 ceph-mon[212750]: pgmap v1019: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 16974/202068 objects degraded (8.400%); 4494/202068 objects misplaced (2.224%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:46 smithi107 ceph-mon[220921]: pgmap v1019: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 16974/202068 objects degraded (8.400%); 4494/202068 objects misplaced (2.224%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:46 smithi150 ceph-mon[194178]: pgmap v1019: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 16974/202068 objects degraded (8.400%); 4494/202068 objects misplaced (2.224%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:28:47.179 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:47.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:47.198 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:28:47.287 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.358 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:47.502 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:47.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:47.607 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:47.678 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:47.773 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:47.830 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:47.831 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:47.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/202068 objects degraded (8.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/202068 objects degraded (8.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:47.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:28:47.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:28:47.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/202068 objects degraded (8.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:47.945 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:47.986 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:48.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[212750]: pgmap v1020: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 16974/198483 objects degraded (8.552%); 4348/198483 objects misplaced (2.191%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:48.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]: dispatch 2023-10-09T06:28:48.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1593788902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.642 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:28:48.651 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.2 primary-affinity to 0.106561 (869832) 2023-10-09T06:28:48.692 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:48.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:48.796 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:48.838 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:48.883 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:48.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:48 smithi150 ceph-mon[194178]: pgmap v1020: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 16974/198483 objects degraded (8.552%); 4348/198483 objects misplaced (2.191%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:48.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]: dispatch 2023-10-09T06:28:48.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1593788902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:48.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:48.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[220921]: pgmap v1020: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 16974/198483 objects degraded (8.552%); 4348/198483 objects misplaced (2.191%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:48.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]: dispatch 2023-10-09T06:28:48.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1593788902' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:28:48.990 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:49.019 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:49.217 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:49.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:49.320 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:49.346 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:49.389 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:49.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:49.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:49.492 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:49.525 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:49.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]': finished 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[212750]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]': finished 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:49 smithi107 ceph-mon[220921]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1706169847' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.10656093448644222}]': finished 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: osdmap e330: 8 total, 8 up, 8 in 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: failed to encode map e330 with expected crc 2023-10-09T06:28:49.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:49 smithi150 ceph-mon[194178]: failed to encode map e330 with expected crc 2023-10-09T06:28:50.436 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:28:50.436 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:28:50.436 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:28:50.437 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:28:50.473 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:50.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:50.582 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:50.614 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:50.798 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: pgmap v1022: 99 pgs: 2 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 16974/192828 objects degraded (8.803%); 4348/192828 objects misplaced (2.255%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.800 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:50 smithi150 ceph-mon[194178]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:50.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:50.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:50.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: pgmap v1022: 99 pgs: 2 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 16974/192828 objects degraded (8.803%); 4348/192828 objects misplaced (2.255%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:50.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:50.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T06:28:50.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[212750]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: pgmap v1022: 99 pgs: 2 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 16974/192828 objects degraded (8.803%); 4348/192828 objects misplaced (2.255%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: osdmap e331: 8 total, 8 up, 8 in 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:50 smithi107 ceph-mon[220921]: failed to encode map e331 with expected crc 2023-10-09T06:28:50.958 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:51.258 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:51.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:51.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:51.459 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:51.477 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:28:51.525 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:51.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:51.628 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:51.689 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:51.996 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:52.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:52.048 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:52.104 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:52.176 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:52.393 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:52.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:28:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:28:52] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:28:52.458 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:28:52.500 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:52.504 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:28:52.538 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:28:52.565 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:52.581 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:28:52.615 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:28:52.650 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:28:52.664 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:52.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:52.696 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:28:52.727 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.734 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.735 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:28:52.774 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:52.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:52 smithi150 ceph-mon[194178]: pgmap v1024: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 16974/191214 objects degraded (8.877%); 4088/191214 objects misplaced (2.138%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:52.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/191214 objects degraded (8.877%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:52.843 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2849471790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:52.856 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[220921]: pgmap v1024: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 16974/191214 objects degraded (8.877%); 4088/191214 objects misplaced (2.138%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/191214 objects degraded (8.877%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2849471790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:52.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[212750]: pgmap v1024: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 16974/191214 objects degraded (8.877%); 4088/191214 objects misplaced (2.138%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:52.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/191214 objects degraded (8.877%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:52.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2849471790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:28:52.933 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:52.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:53.038 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:53.104 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:53.198 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:53.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:53.302 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:53.366 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:53.478 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:53.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:53.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:53.590 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:53.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:53.879 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:53.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:53.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:53.986 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.019 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:28:54.066 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:54.146 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:54.146 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:54.184 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:54.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:54.290 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:54.340 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:54.428 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:54.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:54.532 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:28:54.584 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:54.627 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:28:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:54.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:28:54.642 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:54.730 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:54.779 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:54 smithi150 ceph-mon[194178]: pgmap v1025: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 16974/189414 objects degraded (8.961%); 4088/189414 objects misplaced (2.158%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:54.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:54 smithi107 ceph-mon[212750]: pgmap v1025: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 16974/189414 objects degraded (8.961%); 4088/189414 objects misplaced (2.158%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:54.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:54 smithi107 ceph-mon[220921]: pgmap v1025: 99 pgs: 1 active+recovering+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 16974/189414 objects degraded (8.961%); 4088/189414 objects misplaced (2.158%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:28:55.119 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:28:55.128 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:55.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:55.232 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:55.331 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:55.505 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:55.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:55.627 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:28:55.720 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:55.830 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:28:55.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:55.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:55 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:55.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:28:55.863 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:28:55.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:28:55.941 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:28:55.965 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:55 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:55.965 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:55 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:28:55.969 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:56.019 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:28:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:56.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:28:56.123 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:28:56.151 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:56.196 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:28:56.196 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:28:56.197 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 4, 6, 7, 3, 5] 2023-10-09T06:28:56.197 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T06:28:56.453 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:28:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:56.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:28:56.558 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:56.609 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:56.652 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:56.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:56.755 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:28:56.817 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:56.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:56 smithi107 ceph-mon[212750]: pgmap v1026: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16974/185322 objects degraded (9.159%); 4088/185322 objects misplaced (2.206%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:56.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:56 smithi107 ceph-mon[220921]: pgmap v1026: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16974/185322 objects degraded (9.159%); 4088/185322 objects misplaced (2.206%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:57.060 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:28:57.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:56 smithi150 ceph-mon[194178]: pgmap v1026: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 16974/185322 objects degraded (9.159%); 4088/185322 objects misplaced (2.206%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:28:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:57.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:28:57.107 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:57.155 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:28:57.170 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:28:57.264 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:57.415 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:28:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:57.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:28:57.523 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:28:57.603 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:57.738 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:28:57.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:57.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:28:57.782 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:28:57.783 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:28:57.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/185322 objects degraded (9.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:57.804 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1226174149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.805 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.857 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/185322 objects degraded (9.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:57.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1226174149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:57.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/185322 objects degraded (9.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:28:57.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1226174149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:28:57.952 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:58.346 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:28:58.684 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:28:58.737 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:28:58.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:28:58.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:28:58.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:28:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:58 smithi150 ceph-mon[194178]: pgmap v1027: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 16974/182835 objects degraded (9.284%); 3925/182835 objects misplaced (2.147%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:28:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:58 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:58 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:58 smithi150 ceph-mon[194178]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T06:28:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:58 smithi150 ceph-mon[194178]: failed to encode map e332 with expected crc 2023-10-09T06:28:58.919 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:28:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[212750]: pgmap v1027: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 16974/182835 objects degraded (9.284%); 3925/182835 objects misplaced (2.147%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:28:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[212750]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T06:28:59.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:28:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[220921]: pgmap v1027: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 16974/182835 objects degraded (9.284%); 3925/182835 objects misplaced (2.147%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:28:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:28:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:28:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[220921]: osdmap e332: 8 total, 8 up, 8 in 2023-10-09T06:28:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:58 smithi107 ceph-mon[220921]: failed to encode map e332 with expected crc 2023-10-09T06:28:59.420 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:00.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:59 smithi150 ceph-mon[194178]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:59 smithi150 ceph-mon[194178]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:28:59 smithi150 ceph-mon[194178]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[220921]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[220921]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[220921]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:28:59 smithi107 ceph-mon[212750]: failed to encode map e332 with expected crc 2023-10-09T06:29:00.419 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:00.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:00.530 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:00.606 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:00.851 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:00.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:00.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:00 smithi150 ceph-mon[194178]: pgmap v1029: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 16974/176196 objects degraded (9.634%); 3925/176196 objects misplaced (2.228%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:00.960 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:01.024 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:01.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:00 smithi107 ceph-mon[212750]: pgmap v1029: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 16974/176196 objects degraded (9.634%); 3925/176196 objects misplaced (2.228%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:01.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:00 smithi107 ceph-mon[220921]: pgmap v1029: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 16974/176196 objects degraded (9.634%); 3925/176196 objects misplaced (2.228%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:01.230 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:01.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:01.353 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:01.409 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:01.540 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:01.541 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:01.541 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:01.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:01.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:01.650 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:01.685 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:01.687 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:01.723 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:01.758 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:01.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:01.771 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:01.818 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:01.853 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:01.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:01.908 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:01.946 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:01.950 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:01.990 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:02.008 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.008 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.008 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.008 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:29:02.069 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:02.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:02.201 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:02.243 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:02] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:29:02.323 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:02.376 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:29:02.452 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:29:02.464 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:02.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:02.505 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 conmon[212706]: 2023-10-09T06:29:02.443+0000 7f7c794f9700 -1 mon.a@0(leader).osd e332 definitely_dead 0 2023-10-09T06:29:02.582 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:02.659 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:02.766 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:02.781 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: pgmap v1030: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 16974/174150 objects degraded (9.747%); 3724/174150 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2581186709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/174150 objects degraded (9.747%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2852204863' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[220921]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:29:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: pgmap v1030: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 16974/174150 objects degraded (9.747%); 3724/174150 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2581186709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/174150 objects degraded (9.747%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2852204863' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:02 smithi107 ceph-mon[212750]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:29:02.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:02.888 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: pgmap v1030: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 16974/174150 objects degraded (9.747%); 3724/174150 objects misplaced (2.138%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2581186709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/174150 objects degraded (9.747%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2852204863' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:29:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:02 smithi150 ceph-mon[194178]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:29:02.971 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:03.001 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:03.090 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:03.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:03.203 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:03.259 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:03.349 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.0. 2023-10-09T06:29:03.363 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:03.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:03.437 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:03.468 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:03.499 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:03.577 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:03.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:03.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:03.707 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: pgmap v1031: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 16974/171909 objects degraded (9.874%); 3724/171909 objects misplaced (2.166%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[212750]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: pgmap v1031: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 16974/171909 objects degraded (9.874%); 3724/171909 objects misplaced (2.166%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:04 smithi107 ceph-mon[220921]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: pgmap v1031: 99 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 16974/171909 objects degraded (9.874%); 3724/171909 objects misplaced (2.166%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: osdmap e333: 8 total, 8 up, 7 in 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:04 smithi150 ceph-mon[194178]: failed to encode map e333 with expected crc 2023-10-09T06:29:04.773 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:04.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:04.879 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:04.907 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:04.926 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:29:04.926 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:29:04.926 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:29:04.926 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:29:04.927 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:29:05.003 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:05.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:05.107 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:05.139 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:05.226 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:29:05.233 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:05.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:05.337 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[212750]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:05 smithi107 ceph-mon[220921]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.402 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: osdmap e334: 8 total, 8 up, 7 in 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:05 smithi150 ceph-mon[194178]: failed to encode map e334 with expected crc 2023-10-09T06:29:05.479 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:05.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:05.585 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:05.654 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:05.720 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:29:06.213 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:06.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:06.319 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:06.392 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:06.427 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:06.469 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:06.518 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:06.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:06.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:06.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: pgmap v1034: 99 pgs: 14 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 16974/167673 objects degraded (10.123%); 3724/167673 objects misplaced (2.221%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:29:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 332'8006 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 332'7704 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 332'7811 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[212750]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'7810 2023-10-09T06:29:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: pgmap v1034: 99 pgs: 14 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 16974/167673 objects degraded (10.123%); 3724/167673 objects misplaced (2.221%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 332'8006 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 332'7704 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 332'7811 2023-10-09T06:29:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:06 smithi107 ceph-mon[220921]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'7810 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: pgmap v1034: 99 pgs: 14 peering, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 16974/167673 objects degraded (10.123%); 3724/167673 objects misplaced (2.221%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: failed to encode map e335 with expected crc 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: 11.b starting backfill to osd.4 from (0'0,0'0] MAX to 332'8006 2023-10-09T06:29:06.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: osdmap e335: 8 total, 8 up, 7 in 2023-10-09T06:29:06.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 332'7704 2023-10-09T06:29:06.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: 11.2 starting backfill to osd.7 from (0'0,0'0] MAX to 332'7811 2023-10-09T06:29:06.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:06 smithi150 ceph-mon[194178]: 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 332'7810 2023-10-09T06:29:06.727 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:06.818 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:06.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:06.940 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:07.161 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:07.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:07.276 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:07.380 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T06:29:07.380 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.380 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.380 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 16974/167673 objects degraded (10.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:07.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T06:29:07.382 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.383 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 16974/167673 objects degraded (10.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:07.387 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:07.565 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:07.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: osdmap e336: 8 total, 8 up, 7 in 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: failed to encode map e336 with expected crc 2023-10-09T06:29:07.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 16974/167673 objects degraded (10.123%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:07.671 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:07.699 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:08.057 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:08.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:08.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:08.191 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:08.238 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:08.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:08.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:08.342 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:08.405 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:08.407 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:29:08.628 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:08.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:08.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:08.641 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:08 smithi150 ceph-mon[194178]: pgmap v1037: 99 pgs: 1 active+recovery_wait+remapped, 14 peering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 16974/166635 objects degraded (10.186%); 7076/166635 objects misplaced (4.246%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:29:08.641 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:08 smithi150 ceph-mon[194178]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T06:29:08.641 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:08 smithi150 ceph-mon[194178]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.641 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:08 smithi150 ceph-mon[194178]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.641 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:08.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[220921]: pgmap v1037: 99 pgs: 1 active+recovery_wait+remapped, 14 peering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 16974/166635 objects degraded (10.186%); 7076/166635 objects misplaced (4.246%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:29:08.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[220921]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[220921]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[220921]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[212750]: pgmap v1037: 99 pgs: 1 active+recovery_wait+remapped, 14 peering, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 16974/166635 objects degraded (10.186%); 7076/166635 objects misplaced (4.246%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[212750]: osdmap e337: 8 total, 8 up, 7 in 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[212750]: failed to encode map e337 with expected crc 2023-10-09T06:29:08.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:08.740 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:102760 11.8 e327/323 11:15603f00:::benchmark_data_smithi107_418363_object12324:head v 327'6393, mlcod=301'224)", 2023-10-09T06:29:08.741 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:25.670039+0000", 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 43.064539486000001, 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.055675398000000001, 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670039+0000", 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.742 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670039+0000", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670038+0000", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670053+0000", 2023-10-09T06:29:08.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5665e-05 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670072+0000", 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8851e-05 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.670082+0000", 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7599999999999997e-06 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.674827+0000", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004744804 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.674867+0000", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0534000000000002e-05 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.725670+0000", 2023-10-09T06:29:08.745 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.050802426999999997 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.725715+0000", 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5111000000000002e-05 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.746 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:102766 11.8 e327/323 11:1b591ce8:::benchmark_data_smithi107_418363_object12330:head v 327'6394, mlcod=301'224)", 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:25.678673+0000", 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 43.055905430000003, 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047072546, 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678673+0000", 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.747 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678673+0000", 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678672+0000", 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.748 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678681+0000", 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1309999999999995e-06 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678698+0000", 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6317e-05 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.678704+0000", 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4980000000000003e-06 2023-10-09T06:29:08.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.682649+0000", 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003944828 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.682682+0000", 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2639999999999999e-05 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.725720+0000", 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.043038010000000002 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:25.725746+0000", 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6197999999999999e-05 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.751 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:103265 11.7 e327/323 11:efec8f03:::benchmark_data_smithi107_418363_object12829:head v 327'6324, mlcod=301'767)", 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:26.452635+0000", 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 42.281944070999998, 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047835825999999998, 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452635+0000", 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.752 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452635+0000", 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452634+0000", 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452642+0000", 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5750000000000001e-06 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452654+0000", 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1742999999999999e-05 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.754 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.452660+0000", 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7699999999999998e-06 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454941+0000", 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022807999999999999 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454965+0000", 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4362000000000002e-05 2023-10-09T06:29:08.755 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.500452+0000", 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.045486882999999999 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.500471+0000", 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8623000000000001e-05 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.756 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:103269 11.7 e327/323 11:ed46af9d:::benchmark_data_smithi107_418363_object12833:head v 327'6325, mlcod=301'767)", 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:26.454464+0000", 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 42.280114490999999, 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046022714999999999, 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.757 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454464+0000", 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454464+0000", 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454464+0000", 2023-10-09T06:29:08.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454470+0000", 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3280000000000002e-06 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454483+0000", 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3083e-05 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.454489+0000", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7989999999999999e-06 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.461539+0000", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.007049507 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.461563+0000", 2023-10-09T06:29:08.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4419000000000001e-05 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.500474+0000", 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.038910879000000002 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:26.500487+0000", 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3230999999999999e-05 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.761 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:104125 11.8 e327/323 11:1cb64b20:::benchmark_data_smithi107_418363_object13689:head v 327'6485, mlcod=301'224)", 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:27.746824+0000", 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 40.987754781, 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046026699999999997, 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.762 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746824+0000", 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746824+0000", 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746823+0000", 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746831+0000", 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2290000000000003e-06 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746841+0000", 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7850000000000003e-06 2023-10-09T06:29:08.764 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746846+0000", 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2480000000000004e-06 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.760705+0000", 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013858962000000001 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.765 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.760723+0000", 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7268000000000001e-05 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.792794+0000", 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.032070988000000002 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.766 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.792851+0000", 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7092999999999997e-05 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:104127 11.8 e327/323 11:1a978f1c:::benchmark_data_smithi107_418363_object13691:head v 327'6486, mlcod=301'224)", 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:27.746852+0000", 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 40.987726842000001, 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046025463000000003, 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.767 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746852+0000", 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746852+0000", 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.768 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746852+0000", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967296 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746855+0000", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2909999999999999e-06 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746858+0000", 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2059999999999999e-06 2023-10-09T06:29:08.769 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.746861+0000", 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2900000000000001e-06 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.765978+0000", 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019117459999999999 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.770 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.766000+0000", 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1795999999999999e-05 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.792854+0000", 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026854303999999999 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.771 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:27.792877+0000", 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3325000000000001e-05 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:113236 11.8 e329/323 11:13e006ad:::benchmark_data_smithi107_418363_object22800:head v 329'7005, mlcod=301'224)", 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:41.983990+0000", 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.750588415999999, 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046415230000000002, 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.772 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.983990+0000", 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.983990+0000", 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.773 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.983990+0000", 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967296 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.983994+0000", 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.41e-06 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.774 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.983999+0000", 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.977e-06 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.984001+0000", 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1799999999999999e-06 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.990703+0000", 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0067024129999999999 2023-10-09T06:29:08.775 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.990731+0000", 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7971e-05 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.030365+0000", 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.039633836999999998 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.776 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.030406+0000", 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0649000000000003e-05 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:113250 11.8 e329/323 11:1fbb0755:::benchmark_data_smithi107_418363_object22814:head v 329'7008, mlcod=301'224)", 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:41.997549+0000", 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.737029546999999, 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046514496000000002, 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.777 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997549+0000", 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997549+0000", 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.778 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997548+0000", 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997560+0000", 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2745000000000001e-05 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997574+0000", 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3876999999999999e-05 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:41.997582+0000", 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9859999999999994e-06 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.013886+0000", 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016303932 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.013921+0000", 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4363999999999998e-05 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.044022+0000", 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.030101124 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.781 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.044064+0000", 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1987999999999998e-05 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:113254 11.8 e329/323 11:1e2c753c:::benchmark_data_smithi107_418363_object22818:head v 329'7010, mlcod=301'224)", 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:42.010367+0000", 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.724212074, 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047456765999999997, 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.782 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010367+0000", 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010367+0000", 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010365+0000", 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010376+0000", 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1345999999999999e-05 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010388+0000", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1759999999999999e-05 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.010396+0000", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8830000000000006e-06 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.030477+0000", 2023-10-09T06:29:08.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020080694999999999 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.030509+0000", 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2651999999999998e-05 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.057782+0000", 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.027272910000000001 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:42.057823+0000", 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1099000000000001e-05 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:117301 11.7 e331/323 11:ed77c3f2:::benchmark_data_smithi107_418363_object26865:head v 331'7206, mlcod=302'1776)", 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:49.896495+0000", 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 18.838084232, 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.050622634999999999, 2023-10-09T06:29:08.787 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896495+0000", 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896495+0000", 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896493+0000", 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896501+0000", 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6969999999999993e-06 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896515+0000", 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3404000000000001e-05 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896521+0000", 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1959999999999996e-06 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896572+0000", 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1739e-05 2023-10-09T06:29:08.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.896596+0000", 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3319000000000002e-05 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.947065+0000", 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.050468728999999997 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.947117+0000", 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2667999999999998e-05 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:117304 11.7 e331/323 11:e8b33e50:::benchmark_data_smithi107_418363_object26868:head v 331'7207, mlcod=302'1776)", 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:49.899386+0000", 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 18.835193077, 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047758555000000001, 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.792 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899386+0000", 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899386+0000", 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899384+0000", 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899399+0000", 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4778e-05 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899416+0000", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6878e-05 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.899425+0000", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6690000000000001e-06 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.904140+0000", 2023-10-09T06:29:08.795 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047141079999999998 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.904169+0000", 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9319000000000002e-05 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.947120+0000", 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.042951346000000001 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.947144+0000", 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4014000000000001e-05 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:118444 11.a e331/323 11:59532841:::benchmark_data_smithi107_418363_object28008:head v 331'7316, mlcod=301'245)", 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:51.765928+0000", 2023-10-09T06:29:08.797 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 16.968651212000001, 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.049298491999999999, 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765928+0000", 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765928+0000", 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765927+0000", 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765933+0000", 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3790000000000003e-06 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765942+0000", 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2609999999999993e-06 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.765947+0000", 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8740000000000003e-06 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.775377+0000", 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0094305670000000008 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.775411+0000", 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4010999999999997e-05 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.815180+0000", 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.039768844999999997 2023-10-09T06:29:08.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:51.815226+0000", 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6091000000000003e-05 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:125769 11.8 e335/323 11:12bce247:::benchmark_data_smithi107_418363_object35333:head v 335'7833, mlcod=302'2187)", 2023-10-09T06:29:08.802 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:05.885915+0000", 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.8486639739999999, 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.048161945999999997, 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885915+0000", 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.803 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885915+0000", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885914+0000", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885922+0000", 2023-10-09T06:29:08.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885931+0000", 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9789999999999999e-06 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.885936+0000", 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6949999999999999e-06 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.902825+0000", 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016889161 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.902853+0000", 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8510999999999999e-05 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.934031+0000", 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.03117731 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.934077+0000", 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6264000000000003e-05 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:125784 11.8 e335/323 11:113a54d4:::benchmark_data_smithi107_418363_object35348:head v 335'7835, mlcod=302'2187)", 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:05.905667+0000", 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.8289119920000001, 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.066754255999999998, 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905667+0000", 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905667+0000", 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905665+0000", 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905687+0000", 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1631e-05 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905703+0000", 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5965e-05 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.810 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.905710+0000", 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6569999999999997e-06 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.917573+0000", 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011862754 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.917600+0000", 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.65e-05 2023-10-09T06:29:08.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972373+0000", 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.054772978999999999 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972421+0000", 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8523000000000003e-05 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.812 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:125785 11.8 e335/323 11:1eca3533:::benchmark_data_smithi107_418363_object35349:head v 335'7836, mlcod=302'2187)", 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:05.908477+0000", 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.826101752, 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.063972734000000003, 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.813 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908477+0000", 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908477+0000", 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908476+0000", 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908486+0000", 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0146000000000001e-05 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908503+0000", 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6560999999999999e-05 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.815 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.908510+0000", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3209999999999999e-06 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925649+0000", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017138833999999999 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925670+0000", 2023-10-09T06:29:08.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1679999999999999e-05 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972426+0000", 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.046755459999999999 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972450+0000", 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3881000000000001e-05 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.817 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:125793 11.8 e335/323 11:1f96ac6f:::benchmark_data_smithi107_418363_object35357:head v 335'7837, mlcod=302'2187)", 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:05.925210+0000", 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.8093688000000001, 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047260280000000002, 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925210+0000", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925210+0000", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925208+0000", 2023-10-09T06:29:08.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925220+0000", 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1222e-05 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925234+0000", 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4381e-05 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.925242+0000", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3950000000000001e-06 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.934132+0000", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0088891309999999998 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.934157+0000", 2023-10-09T06:29:08.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5497999999999999e-05 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972452+0000", 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.038294704999999998 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.972470+0000", 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8561000000000001e-05 2023-10-09T06:29:08.822 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:125999 11.a e335/323 11:510f01d9:::benchmark_data_smithi107_418363_object35563:head v 335'7777, mlcod=302'2283)", 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:06.227824+0000", 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.5067548130000001, 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.056233814, 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227824+0000", 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227824+0000", 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227823+0000", 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227833+0000", 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9790000000000005e-06 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227854+0000", 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0611999999999998e-05 2023-10-09T06:29:08.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227861+0000", 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1890000000000001e-06 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.228007+0000", 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000145847 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.228030+0000", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3419000000000001e-05 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284014+0000", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.055984226999999998 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284058+0000", 2023-10-09T06:29:08.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3547999999999997e-05 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDPGPush(11.a 335/323 [PushOp(11:55ea654b:::benchmark_data_smithi107_418363_object37104:head, version: 302'2286, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(11:55ea654b:::benchmark_data_smithi107_418363_object37104:head@302'2286, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:06.227874+0000", 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.5067047540000003, 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.060071917000000002, 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:29:08.828 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227874+0000", 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227874+0000", 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227874+0000", 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227920+0000", 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5952000000000003e-05 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227938+0000", 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8309e-05 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.227947+0000", 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7509999999999997e-06 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284241+0000", 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.056293805000000002 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284251+0000", 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0529e-05 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.287946+0000", 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003694868 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:126005 11.a e335/323 11:5687c16b:::benchmark_data_smithi107_418363_object35569:head v 335'7778, mlcod=302'2283)", 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:06.231851+0000", 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.5027280620000001, 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.052239002, 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231851+0000", 2023-10-09T06:29:08.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231851+0000", 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231850+0000", 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:08.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231860+0000", 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0753e-05 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231892+0000", 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1313999999999999e-05 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.231900+0000", 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5399999999999996e-06 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.236102+0000", 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042021389999999997 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.236127+0000", 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4995e-05 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284066+0000", 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.047938833 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284090+0000", 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3669e-05 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:126006 11.a e335/323 11:534907b2:::benchmark_data_smithi107_418363_object35570:head v 335'7779, mlcod=302'2283)", 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:06.235428+0000", 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 2.4991511969999998, 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.048686970000000003, 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235428+0000", 2023-10-09T06:29:08.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235428+0000", 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235427+0000", 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235436+0000", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9819999999999997e-06 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235447+0000", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1786e-05 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.235454+0000", 2023-10-09T06:29:08.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5420000000000002e-06 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.243961+0000", 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0085076609999999997 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.243985+0000", 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3743999999999998e-05 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284094+0000", 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.040108775999999999 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:06.284115+0000", 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.056e-05 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:08.842 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:08.843 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:29:08.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:09.067 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:09.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:09.179 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:09.223 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:09.559 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:09.563 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.564 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:09 smithi150 ceph-mon[194178]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:09.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[212750]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/229104075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: osdmap e338: 8 total, 8 up, 7 in 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:09 smithi107 ceph-mon[220921]: failed to encode map e338 with expected crc 2023-10-09T06:29:09.653 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:29:09.665 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:09.759 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:09.823 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:29:09.824 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:29:09.824 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T06:29:09.824 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T06:29:09.921 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:09.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:09.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:10.035 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:10.089 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:10.236 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:10.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:10.295 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:10.322 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:10.347 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:10.386 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:10.424 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:10.429 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:10.473 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:10.510 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:10.513 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:10.554 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:10.563 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:10.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:10.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:10 smithi150 ceph-mon[194178]: pgmap v1040: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 19086/161940 objects degraded (11.786%); 20199/161940 objects misplaced (12.473%); 6.1 MiB/s, 100 objects/s recovering 2023-10-09T06:29:10.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:10 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T06:29:10.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/158380927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:10.635 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:10.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[212750]: pgmap v1040: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 19086/161940 objects degraded (11.786%); 20199/161940 objects misplaced (12.473%); 6.1 MiB/s, 100 objects/s recovering 2023-10-09T06:29:10.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T06:29:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/158380927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[220921]: pgmap v1040: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 19086/161940 objects degraded (11.786%); 20199/161940 objects misplaced (12.473%); 6.1 MiB/s, 100 objects/s recovering 2023-10-09T06:29:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-10-09T06:29:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/158380927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:10.671 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:10.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.706 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:29:10.753 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:10.965 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:10.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:11.074 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:11.130 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:11.209 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:11.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:11.313 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:11.340 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:11.662 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:11.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:11.772 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:11.797 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:29:11.828 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:11.892 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:11.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:11.999 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:12.025 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:12] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:29:12.038 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:12.198 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:12.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:12.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:12.309 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:12.400 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:12.523 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:12.523 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:12.554 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:12 smithi150 ceph-mon[194178]: pgmap v1041: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 19086/160149 objects degraded (11.918%); 20018/160149 objects misplaced (12.500%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T06:29:12.555 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19086/160149 objects degraded (11.918%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:12 smithi107 ceph-mon[212750]: pgmap v1041: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 19086/160149 objects degraded (11.918%); 20018/160149 objects misplaced (12.500%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T06:29:12.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19086/160149 objects degraded (11.918%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:12 smithi107 ceph-mon[220921]: pgmap v1041: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 424 op/s; 19086/160149 objects degraded (11.918%); 20018/160149 objects misplaced (12.500%); 4.1 MiB/s, 68 objects/s recovering 2023-10-09T06:29:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19086/160149 objects degraded (11.918%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T06:29:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:13.123 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:13.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:13.224 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:13.268 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:13.373 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:13.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:13.379 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:13.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:29:13.479 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:13.550 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:13.704 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:13.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:13.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:13.812 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:13.884 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:14.012 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:14.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:14.098 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:14.130 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:14.217 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:14.476 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.0. 2023-10-09T06:29:14.500 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[220921]: pgmap v1042: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 19086/157566 objects degraded (12.113%); 20018/157566 objects misplaced (12.705%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:29:14.501 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:14.501 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1825771079' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.501 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.501 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:14 smithi107 conmon[212706]: 2023-10-09T06:29:14.368+0000 7f7c794f9700 -1 mon.a@0(leader).osd e338 definitely_dead 0 2023-10-09T06:29:14.501 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[212750]: pgmap v1042: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 19086/157566 objects degraded (12.113%); 20018/157566 objects misplaced (12.705%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:29:14.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:14.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1825771079' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.604 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:14.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:14.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:14 smithi150 ceph-mon[194178]: pgmap v1042: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 19086/157566 objects degraded (12.113%); 20018/157566 objects misplaced (12.705%); 1.8 MiB/s, 29 objects/s recovering 2023-10-09T06:29:14.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:29:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1825771079' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:29:14.708 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:14.737 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:15.149 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:29:15.790 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.790 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:29:15.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:29:15.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.791 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:15 smithi150 ceph-mon[194178]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:29:15.820 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:29:15.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:29:15.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.822 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.822 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.822 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[220921]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: osdmap e339: 8 total, 8 up, 8 in 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: failed to encode map e339 with expected crc 2023-10-09T06:29:15.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:15 smithi107 ceph-mon[212750]: osdmap e340: 8 total, 8 up, 8 in 2023-10-09T06:29:16.029 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:16.029 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:16.523 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:16.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:16.633 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:16.672 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:16.684 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T06:29:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: pgmap v1044: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 19086/153114 objects degraded (12.465%); 19297/153114 objects misplaced (12.603%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T06:29:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[220921]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: pgmap v1044: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 19086/153114 objects degraded (12.465%); 19297/153114 objects misplaced (12.603%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T06:29:16.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:16 smithi107 ceph-mon[212750]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: pgmap v1044: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 19086/153114 objects degraded (12.465%); 19297/153114 objects misplaced (12.603%); 2.7 MiB/s, 44 objects/s recovering 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:16 smithi150 ceph-mon[194178]: failed to encode map e340 with expected crc 2023-10-09T06:29:17.043 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:17.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:17.151 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:17.177 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:29:17.226 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:17.227 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:17.292 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:17.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:17.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:17.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:17.507 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:17.616 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:17.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:17.727 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:17.771 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:17.771 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 19086/153114 objects degraded (12.465%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:17.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:17.771 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 19086/153114 objects degraded (12.465%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:17.776 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:17.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 19086/153114 objects degraded (12.465%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:17.996 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:18.007 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:18.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:18.113 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:18.162 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:18.313 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:18.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:18.420 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:18.481 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:18.543 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:29:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:18 smithi150 ceph-mon[194178]: pgmap v1046: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19086/151983 objects degraded (12.558%); 16159/151983 objects misplaced (10.632%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:29:18.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:18.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:18 smithi107 ceph-mon[212750]: pgmap v1046: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19086/151983 objects degraded (12.558%); 16159/151983 objects misplaced (10.632%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:29:18.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:18.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:18 smithi107 ceph-mon[220921]: pgmap v1046: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19086/151983 objects degraded (12.558%); 16159/151983 objects misplaced (10.632%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T06:29:18.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[220921]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:19 smithi107 ceph-mon[212750]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/704930828' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: osdmap e341: 8 total, 8 up, 8 in 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: failed to encode map e341 with expected crc 2023-10-09T06:29:19.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:19 smithi150 ceph-mon[194178]: failed to encode map e341 with expected crc 2023-10-09T06:29:20.226 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:20.226 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:20.368 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:20.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:20.472 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:20.503 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:20.630 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 13335/146238 objects degraded (9.119%); 6624/146238 objects misplaced (4.530%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T06:29:20.630 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:29:20.630 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.630 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 13335/146238 objects degraded (9.119%); 6624/146238 objects misplaced (4.530%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:29:20.631 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.632 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:20 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:20.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:20.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:20.814 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:20.850 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:20.898 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:20.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: pgmap v1048: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 13335/146238 objects degraded (9.119%); 6624/146238 objects misplaced (4.530%); 3.5 MiB/s, 83 objects/s recovering 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: osdmap e342: 8 total, 8 up, 8 in 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:20 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.003 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:21.086 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:21.120 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:21.175 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:21.217 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:21.219 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:21.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:21.258 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:21.297 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:21.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:21.340 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:21.397 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:29:21.398 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:21.423 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:21.440 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:21.462 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:29:21.482 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:21.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:21.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:21.595 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:21.640 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:21 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.640 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:21 smithi150 ceph-mon[194178]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.640 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:21 smithi150 ceph-mon[194178]: failed to encode map e343 with expected crc 2023-10-09T06:29:21.640 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:21 smithi150 ceph-mon[194178]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:29:21.640 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/730499738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:21.667 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:21.685 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:29:21.685 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:29:21.685 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:29:21.685 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-10-09T06:29:21.686 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:29:21.770 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:21.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:21.885 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[220921]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[220921]: failed to encode map e343 with expected crc 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[220921]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/730499738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[212750]: failed to encode map e342 with expected crc 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:29:21.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[212750]: osdmap e343: 8 total, 8 up, 8 in 2023-10-09T06:29:21.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/730499738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:21.967 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:22.065 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:22.403 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:22] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T06:29:22.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:29:22.692 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:22.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:22.801 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:22.826 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[220921]: failed to encode map e343 with expected crc 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[220921]: pgmap v1051: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13335/144537 objects degraded (9.226%); 6624/144537 objects misplaced (4.583%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13335/144537 objects degraded (9.226%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[212750]: failed to encode map e343 with expected crc 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[212750]: pgmap v1051: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13335/144537 objects degraded (9.226%); 6624/144537 objects misplaced (4.583%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13335/144537 objects degraded (9.226%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:22.827 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.828 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.901 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:22 smithi150 ceph-mon[194178]: failed to encode map e343 with expected crc 2023-10-09T06:29:22.901 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:22 smithi150 ceph-mon[194178]: pgmap v1051: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 13335/144537 objects degraded (9.226%); 6624/144537 objects misplaced (4.583%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T06:29:22.901 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13335/144537 objects degraded (9.226%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:22.901 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.901 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:22.930 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:22.984 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:23.056 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:23.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:23.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:23.164 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:23.218 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:23.338 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:23.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:23.443 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:23.518 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:23.767 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:23.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:23.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:23.951 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:24.103 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:24.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:24.214 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:24.259 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:24.283 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:29:24.283 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:29:24.283 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:29:24.283 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:29:24.311 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:24.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:24.414 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:24.441 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:24.571 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:24 smithi107 ceph-mon[212750]: pgmap v1052: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 13335/142668 objects degraded (9.347%); 6624/142668 objects misplaced (4.643%); 1.9 MiB/s, 86 objects/s recovering 2023-10-09T06:29:24.605 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:24.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:24.711 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:24.736 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:24.778 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:24.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:24.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:24.827 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:29:24.881 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:24.882 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:24 smithi107 ceph-mon[220921]: pgmap v1052: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 13335/142668 objects degraded (9.347%); 6624/142668 objects misplaced (4.643%); 1.9 MiB/s, 86 objects/s recovering 2023-10-09T06:29:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:24 smithi150 ceph-mon[194178]: pgmap v1052: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 13335/142668 objects degraded (9.347%); 6624/142668 objects misplaced (4.643%); 1.9 MiB/s, 86 objects/s recovering 2023-10-09T06:29:24.924 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:25.090 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:25.177 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:29:25.207 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:25.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:25.320 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:25.420 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:25.703 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:25.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:25.775 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:25.811 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:25.911 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:26.352 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:26.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:26.470 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:26.547 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:26 smithi107 ceph-mon[212750]: pgmap v1053: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13335/139128 objects degraded (9.585%); 6450/139128 objects misplaced (4.636%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:29:26.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:26 smithi107 ceph-mon[220921]: pgmap v1053: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13335/139128 objects degraded (9.585%); 6450/139128 objects misplaced (4.636%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:29:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:26 smithi150 ceph-mon[194178]: pgmap v1053: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13335/139128 objects degraded (9.585%); 6450/139128 objects misplaced (4.636%); 3.3 MiB/s, 77 objects/s recovering 2023-10-09T06:29:27.226 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:27.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:27.347 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:27.388 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:27.434 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:27.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:27.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:27.567 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:27.578 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1421180121' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.578 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13335/139128 objects degraded (9.585%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1421180121' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13335/139128 objects degraded (9.585%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.579 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.611 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:29:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1421180121' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:29:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13335/139128 objects degraded (9.585%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:27.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:27.655 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:27.715 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:29:27.715 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:29:27.715 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:29:27.715 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:29:27.715 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:29:28.309 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:28.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:28.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:28.445 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:28.586 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:29:28.879 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: pgmap v1054: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13335/137133 objects degraded (9.724%); 6450/137133 objects misplaced (4.703%); 1.4 MiB/s, 42 objects/s recovering 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.891 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[212750]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:28.897 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: pgmap v1054: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13335/137133 objects degraded (9.724%); 6450/137133 objects misplaced (4.703%); 1.4 MiB/s, 42 objects/s recovering 2023-10-09T06:29:28.897 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:28 smithi107 ceph-mon[220921]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: pgmap v1054: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13335/137133 objects degraded (9.724%); 6450/137133 objects misplaced (4.703%); 1.4 MiB/s, 42 objects/s recovering 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: osdmap e344: 8 total, 8 up, 8 in 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:28 smithi150 ceph-mon[194178]: failed to encode map e344 with expected crc 2023-10-09T06:29:28.999 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:29.054 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:29.140 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:29.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:29.196 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:29:29.246 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:29.330 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:29.546 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:29.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:29.649 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:29.747 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:29.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:29.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:29.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:29.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/893577298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:29.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:29 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/893577298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:29:29.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:29:29.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/893577298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:29.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:29 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:29:30.002 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:30.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:30.050 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:29:30.111 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:30.150 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:30.263 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:30.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:30.374 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:30.438 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:30.480 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:30.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:30.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:30.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: pgmap v1056: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13335/130758 objects degraded (10.198%); 6305/130758 objects misplaced (4.822%); 2.3 MiB/s, 55 objects/s recovering 2023-10-09T06:29:30.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:30.604 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.605 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.605 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:29:30.605 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[220921]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:29:30.658 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: pgmap v1056: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13335/130758 objects degraded (10.198%); 6305/130758 objects misplaced (4.822%); 2.3 MiB/s, 55 objects/s recovering 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:29:30.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:30 smithi107 ceph-mon[212750]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: pgmap v1056: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 13335/130758 objects degraded (10.198%); 6305/130758 objects misplaced (4.822%); 2.3 MiB/s, 55 objects/s recovering 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: failed to encode map e345 with expected crc 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:29:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:30 smithi150 ceph-mon[194178]: osdmap e345: 8 total, 8 up, 8 in 2023-10-09T06:29:31.379 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:29:31.379 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:29:31.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:29:31.379 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128124 9.3 9:c89580c2:::benchmark_data_smithi107_300016_object30552:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.22139890599999, 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0212228619999999, 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.380 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128124 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155177+0000", 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155176+0000", 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155182+0000", 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.421e-06 2023-10-09T06:29:31.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155193+0000", 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0852e-05 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155199+0000", 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1369999999999999e-06 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155278+0000", 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9202000000000005e-05 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155289+0000", 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0137e-05 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176313+0000", 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.021024865 2023-10-09T06:29:31.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176400+0000", 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6502000000000006e-05 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128130 9.3 9:c8d10970:::benchmark_data_smithi107_300016_object30558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.22136818000001, 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0213139, 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128130 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155208+0000", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155207+0000", 2023-10-09T06:29:31.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155210+0000", 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3010000000000001e-06 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155216+0000", 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8850000000000004e-06 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155220+0000", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1489999999999996e-06 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155312+0000", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2647999999999998e-05 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155321+0000", 2023-10-09T06:29:31.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3299999999999999e-06 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176444+0000", 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0211233209999999 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176522+0000", 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7644000000000004e-05 2023-10-09T06:29:31.390 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128141 9.3 9:ccf706f5:::benchmark_data_smithi107_300016_object30569:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.22135220899997, 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.051517185, 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128141 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155224+0000", 2023-10-09T06:29:31.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155223+0000", 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155226+0000", 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7269999999999999e-06 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155231+0000", 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2979999999999999e-06 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155235+0000", 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4400000000000001e-06 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155342+0000", 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000107181 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.155350+0000", 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0809999999999994e-06 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176542+0000", 2023-10-09T06:29:31.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.021192391 2023-10-09T06:29:31.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176616+0000", 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3244000000000005e-05 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.176694+0000", 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8126000000000004e-05 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188370+0000", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011675899999999999 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206655+0000", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.018285286000000001 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206684+0000", 2023-10-09T06:29:31.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8500000000000002e-05 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206717+0000", 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3290999999999997e-05 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206741+0000", 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4049e-05 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.399 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128156 9.3 9:c32ea811:::benchmark_data_smithi107_300016_object30584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.20793729900004, 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.038242423, 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.400 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128156 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168639+0000", 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168638+0000", 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168643+0000", 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5550000000000002e-06 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168655+0000", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2065999999999999e-05 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168662+0000", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3929999999999996e-06 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168734+0000", 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2033000000000006e-05 2023-10-09T06:29:31.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.168747+0000", 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.322e-05 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.179788+0000", 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.011041641 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.179917+0000", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012808599999999999 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.180032+0000", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115358 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.191939+0000", 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011907449000000001 2023-10-09T06:29:31.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206789+0000", 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014849571000000001 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206832+0000", 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2661999999999999e-05 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206859+0000", 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7631999999999999e-05 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206881+0000", 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1809000000000001e-05 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.407 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128192 9.3 9:c85dd9b2:::benchmark_data_smithi107_300016_object30620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.16013983300002, 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99057770000000001, 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128192 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216436+0000", 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:31.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216438+0000", 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0999999999999998e-06 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216445+0000", 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8929999999999999e-06 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216449+0000", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1679999999999997e-06 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216513+0000", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3713000000000005e-05 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.216523+0000", 2023-10-09T06:29:31.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0349e-05 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.182852+0000", 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96632860799999998 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.182975+0000", 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012307299999999999 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.183107+0000", 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013245499999999999 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.191948+0000", 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0088404629999999998 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206813+0000", 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014865295000000001 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206965+0000", 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015169000000000001 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.206990+0000", 2023-10-09T06:29:31.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.58e-05 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207014+0000", 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3433000000000001e-05 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.415 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128198 9.3 9:c0c2ed7c:::benchmark_data_smithi107_300016_object30626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.14719567300006, 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97778661200000005, 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128198 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229380+0000", 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229379+0000", 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229384+0000", 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.172e-06 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229393+0000", 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.927e-06 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229400+0000", 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8449999999999997e-06 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229463+0000", 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2298999999999994e-05 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229475+0000", 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1949e-05 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186026+0000", 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95655169900000003 2023-10-09T06:29:31.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186102+0000", 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5545999999999998e-05 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.186180+0000", 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7992999999999997e-05 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.197183+0000", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011003527000000001 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207065+0000", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0098818469999999992 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207101+0000", 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6117e-05 2023-10-09T06:29:31.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207132+0000", 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0901000000000001e-05 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207167+0000", 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4691999999999999e-05 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.423 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:128203 9.3 9:cb76ab71:::benchmark_data_smithi107_300016_object30631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 551.14713488699999, 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97972671, 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 128203 2023-10-09T06:29:31.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229441+0000", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229443+0000", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.61e-06 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229451+0000", 2023-10-09T06:29:31.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229455+0000", 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0099999999999997e-06 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229506+0000", 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1239000000000002e-05 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:20.229515+0000", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9570000000000008e-06 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188401+0000", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.958885498 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188474+0000", 2023-10-09T06:29:31.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2731999999999994e-05 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.188540+0000", 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6625999999999995e-05 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.197191+0000", 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0086507489999999992 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.207200+0000", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010008993000000001 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209114+0000", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019145500000000001 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209144+0000", 2023-10-09T06:29:31.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9615999999999999e-05 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:20:21.209168+0000", 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3657000000000001e-05 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189411 9.7 9:e8e21091:::benchmark_data_smithi107_300016_object91839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:29:31.431 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 460.854411744, 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99458555100000001, 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189411 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:29:31.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522164+0000", 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522162+0000", 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522169+0000", 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8929999999999999e-06 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522184+0000", 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4867e-05 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522205+0000", 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0511999999999999e-05 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522246+0000", 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1007999999999998e-05 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.522252+0000", 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5060000000000001e-06 2023-10-09T06:29:31.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496087+0000", 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97383483500000001 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496175+0000", 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7767000000000002e-05 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.496252+0000", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7286999999999994e-05 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499058+0000", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00280555 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511729+0000", 2023-10-09T06:29:31.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012671003 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516675+0000", 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049465480000000003 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516706+0000", 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1047999999999998e-05 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.516750+0000", 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3436000000000002e-05 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189413 9.d 9:b8ff20a4:::benchmark_data_smithi107_300016_object91841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 460.84975924700001, 2023-10-09T06:29:31.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97899737899999995, 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189413 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:29:31.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526817+0000", 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526816+0000", 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526821+0000", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6450000000000002e-06 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526834+0000", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2975e-05 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526840+0000", 2023-10-09T06:29:31.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1900000000000003e-06 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526893+0000", 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2958999999999999e-05 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.526903+0000", 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9480000000000003e-06 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494707+0000", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.96780485699999996 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494861+0000", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015316799999999999 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.494947+0000", 2023-10-09T06:29:31.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6058999999999997e-05 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505292+0000", 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010345277999999999 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505617+0000", 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000325345 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505718+0000", 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100776 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505783+0000", 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4508e-05 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.505814+0000", 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1517999999999999e-05 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189420 9.7 9:ee572d28:::benchmark_data_smithi107_300016_object91848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 460.83526678499999, 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98343943099999998, 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.447 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189420 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541309+0000", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541307+0000", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541317+0000", 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6630000000000005e-06 2023-10-09T06:29:31.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541338+0000", 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.073e-05 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541346+0000", 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4640000000000006e-06 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541405+0000", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9061999999999998e-05 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.541417+0000", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1871e-05 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499255+0000", 2023-10-09T06:29:31.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95783806999999999 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499327+0000", 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1607999999999996e-05 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.499399+0000", 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1766000000000004e-05 2023-10-09T06:29:31.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.506518+0000", 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071195950000000003 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511815+0000", 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0052969769999999996 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524668+0000", 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012852763 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524703+0000", 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5642999999999997e-05 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524749+0000", 2023-10-09T06:29:31.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.507e-05 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45275.0:189422 9.7 9:e5521dd9:::benchmark_data_smithi107_300016_object91850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 460.83295756500002, 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98124100700000005, 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.455 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45275", 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 189422 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543618+0000", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543617+0000", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543623+0000", 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3809999999999998e-06 2023-10-09T06:29:31.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543636+0000", 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2639e-05 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543642+0000", 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8619999999999997e-06 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543692+0000", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9459999999999997e-05 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:50.543699+0000", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8839999999999997e-06 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501750+0000", 2023-10-09T06:29:31.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95805110599999999 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501835+0000", 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5227000000000002e-05 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.501898+0000", 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3380000000000006e-05 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.506526+0000", 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046276709999999999 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.511862+0000", 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053355900000000003 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524806+0000", 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.012944638 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524837+0000", 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0956e-05 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:21:51.524859+0000", 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2084999999999999e-05 2023-10-09T06:29:31.462 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67292 11.5 11:ae93ee2a:::benchmark_data_smithi107_418363_object67291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 126.642825672, 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3007185510000001, 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.463 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67292 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733749+0000", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733812+0000", 2023-10-09T06:29:31.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3002000000000004e-05 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733823+0000", 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0909e-05 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733828+0000", 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6720000000000003e-06 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733869+0000", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1508999999999998e-05 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733970+0000", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100907 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.734066+0000", 2023-10-09T06:29:31.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5404999999999999e-05 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.735966+0000", 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001899916 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.736151+0000", 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018566700000000001 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872581+0000", 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13643008600000001 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872592+0000", 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0746e-05 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871235+0000", 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99864300699999997 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871251+0000", 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5563e-05 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034370+0000", 2023-10-09T06:29:31.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631196100000001 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034469+0000", 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8395999999999998e-05 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67301 11.5 11:a1b1512d:::benchmark_data_smithi107_418363_object67300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:29:31.471 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 126.52832251, 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1863352649999999, 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67301 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848252+0000", 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848308+0000", 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5899999999999997e-05 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848320+0000", 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1922e-05 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848326+0000", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2099999999999998e-06 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848378+0000", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1793999999999999e-05 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848521+0000", 2023-10-09T06:29:31.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142769 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848677+0000", 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000155828 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.850815+0000", 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021378920000000002 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852117+0000", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00130197 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872639+0000", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020522744999999998 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872646+0000", 2023-10-09T06:29:31.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7649999999999996e-06 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871333+0000", 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99868642699999999 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871343+0000", 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0808999999999999e-05 2023-10-09T06:29:31.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034505+0000", 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631621480000001 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034589+0000", 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3185999999999999e-05 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67305 11.5 11:aa961c7b:::benchmark_data_smithi107_418363_object67304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 126.524255713, 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.182389261, 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67305 2023-10-09T06:29:31.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852319+0000", 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852365+0000", 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6390999999999997e-05 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852381+0000", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5128e-05 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852388+0000", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2640000000000002e-06 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853387+0000", 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00099880900000000002 2023-10-09T06:29:31.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853487+0000", 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100614 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853585+0000", 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7887999999999995e-05 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.855632+0000", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020469289999999998 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.857531+0000", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0018985919999999999 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872679+0000", 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015148790000000001 2023-10-09T06:29:31.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872686+0000", 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.922e-06 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871386+0000", 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99869956000000004 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871393+0000", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2060000000000001e-06 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034627+0000", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1632337100000001 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034709+0000", 2023-10-09T06:29:31.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2608000000000006e-05 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67306 11.5 11:aea7a0bf:::benchmark_data_smithi107_418363_object67305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 126.512030065, 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.17028496, 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:29:31.488 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67306 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.489 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864545+0000", 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864595+0000", 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0657e-05 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864605+0000", 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9380000000000004e-06 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864610+0000", 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0130000000000001e-06 2023-10-09T06:29:31.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864660+0000", 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0006000000000002e-05 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865068+0000", 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00040736199999999998 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865233+0000", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000165496 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872957+0000", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0077238020000000001 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872962+0000", 2023-10-09T06:29:31.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2599999999999996e-06 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871429+0000", 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99846622200000001 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871438+0000", 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8109999999999997e-06 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034745+0000", 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1633062599999999 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034831+0000", 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6227e-05 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.495 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67309 11.5 11:af8d0774:::benchmark_data_smithi107_418363_object67308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 126.497216278, 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.174921791, 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.496 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67309 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879358+0000", 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879394+0000", 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5299000000000002e-05 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879411+0000", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7544999999999999e-05 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879417+0000", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.784e-06 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879464+0000", 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6631999999999997e-05 2023-10-09T06:29:31.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879484+0000", 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0475999999999998e-05 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871466+0000", 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99198237899999997 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871472+0000", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6790000000000002e-06 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034876+0000", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1634033669999999 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035033+0000", 2023-10-09T06:29:31.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000157217 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035259+0000", 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022594499999999999 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041596+0000", 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0063371349999999998 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041946+0000", 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00035007099999999998 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051163+0000", 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0092172039999999997 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.503 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051212+0000", 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8730999999999998e-05 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054281+0000", 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030695850000000001 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.504 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:136631 11.4 11:21fdf331:::benchmark_data_smithi107_418363_object46195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:26.516436+0000", 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.8601394750000004, 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.18455815, 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 136631 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.505 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516436+0000", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516436+0000", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516435+0000", 2023-10-09T06:29:31.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516442+0000", 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3130000000000001e-06 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516455+0000", 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2807e-05 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516462+0000", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5869999999999998e-06 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516513+0000", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1382999999999999e-05 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516625+0000", 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000111766 2023-10-09T06:29:31.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.516734+0000", 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000109298 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.520477+0000", 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003742506 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.521423+0000", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00094638299999999997 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700913+0000", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.17949032000000001 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700956+0000", 2023-10-09T06:29:31.510 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2499000000000001e-05 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700995+0000", 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8849e-05 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:136640 11.5 11:a30aa8e9:::benchmark_data_smithi107_418363_object46204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e343)", 2023-10-09T06:29:31.511 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:26.525983+0000", 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.8505926490000002, 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.17800949799999999, 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 136640 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.512 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.525983+0000", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.525983+0000", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.525982+0000", 2023-10-09T06:29:31.513 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.525988+0000", 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0399999999999998e-06 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.526005+0000", 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7096999999999999e-05 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.526016+0000", 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.079e-05 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.526130+0000", 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011401800000000001 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.526286+0000", 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015638100000000001 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.526440+0000", 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015400400000000001 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.534657+0000", 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082165680000000005 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700820+0000", 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16616297799999999 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700966+0000", 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000146534 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.701011+0000", 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4804e-05 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.703993+0000", 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0029814809999999998 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.518 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.45989.0:136641 11.3 e343/297 11:cc457ca3:::benchmark_data_smithi107_418363_object46205:head v 343'8663, mlcod=343'8661)", 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:26.535966+0000", 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.8406099339999997, 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.17903380199999999, 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535966+0000", 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.519 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535966+0000", 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535966+0000", 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535970+0000", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0389999999999998e-06 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535974+0000", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3170000000000003e-06 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.535977+0000", 2023-10-09T06:29:31.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9019999999999999e-06 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.536009+0000", 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2190000000000002e-05 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.700714+0000", 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16470526599999999 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.714959+0000", 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014244651000000001 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:26.715000+0000", 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0796e-05 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:137548 11.5 11:a32f6745:::benchmark_data_smithi107_418363_object47112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:28.072494+0000", 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 3.3040821660000002, 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.17759301999999999, 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 137548 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.524 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072494+0000", 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072494+0000", 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072492+0000", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072503+0000", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1663e-05 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072529+0000", 2023-10-09T06:29:31.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5154e-05 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072541+0000", 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2398999999999999e-05 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072644+0000", 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010257200000000001 2023-10-09T06:29:31.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072889+0000", 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024502600000000002 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 5,6", 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.072995+0000", 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106358 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.076117+0000", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00312167 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.079453+0000", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003335944 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.247085+0000", 2023-10-09T06:29:31.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.167632264 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.247166+0000", 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1205999999999995e-05 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:28.250087+0000", 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0029207510000000001 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.530 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.531 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.531 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:29:31.531 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:29:31.531 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:29:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:31.532 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:31.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:31.633 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:31.703 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:31.713 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:31.775 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:31.776 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:31.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:31.800 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:31.822 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:31.855 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:31.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:31.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:31.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:31.910 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:32.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.132 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:29:32.133 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:32.134 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:32.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:32.235 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:32.315 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:32.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:32] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:29:32.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:29:32.428 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:32.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:32.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:32.615 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:32.679 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:32.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:32.783 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:32.839 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[220921]: pgmap v1058: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13335/128637 objects degraded (10.366%); 6305/128637 objects misplaced (4.901%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:29:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4242213616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:32.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13335/128637 objects degraded (10.366%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:32.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[212750]: pgmap v1058: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13335/128637 objects degraded (10.366%); 6305/128637 objects misplaced (4.901%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:29:32.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4242213616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:32.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13335/128637 objects degraded (10.366%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:32.946 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:32.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:32 smithi150 ceph-mon[194178]: pgmap v1058: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13335/128637 objects degraded (10.366%); 6305/128637 objects misplaced (4.901%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:29:32.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4242213616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:32.952 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13335/128637 objects degraded (10.366%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:32.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:33.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:33.145 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:33.219 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:33.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:33.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:33.329 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:33.350 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:33.413 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:33.521 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:33.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:33.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:33.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:29:33.626 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:33.664 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:33.735 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:33.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:33.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:33.843 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:33.885 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:33.965 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:33.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:33.997 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:29:33.997 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:29:34.074 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:34.145 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:34.306 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:34.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:34.428 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:34.483 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:34.560 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:34.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:34.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:34.677 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:34.704 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:34.746 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:34.750 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:34 smithi150 ceph-mon[194178]: pgmap v1059: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 13335/126207 objects degraded (10.566%); 6305/126207 objects misplaced (4.996%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:29:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:34.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:34.852 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:34.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:35.015 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:35.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:35.123 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:34 smithi107 ceph-mon[212750]: pgmap v1059: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 13335/126207 objects degraded (10.566%); 6305/126207 objects misplaced (4.996%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:29:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:34 smithi107 ceph-mon[220921]: pgmap v1059: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 13335/126207 objects degraded (10.566%); 6305/126207 objects misplaced (4.996%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T06:29:35.185 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:35.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:29:35.226 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:35.381 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:35.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:35.478 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:35.488 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:35.642 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:35.927 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:35.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:35.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:36.038 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:36.163 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:36.269 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:29:36.269 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:29:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:36 smithi107 ceph-mon[212750]: pgmap v1060: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 13335/121506 objects degraded (10.975%); 6156/121506 objects misplaced (5.066%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:36 smithi107 ceph-mon[220921]: pgmap v1060: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 13335/121506 objects degraded (10.975%); 6156/121506 objects misplaced (5.066%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:37.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:36 smithi150 ceph-mon[194178]: pgmap v1060: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 13335/121506 objects degraded (10.975%); 6156/121506 objects misplaced (5.066%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:29:37.560 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:37.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:37.674 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:37.720 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:37.840 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:37.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:37.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:37.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13335/121506 objects degraded (10.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:37.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13335/121506 objects degraded (10.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:37.948 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:37.975 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13335/121506 objects degraded (10.975%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:38.211 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:38.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:38.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:38.336 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:38.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:38.487 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:38.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:38.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:38.526 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:29:38.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:38.703 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:38 smithi107 ceph-mon[220921]: pgmap v1061: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 13335/119175 objects degraded (11.189%); 6156/119175 objects misplaced (5.166%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:29:38.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:38 smithi107 ceph-mon[212750]: pgmap v1061: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 13335/119175 objects degraded (11.189%); 6156/119175 objects misplaced (5.166%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:29:39.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:38 smithi150 ceph-mon[194178]: pgmap v1061: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 13335/119175 objects degraded (11.189%); 6156/119175 objects misplaced (5.166%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:29:39.439 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:39.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:39.547 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:39.634 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:39.694 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:39.778 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:39.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:39.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:39.881 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:39.942 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:40.008 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:40.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:40.123 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:40.263 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:40.290 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:40.412 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:40.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:40.518 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:40.555 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:40.637 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:40.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:40.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[212750]: pgmap v1062: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13357/112071 objects degraded (11.918%); 5965/112071 objects misplaced (5.323%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:40.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3272681380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:40.731 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:40.741 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:29:40.745 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:40.823 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:40.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:40 smithi150 ceph-mon[194178]: pgmap v1062: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13357/112071 objects degraded (11.918%); 5965/112071 objects misplaced (5.323%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:40.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3272681380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:40.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:41.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[220921]: pgmap v1062: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 13357/112071 objects degraded (11.918%); 5965/112071 objects misplaced (5.323%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:41.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3272681380' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:41.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:29:41.776 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:29:42.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:29:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.018 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.018 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.018 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.018 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:29:42.020 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:29:42.022 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:42] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: osdmap e346: 8 total, 8 up, 8 in 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: failed to encode map e346 with expected crc 2023-10-09T06:29:42.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:29:42.581 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:42.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:42.688 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:42.750 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:42.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: pgmap v1064: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13357/109611 objects degraded (12.186%); 5965/109611 objects misplaced (5.442%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13357/109611 objects degraded (12.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: pgmap v1064: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13357/109611 objects degraded (12.186%); 5965/109611 objects misplaced (5.442%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:42.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:29:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:29:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: failed to encode map e347 with expected crc 2023-10-09T06:29:42.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13357/109611 objects degraded (12.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:43.104 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:43.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: pgmap v1064: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13357/109611 objects degraded (12.186%); 5965/109611 objects misplaced (5.442%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2454892501' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: osdmap e347: 8 total, 8 up, 8 in 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: failed to encode map e347 with expected crc 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: failed to encode map e347 with expected crc 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: failed to encode map e347 with expected crc 2023-10-09T06:29:43.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13357/109611 objects degraded (12.186%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:43.207 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:43.242 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:43.336 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:43.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:43.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:43.398 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:29:43.457 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:43.528 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:43.778 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:43.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:43.894 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:43.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:44.539 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:44.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:44.646 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:44.715 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:44.835 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:44 smithi107 ceph-mon[212750]: pgmap v1066: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13357/107172 objects degraded (12.463%); 5965/107172 objects misplaced (5.566%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:29:44.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:44 smithi107 ceph-mon[220921]: pgmap v1066: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13357/107172 objects degraded (12.463%); 5965/107172 objects misplaced (5.566%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:29:44.903 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:44.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:45.024 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:45.108 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:45.137 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:44 smithi150 ceph-mon[194178]: pgmap v1066: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 13357/107172 objects degraded (12.463%); 5965/107172 objects misplaced (5.566%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:29:45.244 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:29:45.279 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:45.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:45.293 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:45.327 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:45.372 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:45.391 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:45.433 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:45.524 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:45.524 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:45.528 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:29:45.528 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:29:45.528 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:29:45.528 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:29:45.528 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:29:45.578 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:45.651 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:45.691 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:45.693 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.693 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.693 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.693 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:29:45.694 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:29:46.002 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:46.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:46.111 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:46.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2442621474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:46.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2442621474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2442621474' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:29:46.194 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:46 smithi150 ceph-mon[194178]: pgmap v1067: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13380/102369 objects degraded (13.070%); 5774/102369 objects misplaced (5.640%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:29:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:47.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:46 smithi107 ceph-mon[212750]: pgmap v1067: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13380/102369 objects degraded (13.070%); 5774/102369 objects misplaced (5.640%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:29:47.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:46 smithi107 ceph-mon[220921]: pgmap v1067: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 13380/102369 objects degraded (13.070%); 5774/102369 objects misplaced (5.640%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:29:47.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:29:47.302 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:29:47.329 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:47.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:47.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:47.503 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:47.775 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:47.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:47.884 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:47.910 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:48.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13380/102369 objects degraded (13.070%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:48.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13380/102369 objects degraded (13.070%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13380/102369 objects degraded (13.070%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:48.525 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:48.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:48.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:48.614 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:48.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:48.736 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:48.965 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:48.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:48 smithi107 ceph-mon[212750]: pgmap v1068: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 13380/99921 objects degraded (13.391%); 5774/99921 objects misplaced (5.779%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:29:48.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:48 smithi107 ceph-mon[220921]: pgmap v1068: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 13380/99921 objects degraded (13.391%); 5774/99921 objects misplaced (5.779%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:29:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:48.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:49.075 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:49.136 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:48 smithi150 ceph-mon[194178]: pgmap v1068: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 13380/99921 objects degraded (13.391%); 5774/99921 objects misplaced (5.779%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:29:49.141 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:49.180 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:49.217 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:49.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:49.335 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:49.416 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:49.630 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:49.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:49.734 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:49.808 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:49.889 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:49.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:49.997 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:50.058 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:50.139 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:50.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:50.245 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:50.271 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:50.333 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:50.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:50.443 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:50.450 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:29:50.543 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:50.716 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:50.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:50.836 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:50.866 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:50.887 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:29:50.935 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:50.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:51.042 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:51.055 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[212750]: pgmap v1069: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 13409/93075 objects degraded (14.407%); 5581/93075 objects misplaced (5.996%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:29:51.055 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3845371247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.056 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.057 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[220921]: pgmap v1069: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 13409/93075 objects degraded (14.407%); 5581/93075 objects misplaced (5.996%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:29:51.057 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3845371247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.057 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:50 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.106 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:50 smithi150 ceph-mon[194178]: pgmap v1069: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 13409/93075 objects degraded (14.407%); 5581/93075 objects misplaced (5.996%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:29:51.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3845371247' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.107 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:50 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:29:51.107 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:51.207 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:51.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:51.320 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:51.357 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:51.454 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:51.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:51.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:51.619 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:51.736 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:51.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:51.849 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:29:51.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:51 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:29:51.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:51 smithi150 ceph-mon[194178]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:29:51.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:51 smithi150 ceph-mon[194178]: failed to encode map e348 with expected crc 2023-10-09T06:29:51.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:51 smithi150 ceph-mon[194178]: failed to encode map e348 with expected crc 2023-10-09T06:29:51.876 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:51.958 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:29:51.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:51.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:29:52.064 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:52.145 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:29:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:29:52] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:29:52.145 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:29:52.145 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[212750]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:29:52.145 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:29:52.145 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[212750]: failed to encode map e348 with expected crc 2023-10-09T06:29:52.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:29:52.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[220921]: osdmap e348: 8 total, 8 up, 8 in 2023-10-09T06:29:52.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[220921]: failed to encode map e348 with expected crc 2023-10-09T06:29:52.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:51 smithi107 ceph-mon[220921]: failed to encode map e348 with expected crc 2023-10-09T06:29:52.159 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:52.781 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:29:52.912 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:52.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:53.024 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:52 smithi107 ceph-mon[212750]: pgmap v1071: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 13409/90969 objects degraded (14.740%); 5581/90969 objects misplaced (6.135%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:29:53.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13409/90969 objects degraded (14.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:53.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:52 smithi107 ceph-mon[220921]: pgmap v1071: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 13409/90969 objects degraded (14.740%); 5581/90969 objects misplaced (6.135%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:29:53.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13409/90969 objects degraded (14.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:52 smithi150 ceph-mon[194178]: pgmap v1071: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 13409/90969 objects degraded (14.740%); 5581/90969 objects misplaced (6.135%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:29:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13409/90969 objects degraded (14.740%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:53.169 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:53.281 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:53.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:53.396 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:29:53.402 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:53.437 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:53.517 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:29:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:53.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:29:53.629 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:53.727 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:53.958 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:53.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:54.060 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189575 9.9 9:9c7590b3:::benchmark_data_smithi107_300016_object92003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:52.150024+0000", 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 481.906590392, 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97474236199999997, 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.061 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189575 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150024+0000", 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150024+0000", 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150023+0000", 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150030+0000", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9280000000000004e-06 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150044+0000", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3804e-05 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150060+0000", 2023-10-09T06:29:54.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6042e-05 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150116+0000", 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5933999999999997e-05 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150126+0000", 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8390000000000002e-06 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.115462+0000", 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96533579400000002 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.115576+0000", 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011464999999999999 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.115652+0000", 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6020999999999996e-05 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.118922+0000", 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032697310000000001 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.121936+0000", 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030135589999999999 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124711+0000", 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002775624 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124742+0000", 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0885e-05 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124767+0000", 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4814999999999999e-05 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189579 9.9 9:9746bc71:::benchmark_data_smithi107_300016_object92007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:52.150447+0000", 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 481.906168342, 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97452103300000004, 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.070 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189579 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150447+0000", 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150447+0000", 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150446+0000", 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150451+0000", 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.707e-06 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150461+0000", 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0139000000000001e-05 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150467+0000", 2023-10-09T06:29:54.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4520000000000002e-06 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150506+0000", 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9218999999999998e-05 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:52.150514+0000", 2023-10-09T06:29:54.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3590000000000002e-06 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.118957+0000", 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96844311500000002 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.119062+0000", 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105016 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.119139+0000", 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6810000000000005e-05 2023-10-09T06:29:54.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124691+0000", 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005552062 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124848+0000", 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015762799999999999 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124884+0000", 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6137000000000003e-05 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124929+0000", 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4186e-05 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.124968+0000", 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8868999999999997e-05 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45275.0:189863 9.5 9:ab26cf66:::benchmark_data_smithi107_300016_object92291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e231)", 2023-10-09T06:29:54.081 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:21:53.204808+0000", 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 480.85180643899997, 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97959113900000006, 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45275", 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/23425022", 2023-10-09T06:29:54.082 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 189863 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204808+0000", 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204808+0000", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204808+0000", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204811+0000", 2023-10-09T06:29:54.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9679999999999998e-06 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204817+0000", 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4819999999999999e-06 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204823+0000", 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4720000000000003e-06 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204910+0000", 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6677999999999999e-05 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:53.204918+0000", 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0160000000000008e-06 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.140621+0000", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.93570335699999996 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.140721+0000", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9856000000000001e-05 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.140818+0000", 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7658999999999995e-05 2023-10-09T06:29:54.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.148413+0000", 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075950410000000003 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.148453+0000", 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9051000000000003e-05 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.184304+0000", 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.035851930999999997 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.184358+0000", 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3328000000000002e-05 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:21:54.184400+0000", 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1774999999999999e-05 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82043 10.8 10:13dd737b:::benchmark_data_smithi107_363200_object82042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.23865526999998, 2023-10-09T06:29:54.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99744534200000001, 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82043 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:29:54.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817959+0000", 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817982+0000", 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3319000000000002e-05 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817989+0000", 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8510000000000001e-06 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817995+0000", 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3319999999999999e-06 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.041469+0000", 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22347494400000001 2023-10-09T06:29:54.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.041480+0000", 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0916e-05 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.815307+0000", 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.77382648399999998 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.815405+0000", 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8045000000000001e-05 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82057 10.14 10:2aa4dde9:::benchmark_data_smithi107_363200_object82056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:29:54.096 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.23861014699997, 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.027758696, 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82057 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.097 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818004+0000", 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818027+0000", 2023-10-09T06:29:54.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2691e-05 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818033+0000", 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0950000000000001e-06 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818038+0000", 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9620000000000001e-06 2023-10-09T06:29:54.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112804+0000", 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29476629399999998 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112813+0000", 2023-10-09T06:29:54.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5290000000000008e-06 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834246+0000", 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72143284799999996 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834408+0000", 2023-10-09T06:29:54.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000162688 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834565+0000", 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000156637 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838810+0000", 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004244772 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842036+0000", 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032265340000000001 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845692+0000", 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036560910000000002 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845731+0000", 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9054000000000003e-05 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845763+0000", 2023-10-09T06:29:54.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1931999999999999e-05 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82062 10.6 10:6651e52c:::benchmark_data_smithi107_363200_object82061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.238572637, 2023-10-09T06:29:54.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.02105132, 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82062 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.108 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818064+0000", 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2092000000000001e-05 2023-10-09T06:29:54.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818070+0000", 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0480000000000004e-06 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818074+0000", 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.476e-06 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.068355+0000", 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25028024399999999 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.068366+0000", 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1059e-05 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.820857+0000", 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.75249149800000004 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.820938+0000", 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0890999999999995e-05 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:29:54.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.821038+0000", 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9990999999999999e-05 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830794+0000", 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0097563709999999998 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837309+0000", 2023-10-09T06:29:54.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065145050000000003 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838985+0000", 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016758540000000001 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839051+0000", 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5963000000000005e-05 2023-10-09T06:29:54.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839094+0000", 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2735000000000003e-05 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82063 10.14 10:28d0e219:::benchmark_data_smithi107_363200_object82062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.23853590099998, 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0277804100000001, 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.117 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82063 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:29:54.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:29:54.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818103+0000", 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.457e-05 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818109+0000", 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.874e-06 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818114+0000", 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5750000000000002e-06 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112833+0000", 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29471960000000003 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112839+0000", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0900000000000001e-06 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835720+0000", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72288063899999999 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835814+0000", 2023-10-09T06:29:54.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4205999999999999e-05 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835915+0000", 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010126100000000001 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.841460+0000", 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055449770000000004 2023-10-09T06:29:54.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842068+0000", 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00060749199999999995 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845793+0000", 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0037246670000000001 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845828+0000", 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5938e-05 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845859+0000", 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0892000000000002e-05 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82065 10.14 10:2b997f2c:::benchmark_data_smithi107_363200_object82064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.19214915100002, 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98147874499999999, 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82065 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.126 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864463+0000", 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999971 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864505+0000", 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2292000000000001e-05 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864533+0000", 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7350000000000001e-05 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864546+0000", 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3811e-05 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112859+0000", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24831273300000001 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112865+0000", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6740000000000002e-06 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.836997+0000", 2023-10-09T06:29:54.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72413222300000002 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837124+0000", 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012672599999999999 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837297+0000", 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017315000000000001 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.841468+0000", 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004171152 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842344+0000", 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00087613600000000002 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845891+0000", 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035471579999999999 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845926+0000", 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4802000000000003e-05 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845944+0000", 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8235000000000001e-05 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82067 10.4 10:24dff4b3:::benchmark_data_smithi107_363200_object82066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 388.10522586299999, 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.87997381100000005, 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.135 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82067 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951387+0000", 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:54.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951421+0000", 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4085000000000003e-05 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951440+0000", 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9405000000000001e-05 2023-10-09T06:29:54.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951453+0000", 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2582000000000001e-05 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112887+0000", 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.161433932 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112894+0000", 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5320000000000001e-06 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.802843+0000", 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.68994917 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.802992+0000", 2023-10-09T06:29:54.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014873499999999999 2023-10-09T06:29:54.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:29:54.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.803142+0000", 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015000899999999999 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.812320+0000", 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091776930000000007 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.819893+0000", 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075735519999999999 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831287+0000", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011393133999999999 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831328+0000", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1437000000000003e-05 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831363+0000", 2023-10-09T06:29:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4864999999999999e-05 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82072 10.6 10:6072d45f:::benchmark_data_smithi107_363200_object82071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 387.926118621, 2023-10-09T06:29:54.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70870548099999997, 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82072 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130494+0000", 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130525+0000", 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1429e-05 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130553+0000", 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7976000000000001e-05 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130570+0000", 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6478e-05 2023-10-09T06:29:54.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130627+0000", 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.766e-05 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130641+0000", 2023-10-09T06:29:54.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3349e-05 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826121+0000", 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.69548077900000005 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826306+0000", 2023-10-09T06:29:54.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018486500000000001 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826495+0000", 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018890399999999999 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830806+0000", 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043105299999999999 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838342+0000", 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075360100000000001 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839133+0000", 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00079077200000000002 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839170+0000", 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7970999999999999e-05 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839202+0000", 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1214999999999998e-05 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82075 10.6 10:61ef0778:::benchmark_data_smithi107_363200_object82074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 387.924067229, 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.71286330200000003, 2023-10-09T06:29:54.155 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82075 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132545+0000", 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132581+0000", 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5306999999999999e-05 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132596+0000", 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5758999999999999e-05 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132607+0000", 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0243e-05 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132661+0000", 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4249000000000002e-05 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132673+0000", 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1867e-05 2023-10-09T06:29:54.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830875+0000", 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.69820269099999999 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831031+0000", 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000155606 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831178+0000", 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014734399999999999 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.836336+0000", 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051574699999999999 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839233+0000", 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002896872 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845317+0000", 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0060840479999999999 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845364+0000", 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7309999999999999e-05 2023-10-09T06:29:54.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845411+0000", 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6808000000000003e-05 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:172873 10.14 10:2d95d584:::benchmark_data_smithi107_363200_object68781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:25:18.685135+0000", 2023-10-09T06:29:54.163 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.37148005400002, 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30143693999999999, 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 172873 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685135+0000", 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685135+0000", 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685133+0000", 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685141+0000", 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7589999999999997e-06 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685153+0000", 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1717e-05 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685161+0000", 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0979999999999994e-06 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.971634+0000", 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28647283200000001 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.971766+0000", 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013231399999999999 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.972286+0000", 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052015000000000004 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.978430+0000", 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0061437719999999996 2023-10-09T06:29:54.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.981194+0000", 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027641509999999998 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.986498+0000", 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053043969999999998 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.986537+0000", 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9094999999999998e-05 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.986572+0000", 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4298000000000003e-05 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:172874 10.18 10:1ca73be6:::benchmark_data_smithi107_363200_object68782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e282)", 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:25:18.685175+0000", 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 275.37143970300002, 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30052368499999998, 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 172874 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.171 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685175+0000", 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685175+0000", 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685175+0000", 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685178+0000", 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1219999999999999e-06 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685184+0000", 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7409999999999998e-06 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.685189+0000", 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1800000000000004e-06 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.971633+0000", 2023-10-09T06:29:54.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28644445800000001 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.971765+0000", 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013233400000000001 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:29:54.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.972299+0000", 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00053396400000000005 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.980006+0000", 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077068359999999999 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.982497+0000", 2023-10-09T06:29:54.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024911529999999999 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.985623+0000", 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031250549999999998 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.985664+0000", 2023-10-09T06:29:54.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1884000000000001e-05 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:25:18.985699+0000", 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4446999999999997e-05 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:49053 11.6 11:6d704939:::benchmark_data_smithi107_418363_object49052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 167.78391700099999, 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31953208300000002, 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 49053 2023-10-09T06:29:54.179 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:29:54.180 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272696+0000", 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.181 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272716+0000", 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9911e-05 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272731+0000", 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5e-05 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272740+0000", 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.816e-06 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272790+0000", 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9577000000000003e-05 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.183 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272963+0000", 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173609 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.273120+0000", 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015633199999999999 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.275458+0000", 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023386370000000002 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.279066+0000", 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036078880000000001 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592130+0000", 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31306408099999999 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592188+0000", 2023-10-09T06:29:54.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7485999999999998e-05 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592230+0000", 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2120999999999999e-05 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:49069 11.6 11:6dcb3553:::benchmark_data_smithi107_418363_object49068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:29:54.187 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 167.76796792600001, 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30429343399999997, 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 49069 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288645+0000", 2023-10-09T06:29:54.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288669+0000", 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3502999999999998e-05 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288686+0000", 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7229000000000001e-05 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288695+0000", 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2599999999999994e-06 2023-10-09T06:29:54.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288734+0000", 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8872000000000003e-05 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288844+0000", 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011034700000000001 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288944+0000", 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9584000000000001e-05 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.290400+0000", 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001455643 2023-10-09T06:29:54.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.291170+0000", 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000770099 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592831+0000", 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30166106100000001 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592902+0000", 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0794e-05 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592940+0000", 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8803000000000001e-05 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.195 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96901 11.7 11:e6f829c8:::benchmark_data_smithi107_418363_object6465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 97.344808807999996, 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33668907599999998, 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.196 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96901 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:29:54.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711804+0000", 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711813+0000", 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:29:54.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711826+0000", 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3749e-05 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711836+0000", 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2690000000000005e-06 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727061+0000", 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015225851 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for rw locks", 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727140+0000", 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8693e-05 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751492+0000", 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024352087000000001 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751556+0000", 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3713999999999993e-05 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751643+0000", 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6861999999999996e-05 2023-10-09T06:29:54.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035341+0000", 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28369838200000003 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035383+0000", 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1430999999999997e-05 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.202 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045288+0000", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099051139999999996 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045323+0000", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4962000000000002e-05 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045372+0000", 2023-10-09T06:29:54.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9527000000000002e-05 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.048495+0000", 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031229059999999999 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.204 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96909 11.7 11:e4f104ee:::benchmark_data_smithi107_418363_object6473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 97.336988615999999, 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31569427700000002, 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96909 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.205 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719624+0000", 2023-10-09T06:29:54.207 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:29:54.207 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.207 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.207 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:29:54.208 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719635+0000", 2023-10-09T06:29:54.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0190000000000001e-05 2023-10-09T06:29:54.208 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.208 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.208 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719648+0000", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3749e-05 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719656+0000", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6860000000000006e-06 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727160+0000", 2023-10-09T06:29:54.209 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007503582 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727297+0000", 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013722100000000001 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727445+0000", 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000147851 2023-10-09T06:29:54.210 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.735152+0000", 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077073970000000004 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.737003+0000", 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001850955 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.738237+0000", 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012342570000000001 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.032138+0000", 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29390069400000002 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.032193+0000", 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5451000000000002e-05 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035321+0000", 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031271390000000001 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.213 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96914 11.7 11:ec492cfc:::benchmark_data_smithi107_418363_object6478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 97.329025568999995, 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31101062400000001, 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.214 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96914 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727588+0000", 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727595+0000", 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.793e-06 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727606+0000", 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1551e-05 2023-10-09T06:29:54.217 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727615+0000", 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.619e-06 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730424+0000", 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028088430000000001 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730560+0000", 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013612499999999999 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730671+0000", 2023-10-09T06:29:54.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011068 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741875+0000", 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011204529 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741912+0000", 2023-10-09T06:29:54.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6882999999999999e-05 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741920+0000", 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7700000000000001e-06 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035421+0000", 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.293500814 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035478+0000", 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7027999999999997e-05 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.038600+0000", 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031222509999999999 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.223 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:151315 11.7 11:e47e5293:::benchmark_data_smithi107_418363_object60879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:50.107896+0000", 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 3.9487184690000001, 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20638583199999999, 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.224 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 151315 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107896+0000", 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107896+0000", 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107895+0000", 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107899+0000", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3950000000000003e-06 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107909+0000", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1249999999999999e-06 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107915+0000", 2023-10-09T06:29:54.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3749999999999999e-06 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.107975+0000", 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0395e-05 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.108141+0000", 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000165926 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.108304+0000", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016229800000000001 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.115280+0000", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069762169999999998 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.121215+0000", 2023-10-09T06:29:54.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059347389999999996 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.300503+0000", 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.179288695 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.312204+0000", 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011701197999999999 2023-10-09T06:29:54.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.312256+0000", 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1907999999999998e-05 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:50.314282+0000", 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020258670000000002 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:152699 11.7 11:e083ba84:::benchmark_data_smithi107_418363_object62263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:52.370544+0000", 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 1.686070432, 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27686965600000002, 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 152699 2023-10-09T06:29:54.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370544+0000", 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370544+0000", 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370543+0000", 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370550+0000", 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5030000000000002e-06 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:29:54.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370562+0000", 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2768999999999999e-05 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.370571+0000", 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2679999999999994e-06 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.374878+0000", 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043077610000000002 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.235 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.375050+0000", 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017199500000000001 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.375217+0000", 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016656 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:29:54.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.551130+0000", 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.175913386 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.553077+0000", 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001946405 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.562880+0000", 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0098030089999999997 2023-10-09T06:29:54.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.645389+0000", 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.082509666999999995 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.645444+0000", 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4604000000000001e-05 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:29:54.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:52.647414+0000", 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019701190000000002 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:29:54.239 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:29:54.240 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:54.330 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:54.920 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:29:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:54 smithi107 ceph-mon[212750]: pgmap v1072: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13409/88704 objects degraded (15.117%); 5581/88704 objects misplaced (6.292%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:29:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:55.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:54 smithi107 ceph-mon[220921]: pgmap v1072: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13409/88704 objects degraded (15.117%); 5581/88704 objects misplaced (6.292%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:29:55.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:54 smithi150 ceph-mon[194178]: pgmap v1072: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 13409/88704 objects degraded (15.117%); 5581/88704 objects misplaced (6.292%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:29:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:29:55.883 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:29:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:55.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:29:55.988 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:56.030 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:56.101 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:56.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[212750]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:55 smithi107 ceph-mon[220921]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2737321442' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: osdmap e349: 8 total, 8 up, 8 in 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:55 smithi150 ceph-mon[194178]: failed to encode map e349 with expected crc 2023-10-09T06:29:56.209 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:29:56.236 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:56.736 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:29:56.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:56.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:29:56.787 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:29:56.848 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:56.962 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:56.998 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:56 smithi107 ceph-mon[220921]: pgmap v1074: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 13409/84135 objects degraded (15.937%); 5388/84135 objects misplaced (6.404%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:57.000 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:56 smithi107 ceph-mon[212750]: pgmap v1074: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 13409/84135 objects degraded (15.937%); 5388/84135 objects misplaced (6.404%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:57.117 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:57.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:56 smithi150 ceph-mon[194178]: pgmap v1074: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 13409/84135 objects degraded (15.937%); 5388/84135 objects misplaced (6.404%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:29:57.224 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:29:57.346 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13409/84135 objects degraded (15.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:58.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13409/84135 objects degraded (15.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13409/84135 objects degraded (15.937%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:29:58.649 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:29:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:58.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:29:58.771 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:58.808 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:58.848 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:29:58.892 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:58.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:58.939 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:29:58.959 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:29:59.002 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:59.025 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:29:59.077 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:29:59.092 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:59.130 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:29:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:58 smithi150 ceph-mon[194178]: pgmap v1075: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 13409/81633 objects degraded (16.426%); 5388/81633 objects misplaced (6.600%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:29:59.225 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:29:59.226 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:58 smithi107 ceph-mon[220921]: pgmap v1075: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 13409/81633 objects degraded (16.426%); 5388/81633 objects misplaced (6.600%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:29:59.226 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:58 smithi107 ceph-mon[212750]: pgmap v1075: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 13409/81633 objects degraded (16.426%); 5388/81633 objects misplaced (6.600%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T06:29:59.227 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:29:59.267 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:29:59.294 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:29:59.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:59.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:29:59.326 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:29:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.363 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.363 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.363 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:29:59.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:29:59.434 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:59.505 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:29:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:59.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:29:59.622 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:29:59.678 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:59.744 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:29:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:29:59.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:29:59.850 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:29:59.902 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:29:59.962 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:30:00.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1033922200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:00.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:29:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:30:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1033922200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:00.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:29:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:30:00.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1033922200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:29:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:30:00.829 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:00.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:00.939 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:00.978 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:00.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: pgmap v1076: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13414/74817 objects degraded (17.929%); 5212/74817 objects misplaced (6.966%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:30:00.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:30:00.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: failed to encode map e350 with expected crc 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: [WRN] PG_DEGRADED: Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: pg 11.5 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: pg 11.8 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:00.989 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[212750]: pg 11.a is stuck undersized for 106s, current state active+recovering+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: pgmap v1076: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13414/74817 objects degraded (17.929%); 5212/74817 objects misplaced (6.966%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: failed to encode map e350 with expected crc 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: failed to encode map e350 with expected crc 2023-10-09T06:30:00.990 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:00.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:30:00.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: [WRN] PG_DEGRADED: Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:00.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: pg 11.5 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-10-09T06:30:00.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: pg 11.8 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:00.991 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:00 smithi107 ceph-mon[220921]: pg 11.a is stuck undersized for 106s, current state active+recovering+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:01.085 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:01.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:01.197 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:01.274 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: pgmap v1076: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 13414/74817 objects degraded (17.929%); 5212/74817 objects misplaced (6.966%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/758440298' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: osdmap e350: 8 total, 8 up, 8 in 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: failed to encode map e350 with expected crc 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: failed to encode map e350 with expected crc 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:30:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: [WRN] PG_DEGRADED: Degraded data redundancy: 13414/74817 objects degraded (17.929%), 3 pgs degraded, 3 pgs undersized 2023-10-09T06:30:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: pg 11.5 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,6] 2023-10-09T06:30:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: pg 11.8 is stuck undersized for 106s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:00 smithi150 ceph-mon[194178]: pg 11.a is stuck undersized for 106s, current state active+recovering+undersized+degraded+remapped, last acting [3,6] 2023-10-09T06:30:02.186 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:02.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:02.205 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:02] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:30:02.206 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:02.206 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:02.305 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:02.354 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:30:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:02.440 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:02.482 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:02.684 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:02.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:02.796 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:02.870 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:03.026 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:03.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:03.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:02 smithi107 ceph-mon[212750]: pgmap v1078: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13414/72540 objects degraded (18.492%); 5212/72540 objects misplaced (7.185%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:03.042 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13414/72540 objects degraded (18.492%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:03.043 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:02 smithi107 ceph-mon[220921]: pgmap v1078: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13414/72540 objects degraded (18.492%); 5212/72540 objects misplaced (7.185%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:03.043 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13414/72540 objects degraded (18.492%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:03.135 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:03.148 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:30:03.149 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:30:03.221 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:02 smithi150 ceph-mon[194178]: pgmap v1078: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 13414/72540 objects degraded (18.492%); 5212/72540 objects misplaced (7.185%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13414/72540 objects degraded (18.492%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:03.603 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:30:04.949 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:04.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:05.057 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:05.096 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:05.111 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:05.155 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:05.156 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:30:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:05.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:05.262 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:05.269 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:04 smithi107 ceph-mon[220921]: pgmap v1079: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13414/70029 objects degraded (19.155%); 5212/70029 objects misplaced (7.443%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:30:05.270 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:04 smithi107 ceph-mon[212750]: pgmap v1079: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13414/70029 objects degraded (19.155%); 5212/70029 objects misplaced (7.443%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:30:05.297 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:04 smithi150 ceph-mon[194178]: pgmap v1079: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 13414/70029 objects degraded (19.155%); 5212/70029 objects misplaced (7.443%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:30:05.300 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:05.388 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:05.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:05.497 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:05.539 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:05.635 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:05.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:05.745 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:05.753 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:05.790 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:05.870 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:05.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:05.981 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:06.033 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:06.118 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:06.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:06.225 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:06.284 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:06.361 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:06.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:06.469 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:06.495 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:06.569 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:06.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:06.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:06.815 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:07.129 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:30:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[212750]: pgmap v1080: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 13414/65337 objects degraded (20.530%); 5013/65337 objects misplaced (7.673%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:30:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2522101919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:07.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[220921]: pgmap v1080: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 13414/65337 objects degraded (20.530%); 5013/65337 objects misplaced (7.673%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:30:07.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2522101919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:07.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:06 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:06 smithi150 ceph-mon[194178]: pgmap v1080: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 13414/65337 objects degraded (20.530%); 5013/65337 objects misplaced (7.673%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T06:30:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2522101919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:06 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:08.132 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:08.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:08.241 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:08.294 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:08 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13414/65337 objects degraded (20.530%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:08.298 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:08.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:30:08.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:08.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:30:08.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13414/65337 objects degraded (20.530%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: osdmap e351: 8 total, 8 up, 8 in 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: failed to encode map e351 with expected crc 2023-10-09T06:30:08.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:08 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13414/65337 objects degraded (20.530%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:08.401 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:08.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:08.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:08.506 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:08.570 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:09.145 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:09.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:09.176 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:09.261 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:09.302 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:09 smithi107 ceph-mon[212750]: pgmap v1082: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 13414/63036 objects degraded (21.280%); 5013/63036 objects misplaced (7.953%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:30:09.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:09 smithi107 ceph-mon[220921]: pgmap v1082: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 13414/63036 objects degraded (21.280%); 5013/63036 objects misplaced (7.953%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:30:09.304 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:09 smithi150 ceph-mon[194178]: pgmap v1082: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 13414/63036 objects degraded (21.280%); 5013/63036 objects misplaced (7.953%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T06:30:09.307 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:09.381 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:09.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:09.504 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:09.620 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:09.739 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:09.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:09.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:09.846 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:09.981 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:10.131 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:10.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:10.190 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:30:10.231 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:30:10.237 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:10.263 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:30:10.277 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:10.301 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:30:10.341 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:30:10.351 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:10.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:10.379 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:30:10.430 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:30:10.469 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:10.478 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:30:10.497 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.497 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.498 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:30:10.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:10.570 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:10.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:10.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:10.673 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:10.699 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:10.741 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:10.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:10.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:10.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:10.904 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:11.053 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:30:11.132 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:30:11.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[220921]: pgmap v1083: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13418/56034 objects degraded (23.946%); 4819/56034 objects misplaced (8.600%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:30:11.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2691241554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:11.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2238106053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[212750]: pgmap v1083: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13418/56034 objects degraded (23.946%); 4819/56034 objects misplaced (8.600%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:30:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2691241554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2238106053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:11 smithi150 ceph-mon[194178]: pgmap v1083: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13418/56034 objects degraded (23.946%); 4819/56034 objects misplaced (8.600%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:30:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2691241554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2238106053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:30:11.829 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:11.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:11.942 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:12.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:12.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:12] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[212750]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[212750]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[220921]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:30:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[220921]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:30:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[220921]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:12 smithi107 ceph-mon[220921]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:12 smithi150 ceph-mon[194178]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:30:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:12 smithi150 ceph-mon[194178]: osdmap e352: 8 total, 8 up, 8 in 2023-10-09T06:30:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:12 smithi150 ceph-mon[194178]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:12 smithi150 ceph-mon[194178]: failed to encode map e352 with expected crc 2023-10-09T06:30:12.780 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:12.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:12.888 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:30:12.897 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:13.031 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:13.168 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:13.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:13.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:13 smithi107 ceph-mon[212750]: pgmap v1085: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 13418/53682 objects degraded (24.995%); 4819/53682 objects misplaced (8.977%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:30:13.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:13 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/53682 objects degraded (24.995%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:13.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:13 smithi107 ceph-mon[220921]: pgmap v1085: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 13418/53682 objects degraded (24.995%); 4819/53682 objects misplaced (8.977%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:30:13.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:13 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/53682 objects degraded (24.995%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:13.277 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:13.345 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:13 smithi150 ceph-mon[194178]: pgmap v1085: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 13418/53682 objects degraded (24.995%); 4819/53682 objects misplaced (8.977%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T06:30:13.345 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:13 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/53682 objects degraded (24.995%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:13.348 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:13.431 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:13.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:13.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:13.502 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:13.537 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:30:13.538 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:30:13.538 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:30:13.538 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:30:13.539 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:30:13.539 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:13.632 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:13.662 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:13.818 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:13.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:13.922 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:13.991 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:14.082 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:14 smithi107 ceph-mon[212750]: pgmap v1086: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13418/51300 objects degraded (26.156%); 4819/51300 objects misplaced (9.394%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:30:14.084 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:14 smithi107 ceph-mon[220921]: pgmap v1086: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13418/51300 objects degraded (26.156%); 4819/51300 objects misplaced (9.394%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:30:14.095 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:14.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:14.202 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:14.230 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:14.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:14 smithi150 ceph-mon[194178]: pgmap v1086: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 13418/51300 objects degraded (26.156%); 4819/51300 objects misplaced (9.394%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:30:14.667 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:14.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:14.775 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:14.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:30:14.847 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:30:14.881 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:15.035 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:15.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:15.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:15.152 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:15.306 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:16 smithi107 ceph-mon[220921]: pgmap v1087: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 973 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 13418/46734 objects degraded (28.711%); 4633/46734 objects misplaced (9.914%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:16 smithi107 ceph-mon[212750]: pgmap v1087: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 973 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 13418/46734 objects degraded (28.711%); 4633/46734 objects misplaced (9.914%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:16.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:16 smithi150 ceph-mon[194178]: pgmap v1087: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 973 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 13418/46734 objects degraded (28.711%); 4633/46734 objects misplaced (9.914%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T06:30:16.868 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:16.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:16.974 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:17.042 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:17.285 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:17.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:17.422 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:17.466 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:17.487 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:17.502 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:17.608 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:17.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:17.718 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:17.734 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:17.734 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:17.734 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/46734 objects degraded (28.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:17.735 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:17.735 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:17.735 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/46734 objects degraded (28.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:17.819 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/46734 objects degraded (28.711%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:18.301 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:18.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:18.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:18.411 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:18.445 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:18.526 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:18.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:18.638 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:18.650 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:30:18.666 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:30:18.723 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[212750]: pgmap v1088: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 924 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13418/44388 objects degraded (30.229%); 4633/44388 objects misplaced (10.438%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:30:18.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4197833509' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:18.724 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[220921]: pgmap v1088: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 924 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13418/44388 objects degraded (30.229%); 4633/44388 objects misplaced (10.438%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:30:18.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4197833509' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:18.725 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:18 smithi150 ceph-mon[194178]: pgmap v1088: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 924 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 13418/44388 objects degraded (30.229%); 4633/44388 objects misplaced (10.438%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:30:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4197833509' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:18 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:30:18.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[220921]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:19 smithi107 ceph-mon[212750]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/556451980' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: osdmap e353: 8 total, 8 up, 8 in 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: failed to encode map e353 with expected crc 2023-10-09T06:30:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:19 smithi150 ceph-mon[194178]: failed to encode map e353 with expected crc 2023-10-09T06:30:21.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:20 smithi107 ceph-mon[220921]: pgmap v1090: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 778 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 13418/37386 objects degraded (35.890%); 4434/37386 objects misplaced (11.860%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:21.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:20 smithi107 ceph-mon[212750]: pgmap v1090: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 778 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 13418/37386 objects degraded (35.890%); 4434/37386 objects misplaced (11.860%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:21.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:20 smithi150 ceph-mon[194178]: pgmap v1090: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 778 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 13418/37386 objects degraded (35.890%); 4434/37386 objects misplaced (11.860%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:21.349 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:21.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:21.456 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:21.523 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:21.959 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:30:21.974 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:21.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:22.054 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:22] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T06:30:22.092 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:22.134 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:22.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:22.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:22.312 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:22.391 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:22.703 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:22.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:22.807 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:30:22.814 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:22.854 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:30:22.875 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:22.920 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:30:22.962 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:30:23.018 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:30:23.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:23.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:22 smithi107 ceph-mon[212750]: pgmap v1091: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 732 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13418/35178 objects degraded (38.143%); 4434/35178 objects misplaced (12.604%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:23.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/35178 objects degraded (38.143%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:23.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:22 smithi107 ceph-mon[220921]: pgmap v1091: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 732 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13418/35178 objects degraded (38.143%); 4434/35178 objects misplaced (12.604%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:23.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/35178 objects degraded (38.143%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:23.052 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:30:23.053 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:30:23.072 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:30:23.098 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:23.127 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:30:23.129 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:22 smithi150 ceph-mon[194178]: pgmap v1091: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 732 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 13418/35178 objects degraded (38.143%); 4434/35178 objects misplaced (12.604%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/35178 objects degraded (38.143%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:23.181 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:30:23.183 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.199 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:30:23.251 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:23.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:23.356 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:23.421 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:24.019 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:30:24.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/10183644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:24.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/10183644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:24.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/10183644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:24.640 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:24.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:24.745 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:24.761 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:30:24.789 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:24.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:24.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:24.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:24 smithi107 ceph-mon[212750]: pgmap v1092: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 680 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13418/32697 objects degraded (41.037%); 4434/32697 objects misplaced (13.561%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:24.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:24 smithi107 ceph-mon[220921]: pgmap v1092: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 680 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13418/32697 objects degraded (41.037%); 4434/32697 objects misplaced (13.561%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:24.993 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:25.111 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:25.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:24 smithi150 ceph-mon[194178]: pgmap v1092: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 680 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 13418/32697 objects degraded (41.037%); 4434/32697 objects misplaced (13.561%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:25.413 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:25.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:25.519 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:25.584 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:25.921 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:25.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:26.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:26.099 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:26.220 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:26.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:26.335 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:26.366 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:26.449 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:26.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:26.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:26.552 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:26.623 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:26.642 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:26.653 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:26.705 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:26.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:26.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:26.824 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:26.883 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:26 smithi150 ceph-mon[194178]: pgmap v1093: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 579 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13418/27861 objects degraded (48.161%); 4223/27861 objects misplaced (15.157%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:26.887 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:26 smithi107 ceph-mon[212750]: pgmap v1093: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 579 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13418/27861 objects degraded (48.161%); 4223/27861 objects misplaced (15.157%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:26.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:26 smithi107 ceph-mon[220921]: pgmap v1093: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 579 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 13418/27861 objects degraded (48.161%); 4223/27861 objects misplaced (15.157%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:26.945 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:26.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:27.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:27.099 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:27.199 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:27.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:27.309 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:27.416 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:27.536 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:30:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/27861 objects degraded (48.161%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:27.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/27861 objects degraded (48.161%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/27861 objects degraded (48.161%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:28.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:28 smithi107 ceph-mon[220921]: pgmap v1094: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 526 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13418/25284 objects degraded (53.069%); 4223/25284 objects misplaced (16.702%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:28.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:28 smithi107 ceph-mon[212750]: pgmap v1094: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 526 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13418/25284 objects degraded (53.069%); 4223/25284 objects misplaced (16.702%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:28 smithi150 ceph-mon[194178]: pgmap v1094: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 526 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13418/25284 objects degraded (53.069%); 4223/25284 objects misplaced (16.702%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T06:30:29.503 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:29.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:29.619 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:29.740 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:30.037 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:29 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:30.041 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:29 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:29 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:30:30.166 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:30.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:30.289 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:30.297 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:30.337 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:30.405 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:30.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:30.476 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:30:30.476 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:30:30.511 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:30.565 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:30.646 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:30.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:30.759 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:30.800 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:30:30.801 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:30.814 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:30:30.893 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:30.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:30.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:31.003 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:31.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:30 smithi150 ceph-mon[194178]: pgmap v1095: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 431 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13418/20757 objects degraded (64.643%); 4223/20757 objects misplaced (20.345%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:31.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/310975742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:30 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1761197392' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:30 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.035 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:31.139 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:31.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[220921]: pgmap v1095: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 431 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13418/20757 objects degraded (64.643%); 4223/20757 objects misplaced (20.345%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/310975742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1761197392' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[212750]: pgmap v1095: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 431 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13418/20757 objects degraded (64.643%); 4223/20757 objects misplaced (20.345%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/310975742' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:30:31.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1761197392' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:30 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:31.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:31.254 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:31.298 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:31.425 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:31.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:31.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:31.562 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:31.611 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:31.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:31.721 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:31.792 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: failed to encode map e354 with expected crc 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: failed to encode map e354 with expected crc 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: failed to encode map e354 with expected crc 2023-10-09T06:30:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: failed to encode map e354 with expected crc 2023-10-09T06:30:31.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:31.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: osdmap e354: 8 total, 8 up, 8 in 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: failed to encode map e354 with expected crc 2023-10-09T06:30:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:32.153 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:32] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:30:32.191 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:32.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:32.296 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:32.322 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:32.809 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:32 smithi107 ceph-mon[220921]: pgmap v1097: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 315 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 13418/15180 objects degraded (88.393%); 3972/15180 objects misplaced (26.166%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:32.809 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/15180 objects degraded (88.393%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:32.988 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:32.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:33.095 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:33.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:32 smithi107 ceph-mon[212750]: pgmap v1097: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 315 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 13418/15180 objects degraded (88.393%); 3972/15180 objects misplaced (26.166%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/15180 objects degraded (88.393%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:33.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:32 smithi150 ceph-mon[194178]: pgmap v1097: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 315 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 13418/15180 objects degraded (88.393%); 3972/15180 objects misplaced (26.166%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:33.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/15180 objects degraded (88.393%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:33.186 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:33.872 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:30:34.017 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:34.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:34.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:34.106 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:30:34.122 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:34.269 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:34.471 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:34.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:34.581 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.662 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:34.773 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:34.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:34.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:34.886 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:34.933 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:35.022 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:35.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:35.101 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:35.135 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:35.148 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:35.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:34 smithi107 ceph-mon[212750]: pgmap v1098: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 269 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 13418/12978 objects degraded (103.390%); 3972/12978 objects misplaced (30.606%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:35.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:34 smithi107 ceph-mon[220921]: pgmap v1098: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 269 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 13418/12978 objects degraded (103.390%); 3972/12978 objects misplaced (30.606%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:35.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:34 smithi150 ceph-mon[194178]: pgmap v1098: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 269 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 13418/12978 objects degraded (103.390%); 3972/12978 objects misplaced (30.606%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:30:35.168 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:35.224 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:35.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:35.300 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:30:35.331 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:35.335 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:30:35.368 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:30:35.406 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:35.421 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:30:35.470 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:30:35.511 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:30:35.550 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:35.563 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:30:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:35.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:35.599 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.603 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.604 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:30:35.672 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:35.719 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:35.767 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:30:35.803 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:35.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:35.914 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:36.027 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2986980186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:36.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[212750]: pgmap v1099: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 182 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 13418/8802 objects degraded (152.443%); 3764/8802 objects misplaced (42.763%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2986980186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:36.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:36 smithi107 ceph-mon[220921]: pgmap v1099: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 182 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 13418/8802 objects degraded (152.443%); 3764/8802 objects misplaced (42.763%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:36.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:36 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:36.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2986980186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:36.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:36 smithi150 ceph-mon[194178]: pgmap v1099: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 182 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 13418/8802 objects degraded (152.443%); 3764/8802 objects misplaced (42.763%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:37.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13418/8802 objects degraded (152.443%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13418/8802 objects degraded (152.443%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13418/8802 objects degraded (152.443%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:37.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 90433 objects 2023-10-09T06:30:37.893 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :152.07 2023-10-09T06:30:38.081 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:38.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:38.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:38.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:38.284 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:38.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:38 smithi107 ceph-mon[212750]: pgmap v1100: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 136 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 13418/6579 objects degraded (203.952%); 3764/6579 objects misplaced (57.212%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:38.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:38 smithi107 ceph-mon[220921]: pgmap v1100: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 136 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 13418/6579 objects degraded (203.952%); 3764/6579 objects misplaced (57.212%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:38 smithi150 ceph-mon[194178]: pgmap v1100: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 136 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 13418/6579 objects degraded (203.952%); 3764/6579 objects misplaced (57.212%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:38.903 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:38.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:39.007 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:39.047 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:39.048 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:39.163 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:39.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:39.268 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:39.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:39.403 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:39.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:39.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:39.507 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:39.539 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:39.546 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:30:39.546 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:30:39.546 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:30:39.546 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:30:39.547 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:30:39.762 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-10-09T06:30:39.763 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-10-09T06:30:39.768 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:39.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:39.872 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:39.901 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:40.036 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:30:40.396 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:40.397 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[220921]: pgmap v1101: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 60 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13418/2901 objects degraded (462.530%); 3764/2901 objects misplaced (129.748%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:40.397 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3819855737' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:40.398 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:40.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:40.410 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:30:40.504 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:40.534 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:40.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[212750]: pgmap v1101: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 60 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13418/2901 objects degraded (462.530%); 3764/2901 objects misplaced (129.748%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3819855737' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:40 smithi150 ceph-mon[194178]: pgmap v1101: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 60 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 13418/2901 objects degraded (462.530%); 3764/2901 objects misplaced (129.748%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:30:40.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3819855737' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:40.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:30:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:41 smithi150 ceph-mon[194178]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:41 smithi150 ceph-mon[194178]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:30:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:41 smithi150 ceph-mon[194178]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:30:41.762 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[212750]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.762 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[212750]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.762 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:30:41.762 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[212750]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:30:41.762 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[220921]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.763 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[220921]: failed to encode map e355 with expected crc 2023-10-09T06:30:41.763 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:30:41.763 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:41 smithi107 ceph-mon[220921]: osdmap e355: 8 total, 8 up, 8 in 2023-10-09T06:30:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:42] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:30:42.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:42 smithi107 ceph-mon[212750]: pgmap v1103: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13186/201 objects degraded (6560.199%); 3757/201 objects misplaced (1869.154%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:42.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 13186/201 objects degraded (6560.199%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:42.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:42 smithi107 ceph-mon[220921]: pgmap v1103: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13186/201 objects degraded (6560.199%); 3757/201 objects misplaced (1869.154%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:42.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 13186/201 objects degraded (6560.199%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:42 smithi150 ceph-mon[194178]: pgmap v1103: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13186/201 objects degraded (6560.199%); 3757/201 objects misplaced (1869.154%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:42.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 13186/201 objects degraded (6560.199%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:43.121 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:30:43.209 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:43.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:43.312 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:43.339 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:43.381 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:43.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:43.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:43.489 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:43.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:43.761 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:43.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:43.865 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:43.883 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:30:43.892 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:43.936 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:43.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:43.940 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:44.017 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:44.040 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:44.074 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:44.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:44 smithi107 ceph-mon[212750]: pgmap v1104: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 13186/72 objects degraded (18313.889%); 3757/72 objects misplaced (5218.056%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:44.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:44 smithi107 ceph-mon[220921]: pgmap v1104: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 13186/72 objects degraded (18313.889%); 3757/72 objects misplaced (5218.056%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:44.673 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:44.679 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:44.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:44.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:44.827 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:44.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:44 smithi150 ceph-mon[194178]: pgmap v1104: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 13186/72 objects degraded (18313.889%); 3757/72 objects misplaced (5218.056%); 1.3 MiB/s, 69 objects/s recovering 2023-10-09T06:30:45.294 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:45.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:45.397 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:45.424 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:30:45.426 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:45.452 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[220921]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[220921]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[220921]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[220921]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[212750]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:30:45.453 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.454 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[212750]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.454 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:30:45.471 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:45.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:45.577 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:45.604 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:45.646 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:45.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:45.752 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:45.775 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:45 smithi150 ceph-mon[194178]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.776 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:45 smithi150 ceph-mon[194178]: osdmap e356: 8 total, 8 up, 8 in 2023-10-09T06:30:45.776 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:45 smithi150 ceph-mon[194178]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.776 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:45 smithi150 ceph-mon[194178]: failed to encode map e356 with expected crc 2023-10-09T06:30:45.776 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:30:45.778 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:45.820 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:45.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:45.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:45.925 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:45.951 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:45.993 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:45.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:46.098 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:46.128 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:46.793 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: pgmap v1106: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 271 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 56 KiB/s, 284 objects/s recovering 2023-10-09T06:30:46.793 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:46.793 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[212750]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: pgmap v1106: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 271 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 56 KiB/s, 284 objects/s recovering 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:30:46.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:46 smithi107 ceph-mon[220921]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:30:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: pgmap v1106: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 271 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 56 KiB/s, 284 objects/s recovering 2023-10-09T06:30:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:30:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: failed to encode map e357 with expected crc 2023-10-09T06:30:46.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3622973438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:30:46.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:46 smithi150 ceph-mon[194178]: osdmap e357: 8 total, 8 up, 8 in 2023-10-09T06:30:47.471 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:47.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:47.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:47.576 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:47.604 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[212750]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:47.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 11171/72 objects degraded (15515.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:47.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[220921]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[220921]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:47.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 11171/72 objects degraded (15515.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:47 smithi150 ceph-mon[194178]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:47 smithi150 ceph-mon[194178]: failed to encode map e357 with expected crc 2023-10-09T06:30:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:30:47.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 11171/72 objects degraded (15515.278%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:48.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:48 smithi107 ceph-mon[212750]: pgmap v1108: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 260 objects/s recovering 2023-10-09T06:30:48.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:48 smithi107 ceph-mon[220921]: pgmap v1108: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 260 objects/s recovering 2023-10-09T06:30:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:48 smithi150 ceph-mon[194178]: pgmap v1108: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 44 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 260 objects/s recovering 2023-10-09T06:30:49.601 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:49.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:49.706 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:49.741 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:50 smithi107 ceph-mon[212750]: pgmap v1109: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 224 objects/s recovering 2023-10-09T06:30:50.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:50 smithi107 ceph-mon[220921]: pgmap v1109: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 224 objects/s recovering 2023-10-09T06:30:50.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:50 smithi150 ceph-mon[194178]: pgmap v1109: 99 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 11171/72 objects degraded (15515.278%); 3757/72 objects misplaced (5218.056%); 0 B/s, 224 objects/s recovering 2023-10-09T06:30:51.714 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:30:51.753 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:51.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:51.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:51.888 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:51.944 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:30:51.968 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:30:51.994 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:30:52.023 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:30:52.053 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:52.054 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:30:52.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:52.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:52.084 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:30:52.116 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:30:52.136 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:30:52.137 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.137 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.137 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.137 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.138 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.138 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.138 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.138 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:30:52.158 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:52.166 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:52.202 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:52.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:30:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:30:52] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:30:52.606 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:52.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:52.712 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:52.737 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:52 smithi150 ceph-mon[194178]: pgmap v1110: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 294 objects/s recovering 2023-10-09T06:30:52.737 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/436516615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:52.738 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.738 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.738 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:52.740 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:52.785 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:52.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:52.890 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:52.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[212750]: pgmap v1110: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 294 objects/s recovering 2023-10-09T06:30:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/436516615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[220921]: pgmap v1110: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 294 objects/s recovering 2023-10-09T06:30:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/436516615' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:30:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:52.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:30:52.919 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:52.968 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:52.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:53.072 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:53.101 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:53.145 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:53.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:53.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:53.279 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:53.321 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:53.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:53.425 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:53.467 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:53.510 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_6' removed 2023-10-09T06:30:53.549 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4156256256' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.549 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.550 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:30:53.550 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:30:53.550 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:53.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4156256256' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:30:53.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:30:53.552 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:53.563 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:53.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:53.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:53.670 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:53.697 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:53.740 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:53.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:53.844 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:53.868 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4156256256' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.868 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:30:53.868 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:30:53.868 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:30:53.868 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:30:53.870 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:53.916 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:53.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:54.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:54.059 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:54.212 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:54.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:54.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:54.299 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:30:54.299 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:30:54.320 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:54.348 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:54.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[212750]: pgmap v1111: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 264 objects/s recovering 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[212750]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[220921]: pgmap v1111: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 264 objects/s recovering 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[220921]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:30:54.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:54 smithi107 ceph-mon[220921]: failed to encode map e358 with expected crc 2023-10-09T06:30:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:54 smithi150 ceph-mon[194178]: pgmap v1111: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10831/72 objects degraded (15043.056%); 3757/72 objects misplaced (5218.056%); 0 B/s, 264 objects/s recovering 2023-10-09T06:30:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:30:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:54 smithi150 ceph-mon[194178]: osdmap e358: 8 total, 8 up, 8 in 2023-10-09T06:30:54.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:54 smithi150 ceph-mon[194178]: failed to encode map e358 with expected crc 2023-10-09T06:30:55.688 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:55.713 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:55 smithi107 ceph-mon[212750]: failed to encode map e358 with expected crc 2023-10-09T06:30:55.713 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:55 smithi107 ceph-mon[220921]: failed to encode map e358 with expected crc 2023-10-09T06:30:55.714 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:30:55.715 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:30:55.715 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-10-09T06:30:55.715 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-10-09T06:30:55.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:55.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:55 smithi150 ceph-mon[194178]: failed to encode map e358 with expected crc 2023-10-09T06:30:55.958 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:55.988 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:56.431 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:56.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:56.534 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:56.568 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:56.617 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:56.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:56.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:56.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:56 smithi107 ceph-mon[212750]: pgmap v1113: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:56 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:30:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:56 smithi107 ceph-mon[220921]: pgmap v1113: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:56 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:30:56.723 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:56.751 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:56.844 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:56.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:56 smithi150 ceph-mon[194178]: pgmap v1113: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:56 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10831/72 objects degraded (15043.056%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T06:30:56.918 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:30:56.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:30:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:57.023 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:30:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:57.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:30:57.130 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:30:57.157 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:57.198 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:30:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:57.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:30:57.306 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:30:57.331 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:57.373 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:30:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:57.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:30:57.479 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:57.506 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:57.551 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:30:57.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:57.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:30:57.657 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:57.665 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:30:57.690 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:57.697 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:57.904 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:30:57.994 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:57.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:58.098 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:30:58.138 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:58.273 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:30:58.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:58.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:30:58.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:30:58.413 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:58.465 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:30:58.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[212750]: pgmap v1114: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/622893024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[220921]: pgmap v1114: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/622893024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:58 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:58 smithi150 ceph-mon[194178]: pgmap v1114: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:30:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/622893024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:58 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:30:59.109 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:30:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:59.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:30:59.214 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:30:59.243 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:59.339 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:30:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:59.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:30:59.443 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.474 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:59.477 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:30:59.521 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:59.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:59.627 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:30:59.654 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:59.697 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:30:59.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:30:59.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:30:59.804 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:30:59.832 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[212750]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[212750]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[212750]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[220921]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[220921]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:30:59.833 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[220921]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:30:59.834 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:30:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:30:59.834 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:30:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:59 smithi150 ceph-mon[194178]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:59 smithi150 ceph-mon[194178]: failed to encode map e359 with expected crc 2023-10-09T06:30:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:30:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:59 smithi150 ceph-mon[194178]: osdmap e359: 8 total, 8 up, 8 in 2023-10-09T06:30:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:30:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:00.116 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:31:00.116 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:31:00.280 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:00.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:00.384 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:00.415 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:00.459 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:00.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:00.566 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:00.593 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:00.723 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: failed to encode map e359 with expected crc 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: pgmap v1116: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:31:00.724 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: failed to encode map e359 with expected crc 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: pgmap v1116: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:31:00.725 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.726 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:00.726 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:31:00.726 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.726 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:00 smithi107 ceph-mon[220921]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.740 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:00.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:00.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:00.870 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: failed to encode map e359 with expected crc 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: pgmap v1116: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1195030454' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: osdmap e360: 8 total, 8 up, 8 in 2023-10-09T06:31:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: failed to encode map e360 with expected crc 2023-10-09T06:31:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:00 smithi150 ceph-mon[194178]: failed to encode map e360 with expected crc 2023-10-09T06:31:01.003 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:01.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:01.036 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:01.107 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:01.212 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:01.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:01.243 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:01.317 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:01.353 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:01.555 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:01.555 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:01.705 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:01.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:01.759 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_7' created 2023-10-09T06:31:01.807 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:01.834 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:01 smithi107 ceph-mon[212750]: failed to encode map e360 with expected crc 2023-10-09T06:31:01.835 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3776839377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:01.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:01 smithi107 ceph-mon[220921]: failed to encode map e360 with expected crc 2023-10-09T06:31:01.835 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3776839377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:01.839 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:02.116 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:02] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.43.0" 2023-10-09T06:31:03.183 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:01 smithi150 ceph-mon[194178]: failed to encode map e360 with expected crc 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3776839377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: pgmap v1118: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:03.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T06:31:03.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:31:03.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.187 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:31:03.188 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:03.188 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: pgmap v1118: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:03.188 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:31:03.189 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.190 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: pgmap v1118: 83 pgs: 83 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:03.190 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T06:31:03.190 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.190 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:03.191 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.191 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: failed to encode map e361 with expected crc 2023-10-09T06:31:03.191 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T06:31:03.191 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: osdmap e361: 8 total, 8 up, 8 in 2023-10-09T06:31:03.191 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:03.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:03.195 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-10-09T06:31:03.294 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:03.326 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:03.378 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:03.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:03.483 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:03.513 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:04.073 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:04.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[212750]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:31:04.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.077 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[212750]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[220921]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:31:04.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[220921]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[220921]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[220921]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:03 smithi107 ceph-mon[220921]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:04.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:03 smithi150 ceph-mon[194178]: osdmap e362: 8 total, 8 up, 8 in 2023-10-09T06:31:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:03 smithi150 ceph-mon[194178]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:03 smithi150 ceph-mon[194178]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:03 smithi150 ceph-mon[194178]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:03 smithi150 ceph-mon[194178]: failed to encode map e362 with expected crc 2023-10-09T06:31:04.167 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:04.180 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:04.221 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:04.269 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:04.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:04.378 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:04.411 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:04.416 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:04.461 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:04.463 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:04.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:04.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:04.482 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:04.506 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:04.532 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:04.552 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:04.569 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:04.572 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:04.594 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:04.597 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:04.614 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:04.617 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.618 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.618 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.618 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.618 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.618 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.619 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.619 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:31:04.642 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:04.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:04.714 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:04.748 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: pgmap v1121: 99 pgs: 2 creating+peering, 12 unknown, 85 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.757 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.758 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2805263891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:04.782 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: pgmap v1121: 99 pgs: 2 creating+peering, 12 unknown, 85 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: failed to encode map e363 with expected crc 2023-10-09T06:31:04.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2805263891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:04.982 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:31:04.984 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:31:05.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: pgmap v1121: 99 pgs: 2 creating+peering, 12 unknown, 85 active+clean; 582 KiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:05.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: osdmap e363: 8 total, 8 up, 8 in 2023-10-09T06:31:05.010 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:31:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:31:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:31:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: failed to encode map e363 with expected crc 2023-10-09T06:31:05.011 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2805263891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:05.329 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:05.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:05.435 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:05.467 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:05.517 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:05.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:05.625 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:05.656 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:06.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:05 smithi107 ceph-mon[212750]: from='client.36408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:06.001 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:05 smithi107 ceph-mon[220921]: from='client.36408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:06.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:06.028 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1122,"stamp":"2023-10-09T06:31:03.753982+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":600556,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":264,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":259,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7082048,"kb_used_data":1020288,"kb_used_omap":20,"kb_used_meta":6061675,"kb_avail":742879168,"statfs":{"total":767960285184,"available":760708268032,"internally_reserved":0,"allocated":1044774912,"data_stored":1034253390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6207155854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.443383"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752428+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:31:01.752428+0000","last_peered":"2023-10-09T06:31:01.752428+0000","last_clean":"2023-10-09T06:31:01.752428+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:31:01.752428+0000","last_undegraded":"2023-10-09T06:31:01.752428+0000","last_fullsized":"2023-10-09T06:31:01.752428+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.415158+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:30:54.415158+0000","last_peered":"2023-10-09T06:30:54.415158+0000","last_clean":"2023-10-09T06:30:54.415158+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:30:54.415158+0000","last_undegraded":"2023-10-09T06:30:54.415158+0000","last_fullsized":"2023-10-09T06:30:54.415158+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":353,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472374+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:30:59.472374+0000","last_peered":"2023-10-09T06:30:59.472374+0000","last_clean":"2023-10-09T06:30:59.472374+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:30:59.472374+0000","last_undegraded":"2023-10-09T06:30:59.472374+0000","last_fullsized":"2023-10-09T06:30:59.472374+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":356,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460424+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:30:58.460424+0000","last_peered":"2023-10-09T06:30:58.460424+0000","last_clean":"2023-10-09T06:30:58.460424+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:30:58.460424+0000","last_undegraded":"2023-10-09T06:30:58.460424+0000","last_fullsized":"2023-10-09T06:30:58.460424+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":75,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412417+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:30:54.412417+0000","last_peered":"2023-10-09T06:30:54.412417+0000","last_clean":"2023-10-09T06:30:54.412417+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:30:54.412417+0000","last_undegraded":"2023-10-09T06:30:54.412417+0000","last_fullsized":"2023-10-09T06:30:54.412417+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":349,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.971139+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:30:58.971139+0000","last_peered":"2023-10-09T06:30:58.971139+0000","last_clean":"2023-10-09T06:30:58.971139+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:30:58.971139+0000","last_undegraded":"2023-10-09T06:30:58.971139+0000","last_fullsized":"2023-10-09T06:30:58.971139+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":31,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460375+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:30:58.460375+0000","last_peered":"2023-10-09T06:30:58.460375+0000","last_clean":"2023-10-09T06:30:58.460375+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:30:58.460375+0000","last_undegraded":"2023-10-09T06:30:58.460375+0000","last_fullsized":"2023-10-09T06:30:58.460375+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.413155+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:30:54.413155+0000","last_peered":"2023-10-09T06:30:54.413155+0000","last_clean":"2023-10-09T06:30:54.413155+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:30:54.413155+0000","last_undegraded":"2023-10-09T06:30:54.413155+0000","last_fullsized":"2023-10-09T06:30:54.413155+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754430+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:31:01.754430+0000","last_peered":"2023-10-09T06:31:01.754430+0000","last_clean":"2023-10-09T06:31:01.754430+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:31:01.754430+0000","last_undegraded":"2023-10-09T06:31:01.754430+0000","last_fullsized":"2023-10-09T06:31:01.754430+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412707+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:30:54.412707+0000","last_peered":"2023-10-09T06:30:54.412707+0000","last_clean":"2023-10-09T06:30:54.412707+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:30:54.412707+0000","last_undegraded":"2023-10-09T06:30:54.412707+0000","last_fullsized":"2023-10-09T06:30:54.412707+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":243,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.756192+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:31:01.756192+0000","last_peered":"2023-10-09T06:31:01.756192+0000","last_clean":"2023-10-09T06:31:01.756192+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:31:01.756192+0000","last_undegraded":"2023-10-09T06:31:01.756192+0000","last_fullsized":"2023-10-09T06:31:01.756192+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":294,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.473160+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:30:59.473160+0000","last_peered":"2023-10-09T06:30:59.473160+0000","last_clean":"2023-10-09T06:30:59.473160+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:30:59.473160+0000","last_undegraded":"2023-10-09T06:30:59.473160+0000","last_fullsized":"2023-10-09T06:30:59.473160+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":368,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752700+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:31:01.752700+0000","last_peered":"2023-10-09T06:31:01.752700+0000","last_clean":"2023-10-09T06:31:01.752700+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:31:01.752700+0000","last_undegraded":"2023-10-09T06:31:01.752700+0000","last_fullsized":"2023-10-09T06:31:01.752700+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":332,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752896+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:31:01.752896+0000","last_peered":"2023-10-09T06:31:01.752896+0000","last_clean":"2023-10-09T06:31:01.752896+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:31:01.752896+0000","last_undegraded":"2023-10-09T06:31:01.752896+0000","last_fullsized":"2023-10-09T06:31:01.752896+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":211,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.756023+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:31:01.756023+0000","last_peered":"2023-10-09T06:31:01.756023+0000","last_clean":"2023-10-09T06:31:01.756023+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:31:01.756023+0000","last_undegraded":"2023-10-09T06:31:01.756023+0000","last_fullsized":"2023-10-09T06:31:01.756023+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.415891+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:30:54.415891+0000","last_peered":"2023-10-09T06:30:54.415891+0000","last_clean":"2023-10-09T06:30:54.415891+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:30:54.415891+0000","last_undegraded":"2023-10-09T06:30:54.415891+0000","last_fullsized":"2023-10-09T06:30:54.415891+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":354,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460660+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:30:58.460660+0000","last_peered":"2023-10-09T06:30:58.460660+0000","last_clean":"2023-10-09T06:30:58.460660+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:30:58.460660+0000","last_undegraded":"2023-10-09T06:30:58.460660+0000","last_fullsized":"2023-10-09T06:30:58.460660+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754737+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:31:01.754737+0000","last_peered":"2023-10-09T06:31:01.754737+0000","last_clean":"2023-10-09T06:31:01.754737+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:31:01.754737+0000","last_undegraded":"2023-10-09T06:31:01.754737+0000","last_fullsized":"2023-10-09T06:31:01.754737+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":205,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755315+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:31:01.755315+0000","last_peered":"2023-10-09T06:31:01.755315+0000","last_clean":"2023-10-09T06:31:01.755315+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:31:01.755315+0000","last_undegraded":"2023-10-09T06:31:01.755315+0000","last_fullsized":"2023-10-09T06:31:01.755315+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":31,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460278+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:30:58.460278+0000","last_peered":"2023-10-09T06:30:58.460278+0000","last_clean":"2023-10-09T06:30:58.460278+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:30:58.460278+0000","last_undegraded":"2023-10-09T06:30:58.460278+0000","last_fullsized":"2023-10-09T06:30:58.460278+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":438,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.970375+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:30:58.970375+0000","last_peered":"2023-10-09T06:30:58.970375+0000","last_clean":"2023-10-09T06:30:58.970375+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:30:58.970375+0000","last_undegraded":"2023-10-09T06:30:58.970375+0000","last_fullsized":"2023-10-09T06:30:58.970375+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.415834+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:30:54.415834+0000","last_peered":"2023-10-09T06:30:54.415834+0000","last_clean":"2023-10-09T06:30:54.415834+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:30:54.415834+0000","last_undegraded":"2023-10-09T06:30:54.415834+0000","last_fullsized":"2023-10-09T06:30:54.415834+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":9,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.215492+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:31:03.215492+0000","last_peered":"2023-10-09T06:31:03.215492+0000","last_clean":"2023-10-09T06:31:03.215492+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:31:03.215492+0000","last_undegraded":"2023-10-09T06:31:03.215492+0000","last_fullsized":"2023-10-09T06:31:03.215492+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":247,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755590+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:31:01.755590+0000","last_peered":"2023-10-09T06:31:01.755590+0000","last_clean":"2023-10-09T06:31:01.755590+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:31:01.755590+0000","last_undegraded":"2023-10-09T06:31:01.755590+0000","last_fullsized":"2023-10-09T06:31:01.755590+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754250+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:31:01.754250+0000","last_peered":"2023-10-09T06:31:01.754250+0000","last_clean":"2023-10-09T06:31:01.754250+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:31:01.754250+0000","last_undegraded":"2023-10-09T06:31:01.754250+0000","last_fullsized":"2023-10-09T06:31:01.754250+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":209,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755837+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:31:01.755837+0000","last_peered":"2023-10-09T06:31:01.755837+0000","last_clean":"2023-10-09T06:31:01.755837+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:31:01.755837+0000","last_undegraded":"2023-10-09T06:31:01.755837+0000","last_fullsized":"2023-10-09T06:31:01.755837+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":359,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752527+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:31:01.752527+0000","last_peered":"2023-10-09T06:31:01.752527+0000","last_clean":"2023-10-09T06:31:01.752527+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:31:01.752527+0000","last_undegraded":"2023-10-09T06:31:01.752527+0000","last_fullsized":"2023-10-09T06:31:01.752527+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":9,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.762020+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:31:02.762020+0000","last_peered":"2023-10-09T06:31:02.762020+0000","last_clean":"2023-10-09T06:31:02.762020+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:31:02.762020+0000","last_undegraded":"2023-10-09T06:31:02.762020+0000","last_fullsized":"2023-10-09T06:31:02.762020+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":265,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755971+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:31:01.755971+0000","last_peered":"2023-10-09T06:31:01.755971+0000","last_clean":"2023-10-09T06:31:01.755971+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:31:01.755971+0000","last_undegraded":"2023-10-09T06:31:01.755971+0000","last_fullsized":"2023-10-09T06:31:01.755971+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":336,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.970766+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:30:58.970766+0000","last_peered":"2023-10-09T06:30:58.970766+0000","last_clean":"2023-10-09T06:30:58.970766+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:30:58.970766+0000","last_undegraded":"2023-10-09T06:30:58.970766+0000","last_fullsized":"2023-10-09T06:30:58.970766+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755613+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:31:01.755613+0000","last_peered":"2023-10-09T06:31:01.755613+0000","last_clean":"2023-10-09T06:31:01.755613+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:31:01.755613+0000","last_undegraded":"2023-10-09T06:31:01.755613+0000","last_fullsized":"2023-10-09T06:31:01.755613+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":319,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460295+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:30:58.460295+0000","last_peered":"2023-10-09T06:30:58.460295+0000","last_clean":"2023-10-09T06:30:58.460295+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:30:58.460295+0000","last_undegraded":"2023-10-09T06:30:58.460295+0000","last_fullsized":"2023-10-09T06:30:58.460295+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":694,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755182+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:31:01.755182+0000","last_peered":"2023-10-09T06:31:01.755182+0000","last_clean":"2023-10-09T06:31:01.755182+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:31:01.755182+0000","last_undegraded":"2023-10-09T06:31:01.755182+0000","last_fullsized":"2023-10-09T06:31:01.755182+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":83,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412739+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:30:54.412739+0000","last_peered":"2023-10-09T06:30:54.412739+0000","last_clean":"2023-10-09T06:30:54.412739+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:30:54.412739+0000","last_undegraded":"2023-10-09T06:30:54.412739+0000","last_fullsized":"2023-10-09T06:30:54.412739+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":342,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752503+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:31:01.752503+0000","last_peered":"2023-10-09T06:31:01.752503+0000","last_clean":"2023-10-09T06:31:01.752503+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:31:01.752503+0000","last_undegraded":"2023-10-09T06:31:01.752503+0000","last_fullsized":"2023-10-09T06:31:01.752503+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":674,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754205+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:31:01.754205+0000","last_peered":"2023-10-09T06:31:01.754205+0000","last_clean":"2023-10-09T06:31:01.754205+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:31:01.754205+0000","last_undegraded":"2023-10-09T06:31:01.754205+0000","last_fullsized":"2023-10-09T06:31:01.754205+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":317,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.970760+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:30:58.970760+0000","last_peered":"2023-10-09T06:30:58.970760+0000","last_clean":"2023-10-09T06:30:58.970760+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:30:58.970760+0000","last_undegraded":"2023-10-09T06:30:58.970760+0000","last_fullsized":"2023-10-09T06:30:58.970760+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":339,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752585+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:31:01.752585+0000","last_peered":"2023-10-09T06:31:01.752585+0000","last_clean":"2023-10-09T06:31:01.752585+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:31:01.752585+0000","last_undegraded":"2023-10-09T06:31:01.752585+0000","last_fullsized":"2023-10-09T06:31:01.752585+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":1,"reported_epoch":361,"state":"creating+peering","last_fresh":"2023-10-09T06:31:01.755706+0000","last_change":"2023-10-09T06:31:01.755706+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.755706+0000","last_undegraded":"2023-10-09T06:31:01.755706+0000","last_fullsized":"2023-10-09T06:31:01.755706+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":343,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752205+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:31:01.752205+0000","last_peered":"2023-10-09T06:31:01.752205+0000","last_clean":"2023-10-09T06:31:01.752205+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:31:01.752205+0000","last_undegraded":"2023-10-09T06:31:01.752205+0000","last_fullsized":"2023-10-09T06:31:01.752205+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":322,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.753036+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:31:01.753036+0000","last_peered":"2023-10-09T06:31:01.753036+0000","last_clean":"2023-10-09T06:31:01.753036+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:31:01.753036+0000","last_undegraded":"2023-10-09T06:31:01.753036+0000","last_fullsized":"2023-10-09T06:31:01.753036+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":1,"reported_epoch":361,"state":"creating+peering","last_fresh":"2023-10-09T06:31:01.755713+0000","last_change":"2023-10-09T06:31:01.755713+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.755713+0000","last_undegraded":"2023-10-09T06:31:01.755713+0000","last_fullsized":"2023-10-09T06:31:01.755713+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":652,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754804+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:31:01.754804+0000","last_peered":"2023-10-09T06:31:01.754804+0000","last_clean":"2023-10-09T06:31:01.754804+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:31:01.754804+0000","last_undegraded":"2023-10-09T06:31:01.754804+0000","last_fullsized":"2023-10-09T06:31:01.754804+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412582+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:30:54.412582+0000","last_peered":"2023-10-09T06:30:54.412582+0000","last_clean":"2023-10-09T06:30:54.412582+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:30:54.412582+0000","last_undegraded":"2023-10-09T06:30:54.412582+0000","last_fullsized":"2023-10-09T06:30:54.412582+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":313,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.759415+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:31:01.759415+0000","last_peered":"2023-10-09T06:31:01.759415+0000","last_clean":"2023-10-09T06:31:01.759415+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:31:01.759415+0000","last_undegraded":"2023-10-09T06:31:01.759415+0000","last_fullsized":"2023-10-09T06:31:01.759415+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":622,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.971131+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:30:58.971131+0000","last_peered":"2023-10-09T06:30:58.971131+0000","last_clean":"2023-10-09T06:30:58.971131+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:30:58.971131+0000","last_undegraded":"2023-10-09T06:30:58.971131+0000","last_fullsized":"2023-10-09T06:30:58.971131+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":364,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472910+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:30:59.472910+0000","last_peered":"2023-10-09T06:30:59.472910+0000","last_clean":"2023-10-09T06:30:59.472910+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:30:59.472910+0000","last_undegraded":"2023-10-09T06:30:59.472910+0000","last_fullsized":"2023-10-09T06:30:59.472910+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754654+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:31:01.754654+0000","last_peered":"2023-10-09T06:31:01.754654+0000","last_clean":"2023-10-09T06:31:01.754654+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:31:01.754654+0000","last_undegraded":"2023-10-09T06:31:01.754654+0000","last_fullsized":"2023-10-09T06:31:01.754654+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":298,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.473012+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:30:59.473012+0000","last_peered":"2023-10-09T06:30:59.473012+0000","last_clean":"2023-10-09T06:30:59.473012+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:30:59.473012+0000","last_undegraded":"2023-10-09T06:30:59.473012+0000","last_fullsized":"2023-10-09T06:30:59.473012+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":688,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754325+0000","last_change":"2023-10-09T06:29:15.532600+0000","last_active":"2023-10-09T06:31:01.754325+0000","last_peered":"2023-10-09T06:31:01.754325+0000","last_clean":"2023-10-09T06:31:01.754325+0000","last_became_active":"2023-10-09T06:29:15.532290+0000","last_became_peered":"2023-10-09T06:29:15.532290+0000","last_unstale":"2023-10-09T06:31:01.754325+0000","last_undegraded":"2023-10-09T06:31:01.754325+0000","last_fullsized":"2023-10-09T06:31:01.754325+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:55.742439+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754579+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:31:01.754579+0000","last_peered":"2023-10-09T06:31:01.754579+0000","last_clean":"2023-10-09T06:31:01.754579+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:31:01.754579+0000","last_undegraded":"2023-10-09T06:31:01.754579+0000","last_fullsized":"2023-10-09T06:31:01.754579+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":343,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752798+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:31:01.752798+0000","last_peered":"2023-10-09T06:31:01.752798+0000","last_clean":"2023-10-09T06:31:01.752798+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:31:01.752798+0000","last_undegraded":"2023-10-09T06:31:01.752798+0000","last_fullsized":"2023-10-09T06:31:01.752798+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":328,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472875+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:30:59.472875+0000","last_peered":"2023-10-09T06:30:59.472875+0000","last_clean":"2023-10-09T06:30:59.472875+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:30:59.472875+0000","last_undegraded":"2023-10-09T06:30:59.472875+0000","last_fullsized":"2023-10-09T06:30:59.472875+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":684,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755787+0000","last_change":"2023-10-09T06:29:15.535685+0000","last_active":"2023-10-09T06:31:01.755787+0000","last_peered":"2023-10-09T06:31:01.755787+0000","last_clean":"2023-10-09T06:31:01.755787+0000","last_became_active":"2023-10-09T06:29:15.535396+0000","last_became_peered":"2023-10-09T06:29:15.535396+0000","last_unstale":"2023-10-09T06:31:01.755787+0000","last_undegraded":"2023-10-09T06:31:01.755787+0000","last_fullsized":"2023-10-09T06:31:01.755787+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:23.142031+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":348,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.498618+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:30:54.498618+0000","last_peered":"2023-10-09T06:30:54.498618+0000","last_clean":"2023-10-09T06:30:54.498618+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:30:54.498618+0000","last_undegraded":"2023-10-09T06:30:54.498618+0000","last_fullsized":"2023-10-09T06:30:54.498618+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":337,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.756979+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:31:01.756979+0000","last_peered":"2023-10-09T06:31:01.756979+0000","last_clean":"2023-10-09T06:31:01.756979+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:31:01.756979+0000","last_undegraded":"2023-10-09T06:31:01.756979+0000","last_fullsized":"2023-10-09T06:31:01.756979+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":359,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.473254+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:30:59.473254+0000","last_peered":"2023-10-09T06:30:59.473254+0000","last_clean":"2023-10-09T06:30:59.473254+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:30:59.473254+0000","last_undegraded":"2023-10-09T06:30:59.473254+0000","last_fullsized":"2023-10-09T06:30:59.473254+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":9,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.218350+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:31:03.218350+0000","last_peered":"2023-10-09T06:31:03.218350+0000","last_clean":"2023-10-09T06:31:03.218350+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:31:03.218350+0000","last_undegraded":"2023-10-09T06:31:03.218350+0000","last_fullsized":"2023-10-09T06:31:03.218350+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":49,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754688+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:31:01.754688+0000","last_peered":"2023-10-09T06:31:01.754688+0000","last_clean":"2023-10-09T06:31:01.754688+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:31:01.754688+0000","last_undegraded":"2023-10-09T06:31:01.754688+0000","last_fullsized":"2023-10-09T06:31:01.754688+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":665,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754719+0000","last_change":"2023-10-09T06:29:15.536561+0000","last_active":"2023-10-09T06:31:01.754719+0000","last_peered":"2023-10-09T06:31:01.754719+0000","last_clean":"2023-10-09T06:31:01.754719+0000","last_became_active":"2023-10-09T06:29:15.535869+0000","last_became_peered":"2023-10-09T06:29:15.535869+0000","last_unstale":"2023-10-09T06:31:01.754719+0000","last_undegraded":"2023-10-09T06:31:01.754719+0000","last_fullsized":"2023-10-09T06:31:01.754719+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:04.229310+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":66,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.414483+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:30:54.414483+0000","last_peered":"2023-10-09T06:30:54.414483+0000","last_clean":"2023-10-09T06:30:54.414483+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:30:54.414483+0000","last_undegraded":"2023-10-09T06:30:54.414483+0000","last_fullsized":"2023-10-09T06:30:54.414483+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":322,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.970312+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:30:58.970312+0000","last_peered":"2023-10-09T06:30:58.970312+0000","last_clean":"2023-10-09T06:30:58.970312+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:30:58.970312+0000","last_undegraded":"2023-10-09T06:30:58.970312+0000","last_fullsized":"2023-10-09T06:30:58.970312+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":646,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472622+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:30:59.472622+0000","last_peered":"2023-10-09T06:30:59.472622+0000","last_clean":"2023-10-09T06:30:59.472622+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:30:59.472622+0000","last_undegraded":"2023-10-09T06:30:59.472622+0000","last_fullsized":"2023-10-09T06:30:59.472622+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":225,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754559+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:31:01.754559+0000","last_peered":"2023-10-09T06:31:01.754559+0000","last_clean":"2023-10-09T06:31:01.754559+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:31:01.754559+0000","last_undegraded":"2023-10-09T06:31:01.754559+0000","last_fullsized":"2023-10-09T06:31:01.754559+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460161+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:30:58.460161+0000","last_peered":"2023-10-09T06:30:58.460161+0000","last_clean":"2023-10-09T06:30:58.460161+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:30:58.460161+0000","last_undegraded":"2023-10-09T06:30:58.460161+0000","last_fullsized":"2023-10-09T06:30:58.460161+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":342,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460486+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:30:58.460486+0000","last_peered":"2023-10-09T06:30:58.460486+0000","last_clean":"2023-10-09T06:30:58.460486+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:30:58.460486+0000","last_undegraded":"2023-10-09T06:30:58.460486+0000","last_fullsized":"2023-10-09T06:30:58.460486+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":652,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.971080+0000","last_change":"2023-10-09T06:29:15.549866+0000","last_active":"2023-10-09T06:30:58.971080+0000","last_peered":"2023-10-09T06:30:58.971080+0000","last_clean":"2023-10-09T06:30:58.971080+0000","last_became_active":"2023-10-09T06:29:15.549495+0000","last_became_peered":"2023-10-09T06:29:15.549495+0000","last_unstale":"2023-10-09T06:30:58.971080+0000","last_undegraded":"2023-10-09T06:30:58.971080+0000","last_fullsized":"2023-10-09T06:30:58.971080+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:46.691754+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":225,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.756104+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:31:01.756104+0000","last_peered":"2023-10-09T06:31:01.756104+0000","last_clean":"2023-10-09T06:31:01.756104+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:31:01.756104+0000","last_undegraded":"2023-10-09T06:31:01.756104+0000","last_fullsized":"2023-10-09T06:31:01.756104+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":354,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472837+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:30:59.472837+0000","last_peered":"2023-10-09T06:30:59.472837+0000","last_clean":"2023-10-09T06:30:59.472837+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:30:59.472837+0000","last_undegraded":"2023-10-09T06:30:59.472837+0000","last_fullsized":"2023-10-09T06:30:59.472837+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":340,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460505+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:30:58.460505+0000","last_peered":"2023-10-09T06:30:58.460505+0000","last_clean":"2023-10-09T06:30:58.460505+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:30:58.460505+0000","last_undegraded":"2023-10-09T06:30:58.460505+0000","last_fullsized":"2023-10-09T06:30:58.460505+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":645,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752407+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:31:01.752407+0000","last_peered":"2023-10-09T06:31:01.752407+0000","last_clean":"2023-10-09T06:31:01.752407+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:31:01.752407+0000","last_undegraded":"2023-10-09T06:31:01.752407+0000","last_fullsized":"2023-10-09T06:31:01.752407+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":87,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.415924+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:30:54.415924+0000","last_peered":"2023-10-09T06:30:54.415924+0000","last_clean":"2023-10-09T06:30:54.415924+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:30:54.415924+0000","last_undegraded":"2023-10-09T06:30:54.415924+0000","last_fullsized":"2023-10-09T06:30:54.415924+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":322,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472603+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:30:59.472603+0000","last_peered":"2023-10-09T06:30:59.472603+0000","last_clean":"2023-10-09T06:30:59.472603+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:30:59.472603+0000","last_undegraded":"2023-10-09T06:30:59.472603+0000","last_fullsized":"2023-10-09T06:30:59.472603+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412499+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:30:54.412499+0000","last_peered":"2023-10-09T06:30:54.412499+0000","last_clean":"2023-10-09T06:30:54.412499+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:30:54.412499+0000","last_undegraded":"2023-10-09T06:30:54.412499+0000","last_fullsized":"2023-10-09T06:30:54.412499+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:31:01.745080+0000","last_change":"2023-10-09T06:31:01.745080+0000","last_active":"2023-10-09T06:31:01.745080+0000","last_peered":"2023-10-09T06:31:01.745080+0000","last_clean":"2023-10-09T06:31:01.745080+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:31:01.745080+0000","last_undegraded":"2023-10-09T06:31:01.745080+0000","last_fullsized":"2023-10-09T06:31:01.745080+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":688,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.416034+0000","last_change":"2023-10-09T06:28:14.734086+0000","last_active":"2023-10-09T06:30:54.416034+0000","last_peered":"2023-10-09T06:30:54.416034+0000","last_clean":"2023-10-09T06:30:54.416034+0000","last_became_active":"2023-10-09T06:28:14.733702+0000","last_became_peered":"2023-10-09T06:28:14.733702+0000","last_unstale":"2023-10-09T06:30:54.416034+0000","last_undegraded":"2023-10-09T06:30:54.416034+0000","last_fullsized":"2023-10-09T06:30:54.416034+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.831911+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":344,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.971184+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:30:58.971184+0000","last_peered":"2023-10-09T06:30:58.971184+0000","last_clean":"2023-10-09T06:30:58.971184+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:30:58.971184+0000","last_undegraded":"2023-10-09T06:30:58.971184+0000","last_fullsized":"2023-10-09T06:30:58.971184+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":323,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460370+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:30:58.460370+0000","last_peered":"2023-10-09T06:30:58.460370+0000","last_clean":"2023-10-09T06:30:58.460370+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:30:58.460370+0000","last_undegraded":"2023-10-09T06:30:58.460370+0000","last_fullsized":"2023-10-09T06:30:58.460370+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":325,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752451+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:31:01.752451+0000","last_peered":"2023-10-09T06:31:01.752451+0000","last_clean":"2023-10-09T06:31:01.752451+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:31:01.752451+0000","last_undegraded":"2023-10-09T06:31:01.752451+0000","last_fullsized":"2023-10-09T06:31:01.752451+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":325,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460450+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:30:58.460450+0000","last_peered":"2023-10-09T06:30:58.460450+0000","last_clean":"2023-10-09T06:30:58.460450+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:30:58.460450+0000","last_undegraded":"2023-10-09T06:30:58.460450+0000","last_fullsized":"2023-10-09T06:30:58.460450+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":347,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-10-09T06:30:59.472795+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:30:59.472795+0000","last_peered":"2023-10-09T06:30:59.472795+0000","last_clean":"2023-10-09T06:30:59.472795+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:30:59.472795+0000","last_undegraded":"2023-10-09T06:30:59.472795+0000","last_fullsized":"2023-10-09T06:30:59.472795+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.755525+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:31:01.755525+0000","last_peered":"2023-10-09T06:31:01.755525+0000","last_clean":"2023-10-09T06:31:01.755525+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:31:01.755525+0000","last_undegraded":"2023-10-09T06:31:01.755525+0000","last_fullsized":"2023-10-09T06:31:01.755525+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":369,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752694+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:31:01.752694+0000","last_peered":"2023-10-09T06:31:01.752694+0000","last_clean":"2023-10-09T06:31:01.752694+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:31:01.752694+0000","last_undegraded":"2023-10-09T06:31:01.752694+0000","last_fullsized":"2023-10-09T06:31:01.752694+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":53,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.754561+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:31:01.754561+0000","last_peered":"2023-10-09T06:31:01.754561+0000","last_clean":"2023-10-09T06:31:01.754561+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:31:01.754561+0000","last_undegraded":"2023-10-09T06:31:01.754561+0000","last_fullsized":"2023-10-09T06:31:01.754561+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":379,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752520+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:31:01.752520+0000","last_peered":"2023-10-09T06:31:01.752520+0000","last_clean":"2023-10-09T06:31:01.752520+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:31:01.752520+0000","last_undegraded":"2023-10-09T06:31:01.752520+0000","last_fullsized":"2023-10-09T06:31:01.752520+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":57,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.414787+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:30:54.414787+0000","last_peered":"2023-10-09T06:30:54.414787+0000","last_clean":"2023-10-09T06:30:54.414787+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:30:54.414787+0000","last_undegraded":"2023-10-09T06:30:54.414787+0000","last_fullsized":"2023-10-09T06:30:54.414787+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":75,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T06:30:54.412635+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:30:54.412635+0000","last_peered":"2023-10-09T06:30:54.412635+0000","last_clean":"2023-10-09T06:30:54.412635+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:30:54.412635+0000","last_undegraded":"2023-10-09T06:30:54.412635+0000","last_fullsized":"2023-10-09T06:30:54.412635+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":353,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T06:30:58.460301+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:30:58.460301+0000","last_peered":"2023-10-09T06:30:58.460301+0000","last_clean":"2023-10-09T06:30:58.460301+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:30:58.460301+0000","last_undegraded":"2023-10-09T06:30:58.460301+0000","last_fullsized":"2023-10-09T06:30:58.460301+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":322,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T06:31:01.752168+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:31:01.752168+0000","last_peered":"2023-10-09T06:31:01.752168+0000","last_clean":"2023-10-09T06:31:01.752168+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:31:01.752168+0000","last_undegraded":"2023-10-09T06:31:01.752168+0000","last_fullsized":"2023-10-09T06:31:01.752168+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":15,"acting":15,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234405,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912320,"kb_used_data":3896,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":92832832,"statfs":{"total":95995035648,"available":95060819968,"internally_reserved":0,"allocated":3989504,"data_stored":3029418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":4.9039999999999999,"15min":4.9569999999999999},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":4.2549999999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.9370000000000003,"15min":5.0670000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":3.2930000000000001,"5min":215.577,"15min":219.399},"last":0.79600000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":7.7770000000000001,"5min":6.6429999999999998,"15min":4.2619999999999996},"min":{"1min":0.318,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":45.502000000000002,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":5.8609999999999998,"5min":5.7720000000000002,"15min":3.794},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.447000000000003,"5min":216.22200000000001,"15min":216.22200000000001},"last":0.54000000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":4.5220000000000002,"15min":6.0960000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":6.3319999999999999,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":3.1200000000000001,"5min":4.7999999999999998,"15min":6.2789999999999999},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.844000000000001,"5min":216.25200000000001,"15min":216.25200000000001},"last":0.70399999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":7.6580000000000004,"15min":3.899},"min":{"1min":0.307,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":187.34200000000001,"5min":215.922,"15min":215.922},"last":0.76400000000000001},{"interface":"front","average":{"1min":16.768000000000001,"5min":7.7969999999999997,"15min":3.9900000000000002},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":187.08199999999999,"5min":215.94900000000001,"15min":215.94900000000001},"last":0.72499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":4.1120000000000001,"15min":2.6160000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":6.1689999999999996,"5min":218.446,"15min":218.446},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.9889999999999999,"15min":2.4079999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.23699999999999999},"max":{"1min":4.891,"5min":216.172,"15min":216.172},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":4.2350000000000003,"15min":2.661},"min":{"1min":0.31,"5min":0.31,"15min":0.22900000000000001},"max":{"1min":8.8170000000000002,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.603,"5min":4.1079999999999997,"15min":2.4809999999999999},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":6.3019999999999996,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":4.343,"15min":2.3929999999999998},"min":{"1min":0.33300000000000002,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":15.162000000000001,"5min":218.19999999999999,"15min":218.19999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.5,"5min":3.9390000000000001,"15min":2.161},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.2919999999999998,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.60199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496040,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902556,"kb_used_data":3540,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92842596,"statfs":{"total":95995035648,"available":95070818304,"internally_reserved":0,"allocated":3624960,"data_stored":2684748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.627,"15min":3.0779999999999998},"min":{"1min":0.24399999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.282,"5min":22.321999999999999,"15min":110.926},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.1320000000000001,"15min":3.3980000000000001},"min":{"1min":0.29399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2870000000000008,"5min":22.338000000000001,"15min":110.911},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 06:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3760000000000003,"5min":6.9649999999999999,"15min":5.0010000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":82.159000000000006,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.41199999999999998},{"interface":"front","average":{"1min":7.1950000000000003,"5min":7.0590000000000002,"15min":4.8810000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":82.174000000000007,"5min":292.904,"15min":292.904},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.109,"15min":2.6459999999999999},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.3670000000000009,"5min":20.331,"15min":75.191999999999993},"last":0.495},{"interface":"front","average":{"1min":2.335,"5min":1.748,"15min":2.351},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":8.2400000000000002,"5min":8.6069999999999993,"15min":75.120000000000005},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":8.4819999999999993,"5min":3.4540000000000002,"15min":2.8650000000000002},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":91.174000000000007,"5min":91.174000000000007,"15min":110.358},"last":0.56699999999999995},{"interface":"front","average":{"1min":8.7750000000000004,"5min":3.3620000000000001,"15min":2.9199999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.311},"max":{"1min":91.438999999999993,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.361,"15min":1.3220000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":12.032,"5min":12.666,"15min":17.643999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.97499999999999998,"15min":1.1739999999999999},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":5.1280000000000001,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.47899999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.3169999999999999,"15min":1.429},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.9830000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.448,"5min":1.379,"15min":1.369},"min":{"1min":0.33300000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.8000000000000007,"5min":16.536999999999999,"15min":24.401},"last":0.53500000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":4.0529999999999999,"15min":2.528},"min":{"1min":0.314,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":7.8899999999999997,"5min":122.937,"15min":122.937},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.3369999999999997,"15min":2.6709999999999998},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.32800000000000001},"max":{"1min":10.423,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.59099999999999997}]}]},{"osd":1,"up_from":110,"seq":472446402778,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972516,"kb_used_data":314780,"kb_used_omap":4,"kb_used_meta":657723,"kb_avail":92772636,"statfs":{"total":95995035648,"available":94999179264,"internally_reserved":0,"allocated":322334720,"data_stored":320684074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":673509262},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":9.8030000000000008,"5min":6.2869999999999999,"15min":5.2409999999999997},"min":{"1min":0.22,"5min":0.189,"15min":0.189},"max":{"1min":60.973999999999997,"5min":269.79199999999997,"15min":269.79199999999997},"last":0.38900000000000001},{"interface":"front","average":{"1min":8.8480000000000008,"5min":6.4539999999999997,"15min":5.6299999999999999},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.17899999999999999},"max":{"1min":60.201000000000001,"5min":271.20999999999998,"15min":271.20999999999998},"last":0.25700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.2789999999999999,"15min":2.9540000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":6.9390000000000001,"5min":7.1449999999999996,"15min":139.75399999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3759999999999999,"15min":3.6970000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":6.673,"5min":12.077,"15min":140.29900000000001},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:30:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":3.2040000000000002,"15min":3.2679999999999998},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":13.214,"5min":31.062000000000001,"15min":153.18199999999999},"last":0.373},{"interface":"front","average":{"1min":2.6389999999999998,"5min":2.2389999999999999,"15min":2.444},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.128,"5min":27.657,"15min":152.52000000000001},"last":0.28299999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.883,"15min":3.3929999999999998},"min":{"1min":0.312,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":13.087999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.823,"15min":3.2989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.1300000000000008,"5min":11.974,"15min":199.32900000000001},"last":0.61099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":1.962,"15min":2.0699999999999998},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":52.512999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.71499999999999997},{"interface":"front","average":{"1min":4.665,"5min":2.3260000000000001,"15min":2.6000000000000001},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":52.664999999999999,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0220000000000002,"5min":2.7250000000000001,"15min":2.8969999999999998},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":52.488,"5min":52.488,"15min":77.292000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":6.556,"5min":2.952,"15min":2.843},"min":{"1min":0.38200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":52.558,"5min":52.558,"15min":77.378},"last":0.75600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0129999999999999,"5min":3.8399999999999999,"15min":3.0470000000000002},"min":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.32500000000000001},"max":{"1min":52.601999999999997,"5min":55.231000000000002,"15min":77.456999999999994},"last":0.78500000000000003},{"interface":"front","average":{"1min":4.8040000000000003,"5min":3.1800000000000002,"15min":2.5179999999999998},"min":{"1min":0.42199999999999999,"5min":0.35799999999999998,"15min":0.34399999999999997},"max":{"1min":52.579999999999998,"5min":55.253999999999998,"15min":77.359999999999999},"last":0.68500000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024211,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129672,"kb_used_data":337536,"kb_used_omap":5,"kb_used_meta":792122,"kb_avail":92615480,"statfs":{"total":95995035648,"available":94838251520,"internally_reserved":0,"allocated":345636864,"data_stored":343722285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":811133562},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":3.6059999999999999,"15min":3.3719999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.16600000000000001},"max":{"1min":8.1389999999999993,"5min":191.446,"15min":191.446},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.6419999999999999,"5min":3.8159999999999998,"15min":3.6709999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.17499999999999999},"max":{"1min":12.959,"5min":191.393,"15min":191.393},"last":0.36599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.7309999999999999,"15min":2.3180000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":16.936,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.6419999999999999,"5min":3.524,"15min":2.6960000000000002},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":15.794,"5min":57.585000000000001,"15min":57.585000000000001},"last":0.34699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:30:06 2023","interfaces":[{"interface":"back","average":{"1min":9.5589999999999993,"5min":6.0220000000000002,"15min":4.6059999999999999},"min":{"1min":0.42899999999999999,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":94.275000000000006,"5min":208.11199999999999,"15min":208.11199999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":8.4440000000000008,"5min":5.6349999999999998,"15min":4.5979999999999999},"min":{"1min":0.38800000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":94.331000000000003,"5min":208.25899999999999,"15min":208.25899999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.518,"15min":1.429},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":7.21,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.7470000000000001,"15min":1.798},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":10.903,"5min":18.081,"15min":18.081},"last":0.67000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.7130000000000001,"15min":1.806},"min":{"1min":0.317,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":7.2409999999999997,"5min":10.865,"15min":62.984999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.9970000000000001,"15min":2.2469999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":10.603,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.625}]},{"osd":6,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":4.2619999999999996,"15min":2.8980000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":12.522,"5min":120.572,"15min":120.572},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.758,"5min":3.827,"15min":2.3330000000000002},"min":{"1min":0.371,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":11.093,"5min":120.64400000000001,"15min":120.64400000000001},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":3.1840000000000002,"15min":2.4969999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":8.8059999999999992,"5min":122.018,"15min":122.018},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.415,"5min":3.6539999999999999,"15min":2.8730000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":10.563000000000001,"5min":122.32299999999999,"15min":122.32299999999999},"last":0.59499999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814080,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525760,"kb_used_data":4536,"kb_used_omap":7,"kb_used_meta":521208,"kb_avail":93219392,"statfs":{"total":95995035648,"available":95456657408,"internally_reserved":0,"allocated":4644864,"data_stored":3025371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":533717073},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":4.2409999999999997,"15min":4.7370000000000001},"min":{"1min":0.19500000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":16.91,"5min":162.21100000000001,"15min":163.40100000000001},"last":0.372},{"interface":"front","average":{"1min":3.4180000000000001,"5min":4.3630000000000004,"15min":4.4260000000000002},"min":{"1min":0.30099999999999999,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":16.888999999999999,"5min":168.49700000000001,"15min":168.49700000000001},"last":0.36399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":3.3719999999999999,"15min":4.1740000000000004},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":24.312000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.714,"5min":4.423,"15min":4.5380000000000003},"min":{"1min":0.29399999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.259,"5min":91.721999999999994,"15min":222.19399999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:30:07 2023","interfaces":[{"interface":"back","average":{"1min":3.9329999999999998,"5min":3.5089999999999999,"15min":2.7280000000000002},"min":{"1min":0.47299999999999998,"5min":0.26500000000000001,"15min":0.22700000000000001},"max":{"1min":16.053000000000001,"5min":91.644000000000005,"15min":91.644000000000005},"last":0.39200000000000002},{"interface":"front","average":{"1min":4.0899999999999999,"5min":4.2160000000000002,"15min":2.9620000000000002},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.206,"5min":91.685000000000002,"15min":91.685000000000002},"last":0.32000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":1.7589999999999999,"15min":2.081},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":13.023999999999999,"5min":17.776,"15min":90.938999999999993},"last":0.54700000000000004},{"interface":"front","average":{"1min":3.0880000000000001,"5min":2.1560000000000001,"15min":2.375},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":90.950000000000003},"last":0.622}]},{"osd":5,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.0339999999999998,"15min":1.605},"min":{"1min":0.315,"5min":0.315,"15min":0.309},"max":{"1min":13.102,"5min":17.800000000000001,"15min":17.800000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":3.2930000000000001,"5min":2.2709999999999999,"15min":1.978},"min":{"1min":0.377,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.57199999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":3.585,"5min":2.3650000000000002,"15min":2.5529999999999999},"min":{"1min":0.41399999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":12.387,"5min":12.387,"15min":138.291},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.639,"5min":1.6459999999999999,"15min":2.1829999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":9.1579999999999995,"5min":12.643000000000001,"15min":138.01599999999999},"last":0.55900000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.052,"15min":1.968},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":9.1859999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.04,"5min":2.113,"15min":2.0110000000000001},"min":{"1min":0.42099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.167,"5min":17.734999999999999,"15min":55.731000000000002},"last":0.59199999999999997}]}]},{"osd":3,"up_from":149,"seq":639950127277,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":847144,"kb_used_data":348320,"kb_used_omap":2,"kb_used_meta":498813,"kb_avail":92898008,"statfs":{"total":95995035648,"available":95127560192,"internally_reserved":0,"allocated":356679680,"data_stored":355134892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":510784565},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.4260000000000002,"5min":2.673,"15min":3.8820000000000001},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19600000000000001},"max":{"1min":11.813000000000001,"5min":34.249000000000002,"15min":187.852},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.0779999999999998,"5min":2.4870000000000001,"15min":4.2140000000000004},"min":{"1min":0.20799999999999999,"5min":0.184,"15min":0.184},"max":{"1min":11.93,"5min":34.276000000000003,"15min":187.923},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.3639999999999999,"15min":3.3399999999999999},"min":{"1min":0.23699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":15.814,"5min":34.646999999999998,"15min":187.89099999999999},"last":0.316},{"interface":"front","average":{"1min":4.8360000000000003,"5min":3.363,"15min":2.7650000000000001},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.02,"5min":34.207999999999998,"15min":67.028000000000006},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":2.2949999999999999,"15min":3.0619999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.2930000000000001,"5min":34.307000000000002,"15min":187.76400000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":3.6150000000000002,"5min":2.7890000000000001,"15min":2.798},"min":{"1min":0.27900000000000003,"5min":0.23599999999999999,"15min":0.22500000000000001},"max":{"1min":11.425000000000001,"5min":35.055999999999997,"15min":187.79499999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9670000000000001,"5min":2.0640000000000001,"15min":1.532},"min":{"1min":0.31,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":25.591999999999999,"5min":25.591999999999999,"15min":25.591999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":3.7669999999999999,"5min":1.9610000000000001,"15min":2.2480000000000002},"min":{"1min":0.36299999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":25.544,"5min":25.544,"15min":187.67400000000001},"last":0.65000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":1.7070000000000001,"15min":2.2930000000000001},"min":{"1min":0.379,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":6.4480000000000004,"5min":8.6159999999999997,"15min":187.833},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.395,"5min":1.427,"15min":2.129},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":6.7649999999999997,"5min":9.8469999999999995,"15min":187.952},"last":0.67000000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":10.112,"5min":3.5539999999999998,"15min":2.8889999999999998},"min":{"1min":0.379,"5min":0.374,"15min":0.33700000000000002},"max":{"1min":130.39599999999999,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.747},{"interface":"front","average":{"1min":9.0419999999999998,"5min":3.0600000000000001,"15min":2.6190000000000002},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":130.67699999999999,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.63800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":12.199999999999999,"5min":3.8879999999999999,"15min":2.871},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34100000000000003},"max":{"1min":121.613,"5min":121.613,"15min":192.20400000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":11.423999999999999,"5min":3.875,"15min":2.9780000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":121.559,"5min":121.559,"15min":187.86500000000001},"last":0.70999999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019314,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886252,"kb_used_data":3556,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":92858900,"statfs":{"total":95995035648,"available":95087513600,"internally_reserved":0,"allocated":3641344,"data_stored":2693263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.696,"15min":2.2210000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":18.625,"5min":37.485999999999997,"15min":76.944999999999993},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.962,"5min":1.77,"15min":2.4740000000000002},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.654,"5min":37.502000000000002,"15min":76.988},"last":0.55800000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":10.651999999999999,"5min":4.3789999999999996,"15min":2.968},"min":{"1min":0.37,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":128.24299999999999,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":10.84,"5min":4.4909999999999997,"15min":3.133},"min":{"1min":0.43099999999999999,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":128.21600000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":0.63700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":3.0659999999999998,"15min":2.48},"min":{"1min":0.309,"5min":0.309,"15min":0.255},"max":{"1min":7.101,"5min":47.125,"15min":69.640000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.601,"5min":2.7080000000000002,"15min":2.508},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.9870000000000001,"5min":47.151000000000003,"15min":69.676000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":21.917000000000002,"5min":6.8090000000000002,"15min":4.2279999999999998},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":133.38900000000001,"5min":133.38900000000001,"15min":133.38900000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":21.408000000000001,"5min":6.3840000000000003,"15min":4.0019999999999998},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":132.57599999999999,"5min":132.57599999999999,"15min":132.57599999999999},"last":0.70899999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.0329999999999999,"15min":1.1200000000000001},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":6.0199999999999996,"5min":6.2670000000000003,"15min":10.673},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.0249999999999999,"15min":1.111},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":5.7000000000000002,"5min":7.2249999999999996,"15min":12.552},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.4660000000000002,"15min":1.3340000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.0629999999999997,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.8029999999999999,"15min":1.7989999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.7210000000000001,"5min":163.381,"15min":163.381},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.0149999999999999,"15min":1.506},"min":{"1min":0.371,"5min":0.312,"15min":0.308},"max":{"1min":5.9850000000000003,"5min":6.2679999999999998,"15min":62.658000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.82399999999999995,"15min":1.4770000000000001},"min":{"1min":0.40899999999999997,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":3.8199999999999998,"5min":4.2599999999999998,"15min":62.573},"last":0.874}]}]},{"osd":5,"up_from":38,"seq":163208757679,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905828,"kb_used_data":4124,"kb_used_omap":0,"kb_used_meta":901696,"kb_avail":92839324,"statfs":{"total":95995035648,"available":95067467776,"internally_reserved":0,"allocated":4222976,"data_stored":3279339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923336704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.0990000000000002,"15min":3.4900000000000002},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":3.819,"5min":131.87200000000001,"15min":131.87200000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.8769999999999998,"15min":3.4260000000000002},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":3.6709999999999998,"5min":131.80199999999999,"15min":131.80199999999999},"last":0.498}]},{"osd":1,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":9.0109999999999992,"5min":4.9610000000000003,"15min":2.843},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23400000000000001},"max":{"1min":87.795000000000002,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":10.266999999999999,"5min":5.2930000000000001,"15min":3.0350000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":87.856999999999999,"5min":126.931,"15min":126.931},"last":0.51200000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3599999999999999,"5min":2.52,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":16.5,"5min":51.393000000000001,"15min":81.260000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.282,"5min":2.1789999999999998,"15min":2.1840000000000002},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5679999999999996,"5min":51.432000000000002,"15min":81.245000000000005},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:31:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5049999999999999,"5min":9.5310000000000006,"15min":5.4580000000000002},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":92.521000000000001,"5min":144.56399999999999,"15min":190.411},"last":0.41899999999999998},{"interface":"front","average":{"1min":5.4720000000000004,"5min":9.4909999999999997,"15min":5.3449999999999998},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":94.492000000000004,"5min":144.62799999999999,"15min":190.70699999999999},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.0649999999999999,"15min":1.3759999999999999},"min":{"1min":0.311,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9119999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.90400000000000003,"15min":1.1619999999999999},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2610000000000001,"5min":6.2610000000000001,"15min":32.82},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3080000000000001,"15min":1.5509999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":3.6859999999999999,"5min":6.2080000000000002,"15min":38.902000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.395,"15min":1.5920000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.6340000000000003,"5min":8.3659999999999997,"15min":35.774000000000001},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:30:14 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.339,"15min":1.524},"min":{"1min":0.46999999999999997,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":7.4290000000000003,"5min":10.461,"15min":30.658999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.2679999999999998,"5min":1.329,"15min":1.502},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":6.2320000000000002,"5min":6.2320000000000002,"15min":30.683},"last":0.72199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:31:06.037 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:31:06.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:06.139 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:06.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:05 smithi150 ceph-mon[194178]: from='client.36408 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:06.171 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:06.219 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:06.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:06.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:06.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:06.353 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:06.397 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:06.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:06.502 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:06.530 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:06.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:06 smithi107 ceph-mon[212750]: pgmap v1123: 99 pgs: 2 creating+peering, 9 unknown, 88 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:06 smithi107 ceph-mon[220921]: pgmap v1123: 99 pgs: 2 creating+peering, 9 unknown, 88 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:06 smithi150 ceph-mon[194178]: pgmap v1123: 99 pgs: 2 creating+peering, 9 unknown, 88 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:07.162 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:07.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:07.271 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:07.303 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:07.351 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:07.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:07.458 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:07.487 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:07.504 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:31:07.531 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:07.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:07.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:07.639 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:07.665 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:07.991 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:08.013 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:08.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:08.118 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:08.146 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:08.188 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:08.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:08.295 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:08.326 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:08.380 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:08.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:08.423 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:31:08.424 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:31:08.424 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:31:08.424 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:31:08.424 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:31:08.484 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:08.511 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:08.552 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:08.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:08.658 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:08.684 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:08.725 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:08.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:08.771 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:31:08.831 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:08.868 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:09.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[212750]: pgmap v1124: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:09.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2542071692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.071 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[220921]: pgmap v1124: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:09.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2542071692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.072 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:08 smithi150 ceph-mon[194178]: pgmap v1124: 99 pgs: 2 creating+peering, 97 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2542071692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:09.784 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-10-09T06:31:09.803 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:09.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:09.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:31:09.909 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:09.936 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.064 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:09 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: osdmap e364: 8 total, 8 up, 8 in 2023-10-09T06:31:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:31:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:09 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.434 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:10.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:10.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:10.538 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:10.570 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:10.620 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:10.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:10.725 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.752 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:10.852 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:10.855 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:10.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: failed to encode map e364 with expected crc 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: pgmap v1126: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: failed to encode map e365 with expected crc 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: failed to encode map e365 with expected crc 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:31:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:10 smithi150 ceph-mon[194178]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:31:10.959 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:10.989 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:11.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: failed to encode map e364 with expected crc 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: pgmap v1126: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:31:11.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[212750]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: failed to encode map e364 with expected crc 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: pgmap v1126: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: failed to encode map e365 with expected crc 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: failed to encode map e365 with expected crc 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2826039951' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:31:11.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:10 smithi107 ceph-mon[220921]: osdmap e365: 8 total, 8 up, 8 in 2023-10-09T06:31:11.139 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:11.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:11.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:11.234 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:11.244 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:11.278 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:11.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:11.436 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:11.465 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:11.489 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:31:11.491 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:31:11.825 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-10-09T06:31:11.828 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:31:11.829 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:31:11.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:31:11.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:31:11.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_479504 2023-10-09T06:31:11.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:31:11.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:31:12.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:11 smithi107 ceph-mon[220921]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:11 smithi107 ceph-mon[220921]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:11 smithi107 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:11 smithi107 ceph-mon[212750]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.019 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:11 smithi150 ceph-mon[194178]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:11 smithi150 ceph-mon[194178]: failed to encode map e365 with expected crc 2023-10-09T06:31:12.319 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:12.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:12.334 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:12] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.43.0" 2023-10-09T06:31:12.356 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1128,"stamp":"2023-10-09T06:31:11.313637+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":600556,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7034284,"kb_used_data":972524,"kb_used_omap":20,"kb_used_meta":6061675,"kb_avail":742926932,"statfs":{"total":767960285184,"available":760757178368,"internally_reserved":0,"allocated":995864576,"data_stored":985301734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6207155854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.559655"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":342,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757314+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:31:02.757314+0000","last_peered":"2023-10-09T06:31:02.757314+0000","last_clean":"2023-10-09T06:31:02.757314+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:31:02.757314+0000","last_undegraded":"2023-10-09T06:31:02.757314+0000","last_fullsized":"2023-10-09T06:31:02.757314+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898996+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:31:03.898996+0000","last_peered":"2023-10-09T06:31:03.898996+0000","last_clean":"2023-10-09T06:31:03.898996+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:31:03.898996+0000","last_undegraded":"2023-10-09T06:31:03.898996+0000","last_fullsized":"2023-10-09T06:31:03.898996+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":355,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755598+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:31:02.755598+0000","last_peered":"2023-10-09T06:31:02.755598+0000","last_clean":"2023-10-09T06:31:02.755598+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:31:02.755598+0000","last_undegraded":"2023-10-09T06:31:02.755598+0000","last_fullsized":"2023-10-09T06:31:02.755598+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":361,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766165+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:31:08.766165+0000","last_peered":"2023-10-09T06:31:08.766165+0000","last_clean":"2023-10-09T06:31:08.766165+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:31:08.766165+0000","last_undegraded":"2023-10-09T06:31:08.766165+0000","last_fullsized":"2023-10-09T06:31:08.766165+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898505+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:31:03.898505+0000","last_peered":"2023-10-09T06:31:03.898505+0000","last_clean":"2023-10-09T06:31:03.898505+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:31:03.898505+0000","last_undegraded":"2023-10-09T06:31:03.898505+0000","last_fullsized":"2023-10-09T06:31:03.898505+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":354,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220197+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:31:09.220197+0000","last_peered":"2023-10-09T06:31:09.220197+0000","last_clean":"2023-10-09T06:31:09.220197+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:31:09.220197+0000","last_undegraded":"2023-10-09T06:31:09.220197+0000","last_fullsized":"2023-10-09T06:31:09.220197+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766805+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:31:08.766805+0000","last_peered":"2023-10-09T06:31:08.766805+0000","last_clean":"2023-10-09T06:31:08.766805+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:31:08.766805+0000","last_undegraded":"2023-10-09T06:31:08.766805+0000","last_fullsized":"2023-10-09T06:31:08.766805+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898650+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:31:03.898650+0000","last_peered":"2023-10-09T06:31:03.898650+0000","last_clean":"2023-10-09T06:31:03.898650+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:31:03.898650+0000","last_undegraded":"2023-10-09T06:31:03.898650+0000","last_fullsized":"2023-10-09T06:31:03.898650+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207361+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:31:03.207361+0000","last_peered":"2023-10-09T06:31:03.207361+0000","last_clean":"2023-10-09T06:31:03.207361+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:31:03.207361+0000","last_undegraded":"2023-10-09T06:31:03.207361+0000","last_fullsized":"2023-10-09T06:31:03.207361+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898853+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:31:03.898853+0000","last_peered":"2023-10-09T06:31:03.898853+0000","last_clean":"2023-10-09T06:31:03.898853+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:31:03.898853+0000","last_undegraded":"2023-10-09T06:31:03.898853+0000","last_fullsized":"2023-10-09T06:31:03.898853+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":244,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759915+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:31:02.759915+0000","last_peered":"2023-10-09T06:31:02.759915+0000","last_clean":"2023-10-09T06:31:02.759915+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:31:02.759915+0000","last_undegraded":"2023-10-09T06:31:02.759915+0000","last_fullsized":"2023-10-09T06:31:02.759915+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":296,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755108+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:31:02.755108+0000","last_peered":"2023-10-09T06:31:02.755108+0000","last_clean":"2023-10-09T06:31:02.755108+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:31:02.755108+0000","last_undegraded":"2023-10-09T06:31:02.755108+0000","last_fullsized":"2023-10-09T06:31:02.755108+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":369,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.756982+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:31:02.756982+0000","last_peered":"2023-10-09T06:31:02.756982+0000","last_clean":"2023-10-09T06:31:02.756982+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:31:02.756982+0000","last_undegraded":"2023-10-09T06:31:02.756982+0000","last_fullsized":"2023-10-09T06:31:02.756982+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":333,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757993+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:31:02.757993+0000","last_peered":"2023-10-09T06:31:02.757993+0000","last_clean":"2023-10-09T06:31:02.757993+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:31:02.757993+0000","last_undegraded":"2023-10-09T06:31:02.757993+0000","last_fullsized":"2023-10-09T06:31:02.757993+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":212,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.760747+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:31:02.760747+0000","last_peered":"2023-10-09T06:31:02.760747+0000","last_clean":"2023-10-09T06:31:02.760747+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:31:02.760747+0000","last_undegraded":"2023-10-09T06:31:02.760747+0000","last_fullsized":"2023-10-09T06:31:02.760747+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898918+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:31:03.898918+0000","last_peered":"2023-10-09T06:31:03.898918+0000","last_clean":"2023-10-09T06:31:03.898918+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:31:03.898918+0000","last_undegraded":"2023-10-09T06:31:03.898918+0000","last_fullsized":"2023-10-09T06:31:03.898918+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":359,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764868+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:31:08.764868+0000","last_peered":"2023-10-09T06:31:08.764868+0000","last_clean":"2023-10-09T06:31:08.764868+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:31:08.764868+0000","last_undegraded":"2023-10-09T06:31:08.764868+0000","last_fullsized":"2023-10-09T06:31:08.764868+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207407+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:31:03.207407+0000","last_peered":"2023-10-09T06:31:03.207407+0000","last_clean":"2023-10-09T06:31:03.207407+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:31:03.207407+0000","last_undegraded":"2023-10-09T06:31:03.207407+0000","last_fullsized":"2023-10-09T06:31:03.207407+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.760105+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:31:02.760105+0000","last_peered":"2023-10-09T06:31:02.760105+0000","last_clean":"2023-10-09T06:31:02.760105+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:31:02.760105+0000","last_undegraded":"2023-10-09T06:31:02.760105+0000","last_fullsized":"2023-10-09T06:31:02.760105+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765924+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:31:08.765924+0000","last_peered":"2023-10-09T06:31:08.765924+0000","last_clean":"2023-10-09T06:31:08.765924+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:31:08.765924+0000","last_undegraded":"2023-10-09T06:31:08.765924+0000","last_fullsized":"2023-10-09T06:31:08.765924+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":443,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219350+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:31:09.219350+0000","last_peered":"2023-10-09T06:31:09.219350+0000","last_clean":"2023-10-09T06:31:09.219350+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:31:09.219350+0000","last_undegraded":"2023-10-09T06:31:09.219350+0000","last_fullsized":"2023-10-09T06:31:09.219350+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.899108+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:31:03.899108+0000","last_peered":"2023-10-09T06:31:03.899108+0000","last_clean":"2023-10-09T06:31:03.899108+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:31:03.899108+0000","last_undegraded":"2023-10-09T06:31:03.899108+0000","last_fullsized":"2023-10-09T06:31:03.899108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":10,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.215899+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:31:03.215899+0000","last_peered":"2023-10-09T06:31:03.215899+0000","last_clean":"2023-10-09T06:31:03.215899+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:31:03.215899+0000","last_undegraded":"2023-10-09T06:31:03.215899+0000","last_fullsized":"2023-10-09T06:31:03.215899+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":248,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.760858+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:31:02.760858+0000","last_peered":"2023-10-09T06:31:02.760858+0000","last_clean":"2023-10-09T06:31:02.760858+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:31:02.760858+0000","last_undegraded":"2023-10-09T06:31:02.760858+0000","last_fullsized":"2023-10-09T06:31:02.760858+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.206853+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:31:03.206853+0000","last_peered":"2023-10-09T06:31:03.206853+0000","last_clean":"2023-10-09T06:31:03.206853+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:31:03.206853+0000","last_undegraded":"2023-10-09T06:31:03.206853+0000","last_fullsized":"2023-10-09T06:31:03.206853+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":12,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219472+0000","last_change":"2023-10-09T06:31:02.763465+0000","last_active":"2023-10-09T06:31:09.219472+0000","last_peered":"2023-10-09T06:31:09.219472+0000","last_clean":"2023-10-09T06:31:09.219472+0000","last_became_active":"2023-10-09T06:31:02.763153+0000","last_became_peered":"2023-10-09T06:31:02.763153+0000","last_unstale":"2023-10-09T06:31:09.219472+0000","last_undegraded":"2023-10-09T06:31:09.219472+0000","last_fullsized":"2023-10-09T06:31:09.219472+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:01.874473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":210,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.760861+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:31:02.760861+0000","last_peered":"2023-10-09T06:31:02.760861+0000","last_clean":"2023-10-09T06:31:02.760861+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:31:02.760861+0000","last_undegraded":"2023-10-09T06:31:02.760861+0000","last_fullsized":"2023-10-09T06:31:02.760861+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":360,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757277+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:31:02.757277+0000","last_peered":"2023-10-09T06:31:02.757277+0000","last_clean":"2023-10-09T06:31:02.757277+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:31:02.757277+0000","last_undegraded":"2023-10-09T06:31:02.757277+0000","last_fullsized":"2023-10-09T06:31:02.757277+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":10,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.762437+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:31:02.762437+0000","last_peered":"2023-10-09T06:31:02.762437+0000","last_clean":"2023-10-09T06:31:02.762437+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:31:02.762437+0000","last_undegraded":"2023-10-09T06:31:02.762437+0000","last_fullsized":"2023-10-09T06:31:02.762437+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":266,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759620+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:31:02.759620+0000","last_peered":"2023-10-09T06:31:02.759620+0000","last_clean":"2023-10-09T06:31:02.759620+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:31:02.759620+0000","last_undegraded":"2023-10-09T06:31:02.759620+0000","last_fullsized":"2023-10-09T06:31:02.759620+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":341,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219650+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:31:09.219650+0000","last_peered":"2023-10-09T06:31:09.219650+0000","last_clean":"2023-10-09T06:31:09.219650+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:31:09.219650+0000","last_undegraded":"2023-10-09T06:31:09.219650+0000","last_fullsized":"2023-10-09T06:31:09.219650+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":13,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764324+0000","last_change":"2023-10-09T06:31:02.761953+0000","last_active":"2023-10-09T06:31:08.764324+0000","last_peered":"2023-10-09T06:31:08.764324+0000","last_clean":"2023-10-09T06:31:08.764324+0000","last_became_active":"2023-10-09T06:31:02.761625+0000","last_became_peered":"2023-10-09T06:31:02.761625+0000","last_unstale":"2023-10-09T06:31:08.764324+0000","last_undegraded":"2023-10-09T06:31:08.764324+0000","last_fullsized":"2023-10-09T06:31:08.764324+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:31.403447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207279+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:31:03.207279+0000","last_peered":"2023-10-09T06:31:03.207279+0000","last_clean":"2023-10-09T06:31:03.207279+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:31:03.207279+0000","last_undegraded":"2023-10-09T06:31:03.207279+0000","last_fullsized":"2023-10-09T06:31:03.207279+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":324,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764682+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:31:08.764682+0000","last_peered":"2023-10-09T06:31:08.764682+0000","last_clean":"2023-10-09T06:31:08.764682+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:31:08.764682+0000","last_undegraded":"2023-10-09T06:31:08.764682+0000","last_fullsized":"2023-10-09T06:31:08.764682+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":12,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219472+0000","last_change":"2023-10-09T06:31:02.762571+0000","last_active":"2023-10-09T06:31:09.219472+0000","last_peered":"2023-10-09T06:31:09.219472+0000","last_clean":"2023-10-09T06:31:09.219472+0000","last_became_active":"2023-10-09T06:31:02.762287+0000","last_became_peered":"2023-10-09T06:31:02.762287+0000","last_unstale":"2023-10-09T06:31:09.219472+0000","last_undegraded":"2023-10-09T06:31:09.219472+0000","last_fullsized":"2023-10-09T06:31:09.219472+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:15.365778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":695,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759971+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:31:02.759971+0000","last_peered":"2023-10-09T06:31:02.759971+0000","last_clean":"2023-10-09T06:31:02.759971+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:31:02.759971+0000","last_undegraded":"2023-10-09T06:31:02.759971+0000","last_fullsized":"2023-10-09T06:31:02.759971+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":88,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898927+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:31:03.898927+0000","last_peered":"2023-10-09T06:31:03.898927+0000","last_clean":"2023-10-09T06:31:03.898927+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:31:03.898927+0000","last_undegraded":"2023-10-09T06:31:03.898927+0000","last_fullsized":"2023-10-09T06:31:03.898927+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":343,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.756794+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:31:02.756794+0000","last_peered":"2023-10-09T06:31:02.756794+0000","last_clean":"2023-10-09T06:31:02.756794+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:31:02.756794+0000","last_undegraded":"2023-10-09T06:31:02.756794+0000","last_fullsized":"2023-10-09T06:31:02.756794+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":12,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220208+0000","last_change":"2023-10-09T06:31:03.217042+0000","last_active":"2023-10-09T06:31:09.220208+0000","last_peered":"2023-10-09T06:31:09.220208+0000","last_clean":"2023-10-09T06:31:09.220208+0000","last_became_active":"2023-10-09T06:31:03.216729+0000","last_became_peered":"2023-10-09T06:31:03.216729+0000","last_unstale":"2023-10-09T06:31:09.220208+0000","last_undegraded":"2023-10-09T06:31:09.220208+0000","last_fullsized":"2023-10-09T06:31:09.220208+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.232295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":675,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.206394+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:31:03.206394+0000","last_peered":"2023-10-09T06:31:03.206394+0000","last_clean":"2023-10-09T06:31:03.206394+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:31:03.206394+0000","last_undegraded":"2023-10-09T06:31:03.206394+0000","last_fullsized":"2023-10-09T06:31:03.206394+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":322,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219563+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:31:09.219563+0000","last_peered":"2023-10-09T06:31:09.219563+0000","last_clean":"2023-10-09T06:31:09.219563+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:31:09.219563+0000","last_undegraded":"2023-10-09T06:31:09.219563+0000","last_fullsized":"2023-10-09T06:31:09.219563+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":340,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.758547+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:31:02.758547+0000","last_peered":"2023-10-09T06:31:02.758547+0000","last_clean":"2023-10-09T06:31:02.758547+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:31:02.758547+0000","last_undegraded":"2023-10-09T06:31:02.758547+0000","last_fullsized":"2023-10-09T06:31:02.758547+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":11,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.762667+0000","last_change":"2023-10-09T06:31:02.762312+0000","last_active":"2023-10-09T06:31:02.762667+0000","last_peered":"2023-10-09T06:31:02.762667+0000","last_clean":"2023-10-09T06:31:02.762667+0000","last_became_active":"2023-10-09T06:31:02.762007+0000","last_became_peered":"2023-10-09T06:31:02.762007+0000","last_unstale":"2023-10-09T06:31:02.762667+0000","last_undegraded":"2023-10-09T06:31:02.762667+0000","last_fullsized":"2023-10-09T06:31:02.762667+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":344,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757002+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:31:02.757002+0000","last_peered":"2023-10-09T06:31:02.757002+0000","last_clean":"2023-10-09T06:31:02.757002+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:31:02.757002+0000","last_undegraded":"2023-10-09T06:31:02.757002+0000","last_fullsized":"2023-10-09T06:31:02.757002+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":323,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759538+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:31:02.759538+0000","last_peered":"2023-10-09T06:31:02.759538+0000","last_clean":"2023-10-09T06:31:02.759538+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:31:02.759538+0000","last_undegraded":"2023-10-09T06:31:02.759538+0000","last_fullsized":"2023-10-09T06:31:02.759538+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":11,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.216145+0000","last_change":"2023-10-09T06:31:03.215765+0000","last_active":"2023-10-09T06:31:03.216145+0000","last_peered":"2023-10-09T06:31:03.216145+0000","last_clean":"2023-10-09T06:31:03.216145+0000","last_became_active":"2023-10-09T06:31:03.215425+0000","last_became_peered":"2023-10-09T06:31:03.215425+0000","last_unstale":"2023-10-09T06:31:03.216145+0000","last_undegraded":"2023-10-09T06:31:03.216145+0000","last_fullsized":"2023-10-09T06:31:03.216145+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":653,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.206431+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:31:03.206431+0000","last_peered":"2023-10-09T06:31:03.206431+0000","last_clean":"2023-10-09T06:31:03.206431+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:31:03.206431+0000","last_undegraded":"2023-10-09T06:31:03.206431+0000","last_fullsized":"2023-10-09T06:31:03.206431+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898600+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:31:03.898600+0000","last_peered":"2023-10-09T06:31:03.898600+0000","last_clean":"2023-10-09T06:31:03.898600+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:31:03.898600+0000","last_undegraded":"2023-10-09T06:31:03.898600+0000","last_fullsized":"2023-10-09T06:31:03.898600+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":315,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.756119+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:31:02.756119+0000","last_peered":"2023-10-09T06:31:02.756119+0000","last_clean":"2023-10-09T06:31:02.756119+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:31:02.756119+0000","last_undegraded":"2023-10-09T06:31:02.756119+0000","last_fullsized":"2023-10-09T06:31:02.756119+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":13,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764691+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:31:08.764691+0000","last_peered":"2023-10-09T06:31:08.764691+0000","last_clean":"2023-10-09T06:31:08.764691+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:31:08.764691+0000","last_undegraded":"2023-10-09T06:31:08.764691+0000","last_fullsized":"2023-10-09T06:31:08.764691+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":627,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220671+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:31:09.220671+0000","last_peered":"2023-10-09T06:31:09.220671+0000","last_clean":"2023-10-09T06:31:09.220671+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:31:09.220671+0000","last_undegraded":"2023-10-09T06:31:09.220671+0000","last_fullsized":"2023-10-09T06:31:09.220671+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":366,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755495+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:31:02.755495+0000","last_peered":"2023-10-09T06:31:02.755495+0000","last_clean":"2023-10-09T06:31:02.755495+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:31:02.755495+0000","last_undegraded":"2023-10-09T06:31:02.755495+0000","last_fullsized":"2023-10-09T06:31:02.755495+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207378+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:31:03.207378+0000","last_peered":"2023-10-09T06:31:03.207378+0000","last_clean":"2023-10-09T06:31:03.207378+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:31:03.207378+0000","last_undegraded":"2023-10-09T06:31:03.207378+0000","last_fullsized":"2023-10-09T06:31:03.207378+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":300,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.754825+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:31:02.754825+0000","last_peered":"2023-10-09T06:31:02.754825+0000","last_clean":"2023-10-09T06:31:02.754825+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:31:02.754825+0000","last_undegraded":"2023-10-09T06:31:02.754825+0000","last_fullsized":"2023-10-09T06:31:02.754825+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":13,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764549+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:31:08.764549+0000","last_peered":"2023-10-09T06:31:08.764549+0000","last_clean":"2023-10-09T06:31:08.764549+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:31:08.764549+0000","last_undegraded":"2023-10-09T06:31:08.764549+0000","last_fullsized":"2023-10-09T06:31:08.764549+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":689,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207002+0000","last_change":"2023-10-09T06:29:15.532600+0000","last_active":"2023-10-09T06:31:03.207002+0000","last_peered":"2023-10-09T06:31:03.207002+0000","last_clean":"2023-10-09T06:31:03.207002+0000","last_became_active":"2023-10-09T06:29:15.532290+0000","last_became_peered":"2023-10-09T06:29:15.532290+0000","last_unstale":"2023-10-09T06:31:03.207002+0000","last_undegraded":"2023-10-09T06:31:03.207002+0000","last_fullsized":"2023-10-09T06:31:03.207002+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:55.742439+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207085+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:31:03.207085+0000","last_peered":"2023-10-09T06:31:03.207085+0000","last_clean":"2023-10-09T06:31:03.207085+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:31:03.207085+0000","last_undegraded":"2023-10-09T06:31:03.207085+0000","last_fullsized":"2023-10-09T06:31:03.207085+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":344,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757837+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:31:02.757837+0000","last_peered":"2023-10-09T06:31:02.757837+0000","last_clean":"2023-10-09T06:31:02.757837+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:31:02.757837+0000","last_undegraded":"2023-10-09T06:31:02.757837+0000","last_fullsized":"2023-10-09T06:31:02.757837+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":330,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755423+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:31:02.755423+0000","last_peered":"2023-10-09T06:31:02.755423+0000","last_clean":"2023-10-09T06:31:02.755423+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:31:02.755423+0000","last_undegraded":"2023-10-09T06:31:02.755423+0000","last_fullsized":"2023-10-09T06:31:02.755423+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":12,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219902+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:31:09.219902+0000","last_peered":"2023-10-09T06:31:09.219902+0000","last_clean":"2023-10-09T06:31:09.219902+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:31:09.219902+0000","last_undegraded":"2023-10-09T06:31:09.219902+0000","last_fullsized":"2023-10-09T06:31:09.219902+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":685,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759922+0000","last_change":"2023-10-09T06:29:15.535685+0000","last_active":"2023-10-09T06:31:02.759922+0000","last_peered":"2023-10-09T06:31:02.759922+0000","last_clean":"2023-10-09T06:31:02.759922+0000","last_became_active":"2023-10-09T06:29:15.535396+0000","last_became_peered":"2023-10-09T06:29:15.535396+0000","last_unstale":"2023-10-09T06:31:02.759922+0000","last_undegraded":"2023-10-09T06:31:02.759922+0000","last_fullsized":"2023-10-09T06:31:02.759922+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:23.142031+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":353,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898647+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:31:03.898647+0000","last_peered":"2023-10-09T06:31:03.898647+0000","last_clean":"2023-10-09T06:31:03.898647+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:31:03.898647+0000","last_undegraded":"2023-10-09T06:31:03.898647+0000","last_fullsized":"2023-10-09T06:31:03.898647+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":339,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.754095+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:31:02.754095+0000","last_peered":"2023-10-09T06:31:02.754095+0000","last_clean":"2023-10-09T06:31:02.754095+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:31:02.754095+0000","last_undegraded":"2023-10-09T06:31:02.754095+0000","last_fullsized":"2023-10-09T06:31:02.754095+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":361,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.756192+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:31:02.756192+0000","last_peered":"2023-10-09T06:31:02.756192+0000","last_clean":"2023-10-09T06:31:02.756192+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:31:02.756192+0000","last_undegraded":"2023-10-09T06:31:02.756192+0000","last_fullsized":"2023-10-09T06:31:02.756192+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":10,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.218593+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:31:03.218593+0000","last_peered":"2023-10-09T06:31:03.218593+0000","last_clean":"2023-10-09T06:31:03.218593+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:31:03.218593+0000","last_undegraded":"2023-10-09T06:31:03.218593+0000","last_fullsized":"2023-10-09T06:31:03.218593+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":50,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207083+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:31:03.207083+0000","last_peered":"2023-10-09T06:31:03.207083+0000","last_clean":"2023-10-09T06:31:03.207083+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:31:03.207083+0000","last_undegraded":"2023-10-09T06:31:03.207083+0000","last_fullsized":"2023-10-09T06:31:03.207083+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":666,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.758572+0000","last_change":"2023-10-09T06:29:15.536561+0000","last_active":"2023-10-09T06:31:02.758572+0000","last_peered":"2023-10-09T06:31:02.758572+0000","last_clean":"2023-10-09T06:31:02.758572+0000","last_became_active":"2023-10-09T06:29:15.535869+0000","last_became_peered":"2023-10-09T06:29:15.535869+0000","last_unstale":"2023-10-09T06:31:02.758572+0000","last_undegraded":"2023-10-09T06:31:02.758572+0000","last_fullsized":"2023-10-09T06:31:02.758572+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:04.229310+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898754+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:31:03.898754+0000","last_peered":"2023-10-09T06:31:03.898754+0000","last_clean":"2023-10-09T06:31:03.898754+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:31:03.898754+0000","last_undegraded":"2023-10-09T06:31:03.898754+0000","last_fullsized":"2023-10-09T06:31:03.898754+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220477+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:31:09.220477+0000","last_peered":"2023-10-09T06:31:09.220477+0000","last_clean":"2023-10-09T06:31:09.220477+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:31:09.220477+0000","last_undegraded":"2023-10-09T06:31:09.220477+0000","last_fullsized":"2023-10-09T06:31:09.220477+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":13,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766251+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:31:08.766251+0000","last_peered":"2023-10-09T06:31:08.766251+0000","last_clean":"2023-10-09T06:31:08.766251+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:31:08.766251+0000","last_undegraded":"2023-10-09T06:31:08.766251+0000","last_fullsized":"2023-10-09T06:31:08.766251+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":648,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.753927+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:31:02.753927+0000","last_peered":"2023-10-09T06:31:02.753927+0000","last_clean":"2023-10-09T06:31:02.753927+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:31:02.753927+0000","last_undegraded":"2023-10-09T06:31:02.753927+0000","last_fullsized":"2023-10-09T06:31:02.753927+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":226,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.758350+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:31:02.758350+0000","last_peered":"2023-10-09T06:31:02.758350+0000","last_clean":"2023-10-09T06:31:02.758350+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:31:02.758350+0000","last_undegraded":"2023-10-09T06:31:02.758350+0000","last_fullsized":"2023-10-09T06:31:02.758350+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764910+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:31:08.764910+0000","last_peered":"2023-10-09T06:31:08.764910+0000","last_clean":"2023-10-09T06:31:08.764910+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:31:08.764910+0000","last_undegraded":"2023-10-09T06:31:08.764910+0000","last_fullsized":"2023-10-09T06:31:08.764910+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":347,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766324+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:31:08.766324+0000","last_peered":"2023-10-09T06:31:08.766324+0000","last_clean":"2023-10-09T06:31:08.766324+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:31:08.766324+0000","last_undegraded":"2023-10-09T06:31:08.766324+0000","last_fullsized":"2023-10-09T06:31:08.766324+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":12,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:04.499808+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:31:04.499808+0000","last_peered":"2023-10-09T06:31:04.499808+0000","last_clean":"2023-10-09T06:31:04.499808+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:31:04.499808+0000","last_undegraded":"2023-10-09T06:31:04.499808+0000","last_fullsized":"2023-10-09T06:31:04.499808+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":657,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219907+0000","last_change":"2023-10-09T06:29:15.549866+0000","last_active":"2023-10-09T06:31:09.219907+0000","last_peered":"2023-10-09T06:31:09.219907+0000","last_clean":"2023-10-09T06:31:09.219907+0000","last_became_active":"2023-10-09T06:29:15.549495+0000","last_became_peered":"2023-10-09T06:29:15.549495+0000","last_unstale":"2023-10-09T06:31:09.219907+0000","last_undegraded":"2023-10-09T06:31:09.219907+0000","last_fullsized":"2023-10-09T06:31:09.219907+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:46.691754+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":226,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.759799+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:31:02.759799+0000","last_peered":"2023-10-09T06:31:02.759799+0000","last_clean":"2023-10-09T06:31:02.759799+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:31:02.759799+0000","last_undegraded":"2023-10-09T06:31:02.759799+0000","last_fullsized":"2023-10-09T06:31:02.759799+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":356,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755350+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:31:02.755350+0000","last_peered":"2023-10-09T06:31:02.755350+0000","last_clean":"2023-10-09T06:31:02.755350+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:31:02.755350+0000","last_undegraded":"2023-10-09T06:31:02.755350+0000","last_fullsized":"2023-10-09T06:31:02.755350+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":345,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764894+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:31:08.764894+0000","last_peered":"2023-10-09T06:31:08.764894+0000","last_clean":"2023-10-09T06:31:08.764894+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:31:08.764894+0000","last_undegraded":"2023-10-09T06:31:08.764894+0000","last_fullsized":"2023-10-09T06:31:08.764894+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":12,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:04.499725+0000","last_change":"2023-10-09T06:31:03.217038+0000","last_active":"2023-10-09T06:31:04.499725+0000","last_peered":"2023-10-09T06:31:04.499725+0000","last_clean":"2023-10-09T06:31:04.499725+0000","last_became_active":"2023-10-09T06:31:03.216747+0000","last_became_peered":"2023-10-09T06:31:03.216747+0000","last_unstale":"2023-10-09T06:31:04.499725+0000","last_undegraded":"2023-10-09T06:31:04.499725+0000","last_fullsized":"2023-10-09T06:31:04.499725+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:22.026356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":646,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.758400+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:31:02.758400+0000","last_peered":"2023-10-09T06:31:02.758400+0000","last_clean":"2023-10-09T06:31:02.758400+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:31:02.758400+0000","last_undegraded":"2023-10-09T06:31:02.758400+0000","last_fullsized":"2023-10-09T06:31:02.758400+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":92,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.899168+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:31:03.899168+0000","last_peered":"2023-10-09T06:31:03.899168+0000","last_clean":"2023-10-09T06:31:03.899168+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:31:03.899168+0000","last_undegraded":"2023-10-09T06:31:03.899168+0000","last_fullsized":"2023-10-09T06:31:03.899168+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":324,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.754315+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:31:02.754315+0000","last_peered":"2023-10-09T06:31:02.754315+0000","last_clean":"2023-10-09T06:31:02.754315+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:31:02.754315+0000","last_undegraded":"2023-10-09T06:31:02.754315+0000","last_fullsized":"2023-10-09T06:31:02.754315+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898572+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:31:03.898572+0000","last_peered":"2023-10-09T06:31:03.898572+0000","last_clean":"2023-10-09T06:31:03.898572+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:31:03.898572+0000","last_undegraded":"2023-10-09T06:31:03.898572+0000","last_fullsized":"2023-10-09T06:31:03.898572+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":12,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219700+0000","last_change":"2023-10-09T06:31:03.215173+0000","last_active":"2023-10-09T06:31:09.219700+0000","last_peered":"2023-10-09T06:31:09.219700+0000","last_clean":"2023-10-09T06:31:09.219700+0000","last_became_active":"2023-10-09T06:31:03.214809+0000","last_became_peered":"2023-10-09T06:31:03.214809+0000","last_unstale":"2023-10-09T06:31:09.219700+0000","last_undegraded":"2023-10-09T06:31:09.219700+0000","last_fullsized":"2023-10-09T06:31:09.219700+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:12.323433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":693,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898993+0000","last_change":"2023-10-09T06:28:14.734086+0000","last_active":"2023-10-09T06:31:03.898993+0000","last_peered":"2023-10-09T06:31:03.898993+0000","last_clean":"2023-10-09T06:31:03.898993+0000","last_became_active":"2023-10-09T06:28:14.733702+0000","last_became_peered":"2023-10-09T06:28:14.733702+0000","last_unstale":"2023-10-09T06:31:03.898993+0000","last_undegraded":"2023-10-09T06:31:03.898993+0000","last_fullsized":"2023-10-09T06:31:03.898993+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.831911+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":349,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220664+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:31:09.220664+0000","last_peered":"2023-10-09T06:31:09.220664+0000","last_clean":"2023-10-09T06:31:09.220664+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:31:09.220664+0000","last_undegraded":"2023-10-09T06:31:09.220664+0000","last_fullsized":"2023-10-09T06:31:09.220664+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":328,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766078+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:31:08.766078+0000","last_peered":"2023-10-09T06:31:08.766078+0000","last_clean":"2023-10-09T06:31:08.766078+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:31:08.766078+0000","last_undegraded":"2023-10-09T06:31:08.766078+0000","last_fullsized":"2023-10-09T06:31:08.766078+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":326,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757185+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:31:02.757185+0000","last_peered":"2023-10-09T06:31:02.757185+0000","last_clean":"2023-10-09T06:31:02.757185+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:31:02.757185+0000","last_undegraded":"2023-10-09T06:31:02.757185+0000","last_fullsized":"2023-10-09T06:31:02.757185+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":330,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764822+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:31:08.764822+0000","last_peered":"2023-10-09T06:31:08.764822+0000","last_clean":"2023-10-09T06:31:08.764822+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:31:08.764822+0000","last_undegraded":"2023-10-09T06:31:08.764822+0000","last_fullsized":"2023-10-09T06:31:08.764822+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":349,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.755265+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:31:02.755265+0000","last_peered":"2023-10-09T06:31:02.755265+0000","last_clean":"2023-10-09T06:31:02.755265+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:31:02.755265+0000","last_undegraded":"2023-10-09T06:31:02.755265+0000","last_fullsized":"2023-10-09T06:31:02.755265+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":54,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207220+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:31:03.207220+0000","last_peered":"2023-10-09T06:31:03.207220+0000","last_clean":"2023-10-09T06:31:03.207220+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:31:03.207220+0000","last_undegraded":"2023-10-09T06:31:03.207220+0000","last_fullsized":"2023-10-09T06:31:03.207220+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":370,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.757742+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:31:02.757742+0000","last_peered":"2023-10-09T06:31:02.757742+0000","last_clean":"2023-10-09T06:31:02.757742+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:31:02.757742+0000","last_undegraded":"2023-10-09T06:31:02.757742+0000","last_fullsized":"2023-10-09T06:31:02.757742+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.207327+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:31:03.207327+0000","last_peered":"2023-10-09T06:31:03.207327+0000","last_clean":"2023-10-09T06:31:03.207327+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:31:03.207327+0000","last_undegraded":"2023-10-09T06:31:03.207327+0000","last_fullsized":"2023-10-09T06:31:03.207327+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":380,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.758935+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:31:02.758935+0000","last_peered":"2023-10-09T06:31:02.758935+0000","last_clean":"2023-10-09T06:31:02.758935+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:31:02.758935+0000","last_undegraded":"2023-10-09T06:31:02.758935+0000","last_fullsized":"2023-10-09T06:31:02.758935+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":62,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.899068+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:31:03.899068+0000","last_peered":"2023-10-09T06:31:03.899068+0000","last_clean":"2023-10-09T06:31:03.899068+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:31:03.899068+0000","last_undegraded":"2023-10-09T06:31:03.899068+0000","last_fullsized":"2023-10-09T06:31:03.899068+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":80,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-10-09T06:31:03.898781+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:31:03.898781+0000","last_peered":"2023-10-09T06:31:03.898781+0000","last_clean":"2023-10-09T06:31:03.898781+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:31:03.898781+0000","last_undegraded":"2023-10-09T06:31:03.898781+0000","last_fullsized":"2023-10-09T06:31:03.898781+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":358,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766793+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:31:08.766793+0000","last_peered":"2023-10-09T06:31:08.766793+0000","last_clean":"2023-10-09T06:31:08.766793+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:31:08.766793+0000","last_undegraded":"2023-10-09T06:31:08.766793+0000","last_fullsized":"2023-10-09T06:31:08.766793+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":323,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T06:31:02.756980+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:31:02.756980+0000","last_peered":"2023-10-09T06:31:02.756980+0000","last_clean":"2023-10-09T06:31:02.756980+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:31:02.756980+0000","last_undegraded":"2023-10-09T06:31:02.756980+0000","last_fullsized":"2023-10-09T06:31:02.756980+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234406,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":912328,"kb_used_data":3904,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":92832824,"statfs":{"total":95995035648,"available":95060811776,"internally_reserved":0,"allocated":3997696,"data_stored":3037536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":4.9039999999999999,"15min":4.9569999999999999},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":4.2549999999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.9370000000000003,"15min":5.0670000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":3.2930000000000001,"5min":215.577,"15min":219.399},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":7.7770000000000001,"5min":6.6429999999999998,"15min":4.2619999999999996},"min":{"1min":0.318,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":45.502000000000002,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":5.8609999999999998,"5min":5.7720000000000002,"15min":3.794},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.447000000000003,"5min":216.22200000000001,"15min":216.22200000000001},"last":0.45000000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":4.5220000000000002,"15min":6.0960000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":6.3319999999999999,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.1200000000000001,"5min":4.7999999999999998,"15min":6.2789999999999999},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.844000000000001,"5min":216.25200000000001,"15min":216.25200000000001},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":7.6580000000000004,"15min":3.899},"min":{"1min":0.307,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":187.34200000000001,"5min":215.922,"15min":215.922},"last":0.371},{"interface":"front","average":{"1min":16.768000000000001,"5min":7.7969999999999997,"15min":3.9900000000000002},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":187.08199999999999,"5min":215.94900000000001,"15min":215.94900000000001},"last":0.505}]},{"osd":4,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":4.1120000000000001,"15min":2.6160000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":6.1689999999999996,"5min":218.446,"15min":218.446},"last":0.314},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.9889999999999999,"15min":2.4079999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.23699999999999999},"max":{"1min":4.891,"5min":216.172,"15min":216.172},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":4.2350000000000003,"15min":2.661},"min":{"1min":0.31,"5min":0.31,"15min":0.22900000000000001},"max":{"1min":8.8170000000000002,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.603,"5min":4.1079999999999997,"15min":2.4809999999999999},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":6.3019999999999996,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":4.343,"15min":2.3929999999999998},"min":{"1min":0.33300000000000002,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":15.162000000000001,"5min":218.19999999999999,"15min":218.19999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.5,"5min":3.9390000000000001,"15min":2.161},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.2919999999999998,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.54100000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496042,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902588,"kb_used_data":3572,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92842564,"statfs":{"total":95995035648,"available":95070785536,"internally_reserved":0,"allocated":3657728,"data_stored":2717240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.627,"15min":3.0779999999999998},"min":{"1min":0.24399999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.282,"5min":22.321999999999999,"15min":110.926},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.8530000000000002,"5min":2.1320000000000001,"15min":3.3980000000000001},"min":{"1min":0.29399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2870000000000008,"5min":22.338000000000001,"15min":110.911},"last":0.46800000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3760000000000003,"5min":6.9649999999999999,"15min":5.0010000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":82.159000000000006,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.623},{"interface":"front","average":{"1min":7.1950000000000003,"5min":7.0590000000000002,"15min":4.8810000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":82.174000000000007,"5min":292.904,"15min":292.904},"last":0.54300000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.109,"15min":2.6459999999999999},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.3670000000000009,"5min":20.331,"15min":75.191999999999993},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.335,"5min":1.748,"15min":2.351},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":8.2400000000000002,"5min":8.6069999999999993,"15min":75.120000000000005},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":8.4819999999999993,"5min":3.4540000000000002,"15min":2.8650000000000002},"min":{"1min":0.32700000000000001,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":91.174000000000007,"5min":91.174000000000007,"15min":110.358},"last":0.438},{"interface":"front","average":{"1min":8.7750000000000004,"5min":3.3620000000000001,"15min":2.9199999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.311},"max":{"1min":91.438999999999993,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.59499999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.361,"15min":1.3220000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":12.032,"5min":12.666,"15min":17.643999999999998},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.97499999999999998,"15min":1.1739999999999999},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":5.1280000000000001,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.3169999999999999,"15min":1.429},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.9830000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.448,"5min":1.379,"15min":1.369},"min":{"1min":0.33300000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.8000000000000007,"5min":16.536999999999999,"15min":24.401},"last":0.67500000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":4.0529999999999999,"15min":2.528},"min":{"1min":0.314,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":7.8899999999999997,"5min":122.937,"15min":122.937},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.3369999999999997,"15min":2.6709999999999998},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.32800000000000001},"max":{"1min":10.423,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.61099999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402779,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":962948,"kb_used_data":305212,"kb_used_omap":4,"kb_used_meta":657723,"kb_avail":92782204,"statfs":{"total":95995035648,"available":95008976896,"internally_reserved":0,"allocated":312537088,"data_stored":310879143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":673509262},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":9.8030000000000008,"5min":6.2869999999999999,"15min":5.2409999999999997},"min":{"1min":0.22,"5min":0.189,"15min":0.189},"max":{"1min":60.973999999999997,"5min":269.79199999999997,"15min":269.79199999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":8.8480000000000008,"5min":6.4539999999999997,"15min":5.6299999999999999},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.17899999999999999},"max":{"1min":60.201000000000001,"5min":271.20999999999998,"15min":271.20999999999998},"last":0.47499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.2789999999999999,"15min":2.9540000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":6.9390000000000001,"5min":7.1449999999999996,"15min":139.75399999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3759999999999999,"15min":3.6970000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":6.673,"5min":12.077,"15min":140.29900000000001},"last":0.437}]},{"osd":3,"last update":"Mon Oct 9 06:30:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":3.2040000000000002,"15min":3.2679999999999998},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":13.214,"5min":31.062000000000001,"15min":153.18199999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.6389999999999998,"5min":2.2389999999999999,"15min":2.444},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.128,"5min":27.657,"15min":152.52000000000001},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.883,"15min":3.3929999999999998},"min":{"1min":0.312,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":13.087999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.823,"15min":3.2989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.1300000000000008,"5min":11.974,"15min":199.32900000000001},"last":0.60099999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":1.962,"15min":2.0699999999999998},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":52.512999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.81999999999999995},{"interface":"front","average":{"1min":4.665,"5min":2.3260000000000001,"15min":2.6000000000000001},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":52.664999999999999,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0220000000000002,"5min":2.7250000000000001,"15min":2.8969999999999998},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":52.488,"5min":52.488,"15min":77.292000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":6.556,"5min":2.952,"15min":2.843},"min":{"1min":0.38200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":52.558,"5min":52.558,"15min":77.378},"last":0.624}]},{"osd":7,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0129999999999999,"5min":3.8399999999999999,"15min":3.0470000000000002},"min":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.32500000000000001},"max":{"1min":52.601999999999997,"5min":55.231000000000002,"15min":77.456999999999994},"last":0.65300000000000002},{"interface":"front","average":{"1min":4.8040000000000003,"5min":3.1800000000000002,"15min":2.5179999999999998},"min":{"1min":0.42199999999999999,"5min":0.35799999999999998,"15min":0.34399999999999997},"max":{"1min":52.579999999999998,"5min":55.253999999999998,"15min":77.359999999999999},"last":0.61399999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024213,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110520,"kb_used_data":318384,"kb_used_omap":5,"kb_used_meta":792122,"kb_avail":92634632,"statfs":{"total":95995035648,"available":94857863168,"internally_reserved":0,"allocated":326025216,"data_stored":324096144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":811133562},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":3.6059999999999999,"15min":3.3719999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.16600000000000001},"max":{"1min":8.1389999999999993,"5min":191.446,"15min":191.446},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.6419999999999999,"5min":3.8159999999999998,"15min":3.6709999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.17499999999999999},"max":{"1min":12.959,"5min":191.393,"15min":191.393},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.7309999999999999,"15min":2.3180000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":16.936,"5min":57.162999999999997,"15min":57.162999999999997},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.6419999999999999,"5min":3.524,"15min":2.6960000000000002},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":15.794,"5min":57.585000000000001,"15min":57.585000000000001},"last":0.25600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:07 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":7.8419999999999996,"15min":5.4000000000000004},"min":{"1min":0.26800000000000002,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":187.172,"5min":208.11199999999999,"15min":208.11199999999999},"last":0.308},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.4950000000000001,"15min":5.391},"min":{"1min":0.31900000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":194.851,"5min":208.25899999999999,"15min":208.25899999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.518,"15min":1.429},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":7.21,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.7470000000000001,"15min":1.798},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":10.903,"5min":18.081,"15min":18.081},"last":0.65200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.7130000000000001,"15min":1.806},"min":{"1min":0.317,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":7.2409999999999997,"5min":10.865,"15min":62.984999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.9970000000000001,"15min":2.2469999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":10.603,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.57999999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":4.2619999999999996,"15min":2.8980000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":12.522,"5min":120.572,"15min":120.572},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.758,"5min":3.827,"15min":2.3330000000000002},"min":{"1min":0.371,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":11.093,"5min":120.64400000000001,"15min":120.64400000000001},"last":0.70899999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":3.1840000000000002,"15min":2.4969999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":8.8059999999999992,"5min":122.018,"15min":122.018},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.415,"5min":3.6539999999999999,"15min":2.8730000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":10.563000000000001,"5min":122.32299999999999,"15min":122.32299999999999},"last":0.68799999999999994}]}]},{"osd":2,"up_from":128,"seq":549755814082,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525800,"kb_used_data":4576,"kb_used_omap":7,"kb_used_meta":521208,"kb_avail":93219352,"statfs":{"total":95995035648,"available":95456616448,"internally_reserved":0,"allocated":4685824,"data_stored":3058910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":533717073},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":4.2409999999999997,"15min":4.7370000000000001},"min":{"1min":0.19500000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":16.91,"5min":162.21100000000001,"15min":163.40100000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":3.4180000000000001,"5min":4.3630000000000004,"15min":4.4260000000000002},"min":{"1min":0.30099999999999999,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":16.888999999999999,"5min":168.49700000000001,"15min":168.49700000000001},"last":0.33400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":3.3719999999999999,"15min":4.1740000000000004},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":24.312000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.714,"5min":4.423,"15min":4.5380000000000003},"min":{"1min":0.29399999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.259,"5min":91.721999999999994,"15min":222.19399999999999},"last":0.35899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.3370000000000002,"15min":2.8090000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8499999999999996,"5min":91.644000000000005,"15min":91.644000000000005},"last":0.255},{"interface":"front","average":{"1min":1.841,"5min":3.9809999999999999,"15min":3.056},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.9059999999999997,"5min":91.685000000000002,"15min":91.685000000000002},"last":0.30199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":1.7589999999999999,"15min":2.081},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":13.023999999999999,"5min":17.776,"15min":90.938999999999993},"last":0.497},{"interface":"front","average":{"1min":3.0880000000000001,"5min":2.1560000000000001,"15min":2.375},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":90.950000000000003},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.0339999999999998,"15min":1.605},"min":{"1min":0.315,"5min":0.315,"15min":0.309},"max":{"1min":13.102,"5min":17.800000000000001,"15min":17.800000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":3.2930000000000001,"5min":2.2709999999999999,"15min":1.978},"min":{"1min":0.377,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.48399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":3.585,"5min":2.3650000000000002,"15min":2.5529999999999999},"min":{"1min":0.41399999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":12.387,"5min":12.387,"15min":138.291},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.639,"5min":1.6459999999999999,"15min":2.1829999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":9.1579999999999995,"5min":12.643000000000001,"15min":138.01599999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.052,"15min":1.968},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":9.1859999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.04,"5min":2.113,"15min":2.0110000000000001},"min":{"1min":0.42099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.167,"5min":17.734999999999999,"15min":55.731000000000002},"last":0.45300000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127279,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":827988,"kb_used_data":329164,"kb_used_omap":2,"kb_used_meta":498813,"kb_avail":92917164,"statfs":{"total":95995035648,"available":95147175936,"internally_reserved":0,"allocated":337063936,"data_stored":335507687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":510784565},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.4260000000000002,"5min":2.673,"15min":3.8820000000000001},"min":{"1min":0.21099999999999999,"5min":0.19800000000000001,"15min":0.19600000000000001},"max":{"1min":11.813000000000001,"5min":34.249000000000002,"15min":187.852},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0779999999999998,"5min":2.4870000000000001,"15min":4.2140000000000004},"min":{"1min":0.20799999999999999,"5min":0.184,"15min":0.184},"max":{"1min":11.93,"5min":34.276000000000003,"15min":187.923},"last":0.32500000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2599999999999998,"5min":2.3639999999999999,"15min":3.3399999999999999},"min":{"1min":0.23699999999999999,"5min":0.214,"15min":0.214},"max":{"1min":15.814,"5min":34.646999999999998,"15min":187.89099999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":4.8360000000000003,"5min":3.363,"15min":2.7650000000000001},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.02,"5min":34.207999999999998,"15min":67.028000000000006},"last":0.312}]},{"osd":2,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":2.2949999999999999,"15min":3.0619999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.2930000000000001,"5min":34.307000000000002,"15min":187.76400000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.6150000000000002,"5min":2.7890000000000001,"15min":2.798},"min":{"1min":0.27900000000000003,"5min":0.23599999999999999,"15min":0.22500000000000001},"max":{"1min":11.425000000000001,"5min":35.055999999999997,"15min":187.79499999999999},"last":0.34999999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.9670000000000001,"5min":2.0640000000000001,"15min":1.532},"min":{"1min":0.31,"5min":0.29999999999999999,"15min":0.27600000000000002},"max":{"1min":25.591999999999999,"5min":25.591999999999999,"15min":25.591999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.7669999999999999,"5min":1.9610000000000001,"15min":2.2480000000000002},"min":{"1min":0.36299999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":25.544,"5min":25.544,"15min":187.67400000000001},"last":0.51300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":1.7070000000000001,"15min":2.2930000000000001},"min":{"1min":0.379,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":6.4480000000000004,"5min":8.6159999999999997,"15min":187.833},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.395,"5min":1.427,"15min":2.129},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":6.7649999999999997,"5min":9.8469999999999995,"15min":187.952},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":10.112,"5min":3.5539999999999998,"15min":2.8889999999999998},"min":{"1min":0.379,"5min":0.374,"15min":0.33700000000000002},"max":{"1min":130.39599999999999,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":9.0419999999999998,"5min":3.0600000000000001,"15min":2.6190000000000002},"min":{"1min":0.36099999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":130.67699999999999,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:30:11 2023","interfaces":[{"interface":"back","average":{"1min":12.199999999999999,"5min":3.8879999999999999,"15min":2.871},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.34100000000000003},"max":{"1min":121.613,"5min":121.613,"15min":192.20400000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":11.423999999999999,"5min":3.875,"15min":2.9780000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":121.559,"5min":121.559,"15min":187.86500000000001},"last":0.44700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019315,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886268,"kb_used_data":3572,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":92858884,"statfs":{"total":95995035648,"available":95087497216,"internally_reserved":0,"allocated":3657728,"data_stored":2709499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.696,"15min":2.2210000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":18.625,"5min":37.485999999999997,"15min":76.944999999999993},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.962,"5min":1.77,"15min":2.4740000000000002},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.654,"5min":37.502000000000002,"15min":76.988},"last":0.503}]},{"osd":1,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":10.651999999999999,"5min":4.3789999999999996,"15min":2.968},"min":{"1min":0.37,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":128.24299999999999,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":10.84,"5min":4.4909999999999997,"15min":3.133},"min":{"1min":0.43099999999999999,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":128.21600000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":3.0659999999999998,"15min":2.48},"min":{"1min":0.309,"5min":0.309,"15min":0.255},"max":{"1min":7.101,"5min":47.125,"15min":69.640000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.601,"5min":2.7080000000000002,"15min":2.508},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.9870000000000001,"5min":47.151000000000003,"15min":69.676000000000002},"last":0.433}]},{"osd":3,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":21.917000000000002,"5min":6.8090000000000002,"15min":4.2279999999999998},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":133.38900000000001,"5min":133.38900000000001,"15min":133.38900000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":21.408000000000001,"5min":6.3840000000000003,"15min":4.0019999999999998},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":132.57599999999999,"5min":132.57599999999999,"15min":132.57599999999999},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.0329999999999999,"15min":1.1200000000000001},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":6.0199999999999996,"5min":6.2670000000000003,"15min":10.673},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.0249999999999999,"15min":1.111},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":5.7000000000000002,"5min":7.2249999999999996,"15min":12.552},"last":0.46500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.4660000000000002,"15min":1.3340000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.0629999999999997,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.8029999999999999,"15min":1.7989999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.7210000000000001,"5min":163.381,"15min":163.381},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.0149999999999999,"15min":1.506},"min":{"1min":0.371,"5min":0.312,"15min":0.308},"max":{"1min":5.9850000000000003,"5min":6.2679999999999998,"15min":62.658000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.82399999999999995,"15min":1.4770000000000001},"min":{"1min":0.40899999999999997,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":3.8199999999999998,"5min":4.2599999999999998,"15min":62.573},"last":0.54200000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757680,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905844,"kb_used_data":4140,"kb_used_omap":0,"kb_used_meta":901696,"kb_avail":92839308,"statfs":{"total":95995035648,"available":95067451392,"internally_reserved":0,"allocated":4239360,"data_stored":3295575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923336704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.0990000000000002,"15min":3.4900000000000002},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":3.819,"5min":131.87200000000001,"15min":131.87200000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.8769999999999998,"15min":3.4260000000000002},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":3.6709999999999998,"5min":131.80199999999999,"15min":131.80199999999999},"last":0.51000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":9.0109999999999992,"5min":4.9610000000000003,"15min":2.843},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23400000000000001},"max":{"1min":87.795000000000002,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.497},{"interface":"front","average":{"1min":10.266999999999999,"5min":5.2930000000000001,"15min":3.0350000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":87.856999999999999,"5min":126.931,"15min":126.931},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3599999999999999,"5min":2.52,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":16.5,"5min":51.393000000000001,"15min":81.260000000000005},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.282,"5min":2.1789999999999998,"15min":2.1840000000000002},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5679999999999996,"5min":51.432000000000002,"15min":81.245000000000005},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5049999999999999,"5min":9.5310000000000006,"15min":5.4580000000000002},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":92.521000000000001,"5min":144.56399999999999,"15min":190.411},"last":0.439},{"interface":"front","average":{"1min":5.4720000000000004,"5min":9.4909999999999997,"15min":5.3449999999999998},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":94.492000000000004,"5min":144.62799999999999,"15min":190.70699999999999},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.0649999999999999,"15min":1.3759999999999999},"min":{"1min":0.311,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9119999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.90400000000000003,"15min":1.1619999999999999},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2610000000000001,"5min":6.2610000000000001,"15min":32.82},"last":0.57099999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3080000000000001,"15min":1.5509999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":3.6859999999999999,"5min":6.2080000000000002,"15min":38.902000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.395,"15min":1.5920000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.6340000000000003,"5min":8.3659999999999997,"15min":35.774000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:30:14 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.339,"15min":1.524},"min":{"1min":0.46999999999999997,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":7.4290000000000003,"5min":10.461,"15min":30.658999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.2679999999999998,"5min":1.329,"15min":1.502},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":6.2320000000000002,"5min":6.2320000000000002,"15min":30.683},"last":0.48599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:31:12.363 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:31:12.363 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:31:12.363 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T06:31:12.364 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:31:12.428 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:12.464 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:12.531 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:12.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:12.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:12.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:12.677 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:12.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2885 2869 179.305 179.312 0.00497485 0.0055601 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[212750]: pgmap v1128: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[212750]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[220921]: pgmap v1128: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[220921]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:13.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:12 smithi150 ceph-mon[194178]: pgmap v1128: 99 pgs: 99 active+clean; 582 KiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-09T06:31:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:12 smithi150 ceph-mon[194178]: from='client.36426 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:13.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:13.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:13.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 5972 5956 186.108 192.938 0.00381536 0.00536252 2023-10-09T06:31:13.997 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:14.034 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:14.078 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:14.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:14.183 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:14.209 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:14.250 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:14.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:14.358 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:14.406 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:14.468 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:14.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:14.575 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:14.615 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:14.682 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:14.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:14.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:14.789 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:14.819 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:14.857 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:14.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 8041 8025 167.166 129.312 0.00435497 0.00596169 2023-10-09T06:31:14.992 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:14.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:14 smithi107 ceph-mon[212750]: pgmap v1129: 99 pgs: 99 active+clean; 41 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 5.1 MiB/s wr, 81 op/s 2023-10-09T06:31:15.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:14 smithi107 ceph-mon[220921]: pgmap v1129: 99 pgs: 99 active+clean; 41 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 5.1 MiB/s wr, 81 op/s 2023-10-09T06:31:15.100 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:15.150 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:14 smithi150 ceph-mon[194178]: pgmap v1129: 99 pgs: 99 active+clean; 41 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 5.1 MiB/s wr, 81 op/s 2023-10-09T06:31:15.228 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:15.312 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:15.323 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:15.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:15.351 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:15.392 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:15.429 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:15.431 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:15.479 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:15.496 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:15.523 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:15.584 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:15.616 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:15.633 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.633 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.634 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:31:15.661 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:15.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:15.680 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:15.680 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:15.768 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:15.846 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:15.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 10274 10258 160.262 139.562 0.00647411 0.00623179 2023-10-09T06:31:16.006 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3347357466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:16.006 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3347357466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:16.006 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:16.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:16.110 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:16.147 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3347357466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:16.151 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:16.239 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:16.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:16.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:16.344 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:16.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:16.455 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:16.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:16.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:16.604 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:16.674 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:16.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:16.784 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:16.823 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:16.888 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:16.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:16.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 13150 13134 164.156 179.75 0.00357067 0.00608487 2023-10-09T06:31:16.995 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:17.059 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[212750]: pgmap v1130: 99 pgs: 99 active+clean; 110 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s 2023-10-09T06:31:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:17.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:17.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[220921]: pgmap v1130: 99 pgs: 99 active+clean; 110 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s 2023-10-09T06:31:17.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:17.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:17.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:16 smithi150 ceph-mon[194178]: pgmap v1130: 99 pgs: 99 active+clean; 110 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 218 op/s 2023-10-09T06:31:17.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:17.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 15559 15543 161.887 150.562 0.00392186 0.00617101 2023-10-09T06:31:18.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 17123 17107 152.723 97.75 0.00659042 0.00654244 2023-10-09T06:31:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:18 smithi107 ceph-mon[212750]: pgmap v1131: 99 pgs: 99 active+clean; 473 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T06:31:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:18 smithi107 ceph-mon[220921]: pgmap v1131: 99 pgs: 99 active+clean; 473 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T06:31:19.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:18 smithi150 ceph-mon[194178]: pgmap v1131: 99 pgs: 99 active+clean; 473 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2023-10-09T06:31:19.540 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:19.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:19.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:19.651 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:19.683 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:19.750 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:19.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:19.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:19.865 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:31:19.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 18844 18828 147.076 107.562 0.00850326 0.00678704 2023-10-09T06:31:19.945 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:20.078 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:20.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:20.186 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:20.237 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:20.317 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:20.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:20.378 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:20.423 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:20.467 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:20.540 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:20.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:20.643 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:20.690 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:20.752 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:20.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:20.834 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:20 smithi107 ceph-mon[212750]: pgmap v1132: 99 pgs: 99 active+clean; 717 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:31:20.834 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:20 smithi107 ceph-mon[220921]: pgmap v1132: 99 pgs: 99 active+clean; 717 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:31:20.857 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:20.934 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:20.938 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 20789 20773 144.24 121.562 0.00912991 0.00692503 2023-10-09T06:31:21.070 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:21.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:20 smithi150 ceph-mon[194178]: pgmap v1132: 99 pgs: 99 active+clean; 717 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:31:21.177 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:21.217 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:21.291 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:21.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:21.299 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:31:21.305 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:31:21.396 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:21.469 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:21.577 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:31:21.620 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:21.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:21.728 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:21.767 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:21.831 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:21.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:21.873 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:31:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4259375629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4259375629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:21.936 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:21.939 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 22829 22813 142.552 127.5 0.00683048 0.00700884 2023-10-09T06:31:21.970 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:22.030 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:22.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4259375629' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:22.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:22.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:22.136 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:22.193 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:22] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T06:31:22.200 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:22.940 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 24772 24756 140.629 121.438 0.00480568 0.00710486 2023-10-09T06:31:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: from='client.46358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: pgmap v1133: 99 pgs: 99 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:31:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:23.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:31:23.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[220921]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: from='client.46358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: pgmap v1133: 99 pgs: 99 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:31:23.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:23.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:23.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:31:23.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:22 smithi107 ceph-mon[212750]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: from='client.46358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: pgmap v1133: 99 pgs: 99 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T06:31:23.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:23.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:23.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: osdmap e366: 8 total, 8 up, 8 in 2023-10-09T06:31:23.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:22 smithi150 ceph-mon[194178]: failed to encode map e366 with expected crc 2023-10-09T06:31:23.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 26531 26515 138.071 109.938 0.00700726 0.00723633 2023-10-09T06:31:24.024 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1132,"stamp":"2023-10-09T06:31:19.317960+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":752097184,"num_objects":11491,"num_object_clones":0,"num_object_copies":34473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11528,"num_write_kb":733917,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":600556,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13293,"ondisk_log_size":13293,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8947756,"kb_used_data":2894188,"kb_used_omap":20,"kb_used_meta":6053483,"kb_avail":741013460,"statfs":{"total":767960285184,"available":758797783040,"internally_reserved":0,"allocated":2963648512,"data_stored":2953063756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6198767246},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":751501312,"num_objects":11467,"num_object_clones":0,"num_object_copies":34401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11467,"num_write_kb":733888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004645"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":344,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220317+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:31:09.220317+0000","last_peered":"2023-10-09T06:31:09.220317+0000","last_clean":"2023-10-09T06:31:09.220317+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:31:09.220317+0000","last_undegraded":"2023-10-09T06:31:09.220317+0000","last_fullsized":"2023-10-09T06:31:09.220317+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217897+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:31:09.217897+0000","last_peered":"2023-10-09T06:31:09.217897+0000","last_clean":"2023-10-09T06:31:09.217897+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:31:09.217897+0000","last_undegraded":"2023-10-09T06:31:09.217897+0000","last_fullsized":"2023-10-09T06:31:09.217897+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":357,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765222+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:31:08.765222+0000","last_peered":"2023-10-09T06:31:08.765222+0000","last_clean":"2023-10-09T06:31:08.765222+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:31:08.765222+0000","last_undegraded":"2023-10-09T06:31:08.765222+0000","last_fullsized":"2023-10-09T06:31:08.765222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":361,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766165+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:31:08.766165+0000","last_peered":"2023-10-09T06:31:08.766165+0000","last_clean":"2023-10-09T06:31:08.766165+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:31:08.766165+0000","last_undegraded":"2023-10-09T06:31:08.766165+0000","last_fullsized":"2023-10-09T06:31:08.766165+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217402+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:31:09.217402+0000","last_peered":"2023-10-09T06:31:09.217402+0000","last_clean":"2023-10-09T06:31:09.217402+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:31:09.217402+0000","last_undegraded":"2023-10-09T06:31:09.217402+0000","last_fullsized":"2023-10-09T06:31:09.217402+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":354,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220197+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:31:09.220197+0000","last_peered":"2023-10-09T06:31:09.220197+0000","last_clean":"2023-10-09T06:31:09.220197+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:31:09.220197+0000","last_undegraded":"2023-10-09T06:31:09.220197+0000","last_fullsized":"2023-10-09T06:31:09.220197+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766805+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:31:08.766805+0000","last_peered":"2023-10-09T06:31:08.766805+0000","last_clean":"2023-10-09T06:31:08.766805+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:31:08.766805+0000","last_undegraded":"2023-10-09T06:31:08.766805+0000","last_fullsized":"2023-10-09T06:31:08.766805+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217582+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:31:09.217582+0000","last_peered":"2023-10-09T06:31:09.217582+0000","last_clean":"2023-10-09T06:31:09.217582+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:31:09.217582+0000","last_undegraded":"2023-10-09T06:31:09.217582+0000","last_fullsized":"2023-10-09T06:31:09.217582+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.115086+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:31:09.115086+0000","last_peered":"2023-10-09T06:31:09.115086+0000","last_clean":"2023-10-09T06:31:09.115086+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:31:09.115086+0000","last_undegraded":"2023-10-09T06:31:09.115086+0000","last_fullsized":"2023-10-09T06:31:09.115086+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217817+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:31:09.217817+0000","last_peered":"2023-10-09T06:31:09.217817+0000","last_clean":"2023-10-09T06:31:09.217817+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:31:09.217817+0000","last_undegraded":"2023-10-09T06:31:09.217817+0000","last_fullsized":"2023-10-09T06:31:09.217817+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":246,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221669+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:31:09.221669+0000","last_peered":"2023-10-09T06:31:09.221669+0000","last_clean":"2023-10-09T06:31:09.221669+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:31:09.221669+0000","last_undegraded":"2023-10-09T06:31:09.221669+0000","last_fullsized":"2023-10-09T06:31:09.221669+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":298,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766475+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:31:08.766475+0000","last_peered":"2023-10-09T06:31:08.766475+0000","last_clean":"2023-10-09T06:31:08.766475+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:31:08.766475+0000","last_undegraded":"2023-10-09T06:31:08.766475+0000","last_fullsized":"2023-10-09T06:31:08.766475+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":371,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219797+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:31:09.219797+0000","last_peered":"2023-10-09T06:31:09.219797+0000","last_clean":"2023-10-09T06:31:09.219797+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:31:09.219797+0000","last_undegraded":"2023-10-09T06:31:09.219797+0000","last_fullsized":"2023-10-09T06:31:09.219797+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":335,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220550+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:31:09.220550+0000","last_peered":"2023-10-09T06:31:09.220550+0000","last_clean":"2023-10-09T06:31:09.220550+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:31:09.220550+0000","last_undegraded":"2023-10-09T06:31:09.220550+0000","last_fullsized":"2023-10-09T06:31:09.220550+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":214,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221327+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:31:09.221327+0000","last_peered":"2023-10-09T06:31:09.221327+0000","last_clean":"2023-10-09T06:31:09.221327+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:31:09.221327+0000","last_undegraded":"2023-10-09T06:31:09.221327+0000","last_fullsized":"2023-10-09T06:31:09.221327+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217886+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:31:09.217886+0000","last_peered":"2023-10-09T06:31:09.217886+0000","last_clean":"2023-10-09T06:31:09.217886+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:31:09.217886+0000","last_undegraded":"2023-10-09T06:31:09.217886+0000","last_fullsized":"2023-10-09T06:31:09.217886+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":359,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764868+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:31:08.764868+0000","last_peered":"2023-10-09T06:31:08.764868+0000","last_clean":"2023-10-09T06:31:08.764868+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:31:08.764868+0000","last_undegraded":"2023-10-09T06:31:08.764868+0000","last_fullsized":"2023-10-09T06:31:08.764868+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.115037+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:31:09.115037+0000","last_peered":"2023-10-09T06:31:09.115037+0000","last_clean":"2023-10-09T06:31:09.115037+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:31:09.115037+0000","last_undegraded":"2023-10-09T06:31:09.115037+0000","last_fullsized":"2023-10-09T06:31:09.115037+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":208,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221122+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:31:09.221122+0000","last_peered":"2023-10-09T06:31:09.221122+0000","last_clean":"2023-10-09T06:31:09.221122+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:31:09.221122+0000","last_undegraded":"2023-10-09T06:31:09.221122+0000","last_fullsized":"2023-10-09T06:31:09.221122+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765924+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:31:08.765924+0000","last_peered":"2023-10-09T06:31:08.765924+0000","last_clean":"2023-10-09T06:31:08.765924+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:31:08.765924+0000","last_undegraded":"2023-10-09T06:31:08.765924+0000","last_fullsized":"2023-10-09T06:31:08.765924+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":443,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219350+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:31:09.219350+0000","last_peered":"2023-10-09T06:31:09.219350+0000","last_clean":"2023-10-09T06:31:09.219350+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:31:09.219350+0000","last_undegraded":"2023-10-09T06:31:09.219350+0000","last_fullsized":"2023-10-09T06:31:09.219350+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.218310+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:31:09.218310+0000","last_peered":"2023-10-09T06:31:09.218310+0000","last_clean":"2023-10-09T06:31:09.218310+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:31:09.218310+0000","last_undegraded":"2023-10-09T06:31:09.218310+0000","last_fullsized":"2023-10-09T06:31:09.218310+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"365'991","reported_seq":1003,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:18.230515+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:31:18.230515+0000","last_peered":"2023-10-09T06:31:18.230515+0000","last_clean":"2023-10-09T06:31:18.230515+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:31:18.230515+0000","last_undegraded":"2023-10-09T06:31:18.230515+0000","last_fullsized":"2023-10-09T06:31:18.230515+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":991,"log_dups_size":0,"ondisk_log_size":991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":991,"num_write_kb":63424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":250,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221580+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:31:09.221580+0000","last_peered":"2023-10-09T06:31:09.221580+0000","last_clean":"2023-10-09T06:31:09.221580+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:31:09.221580+0000","last_undegraded":"2023-10-09T06:31:09.221580+0000","last_fullsized":"2023-10-09T06:31:09.221580+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114659+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:31:09.114659+0000","last_peered":"2023-10-09T06:31:09.114659+0000","last_clean":"2023-10-09T06:31:09.114659+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:31:09.114659+0000","last_undegraded":"2023-10-09T06:31:09.114659+0000","last_fullsized":"2023-10-09T06:31:09.114659+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"365'676","reported_seq":687,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:16.177672+0000","last_change":"2023-10-09T06:31:02.763465+0000","last_active":"2023-10-09T06:31:16.177672+0000","last_peered":"2023-10-09T06:31:16.177672+0000","last_clean":"2023-10-09T06:31:16.177672+0000","last_became_active":"2023-10-09T06:31:02.763153+0000","last_became_peered":"2023-10-09T06:31:02.763153+0000","last_unstale":"2023-10-09T06:31:16.177672+0000","last_undegraded":"2023-10-09T06:31:16.177672+0000","last_fullsized":"2023-10-09T06:31:16.177672+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":676,"log_dups_size":0,"ondisk_log_size":676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:01.874473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44302336,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":676,"num_write_kb":43264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":212,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221600+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:31:09.221600+0000","last_peered":"2023-10-09T06:31:09.221600+0000","last_clean":"2023-10-09T06:31:09.221600+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:31:09.221600+0000","last_undegraded":"2023-10-09T06:31:09.221600+0000","last_fullsized":"2023-10-09T06:31:09.221600+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":362,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219847+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:31:09.219847+0000","last_peered":"2023-10-09T06:31:09.219847+0000","last_clean":"2023-10-09T06:31:09.219847+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:31:09.219847+0000","last_undegraded":"2023-10-09T06:31:09.219847+0000","last_fullsized":"2023-10-09T06:31:09.219847+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"365'1018","reported_seq":1030,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:18.226729+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:31:18.226729+0000","last_peered":"2023-10-09T06:31:18.226729+0000","last_clean":"2023-10-09T06:31:18.226729+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:31:18.226729+0000","last_undegraded":"2023-10-09T06:31:18.226729+0000","last_fullsized":"2023-10-09T06:31:18.226729+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":1018,"log_dups_size":0,"ondisk_log_size":1018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66715648,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1018,"num_write_kb":65152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":268,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221490+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:31:09.221490+0000","last_peered":"2023-10-09T06:31:09.221490+0000","last_clean":"2023-10-09T06:31:09.221490+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:31:09.221490+0000","last_undegraded":"2023-10-09T06:31:09.221490+0000","last_fullsized":"2023-10-09T06:31:09.221490+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":341,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219650+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:31:09.219650+0000","last_peered":"2023-10-09T06:31:09.219650+0000","last_clean":"2023-10-09T06:31:09.219650+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:31:09.219650+0000","last_undegraded":"2023-10-09T06:31:09.219650+0000","last_fullsized":"2023-10-09T06:31:09.219650+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"365'587","reported_seq":600,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:15.637218+0000","last_change":"2023-10-09T06:31:02.761953+0000","last_active":"2023-10-09T06:31:15.637218+0000","last_peered":"2023-10-09T06:31:15.637218+0000","last_clean":"2023-10-09T06:31:15.637218+0000","last_became_active":"2023-10-09T06:31:02.761625+0000","last_became_peered":"2023-10-09T06:31:02.761625+0000","last_unstale":"2023-10-09T06:31:15.637218+0000","last_undegraded":"2023-10-09T06:31:15.637218+0000","last_fullsized":"2023-10-09T06:31:15.637218+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":587,"log_dups_size":0,"ondisk_log_size":587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:31.403447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38469632,"num_objects":587,"num_object_clones":0,"num_object_copies":1761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":587,"num_write_kb":37568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114972+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:31:09.114972+0000","last_peered":"2023-10-09T06:31:09.114972+0000","last_clean":"2023-10-09T06:31:09.114972+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:31:09.114972+0000","last_undegraded":"2023-10-09T06:31:09.114972+0000","last_fullsized":"2023-10-09T06:31:09.114972+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":324,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764682+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:31:08.764682+0000","last_peered":"2023-10-09T06:31:08.764682+0000","last_clean":"2023-10-09T06:31:08.764682+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:31:08.764682+0000","last_undegraded":"2023-10-09T06:31:08.764682+0000","last_fullsized":"2023-10-09T06:31:08.764682+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"365'667","reported_seq":679,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:16.157201+0000","last_change":"2023-10-09T06:31:02.762571+0000","last_active":"2023-10-09T06:31:16.157201+0000","last_peered":"2023-10-09T06:31:16.157201+0000","last_clean":"2023-10-09T06:31:16.157201+0000","last_became_active":"2023-10-09T06:31:02.762287+0000","last_became_peered":"2023-10-09T06:31:02.762287+0000","last_unstale":"2023-10-09T06:31:16.157201+0000","last_undegraded":"2023-10-09T06:31:16.157201+0000","last_fullsized":"2023-10-09T06:31:16.157201+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":667,"log_dups_size":0,"ondisk_log_size":667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:15.365778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":667,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":697,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221001+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:31:09.221001+0000","last_peered":"2023-10-09T06:31:09.221001+0000","last_clean":"2023-10-09T06:31:09.221001+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:31:09.221001+0000","last_undegraded":"2023-10-09T06:31:09.221001+0000","last_fullsized":"2023-10-09T06:31:09.221001+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":89,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217932+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:31:09.217932+0000","last_peered":"2023-10-09T06:31:09.217932+0000","last_clean":"2023-10-09T06:31:09.217932+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:31:09.217932+0000","last_undegraded":"2023-10-09T06:31:09.217932+0000","last_fullsized":"2023-10-09T06:31:09.217932+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":345,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219588+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:31:09.219588+0000","last_peered":"2023-10-09T06:31:09.219588+0000","last_clean":"2023-10-09T06:31:09.219588+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:31:09.219588+0000","last_undegraded":"2023-10-09T06:31:09.219588+0000","last_fullsized":"2023-10-09T06:31:09.219588+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"365'688","reported_seq":700,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:16.175945+0000","last_change":"2023-10-09T06:31:03.217042+0000","last_active":"2023-10-09T06:31:16.175945+0000","last_peered":"2023-10-09T06:31:16.175945+0000","last_clean":"2023-10-09T06:31:16.175945+0000","last_became_active":"2023-10-09T06:31:03.216729+0000","last_became_peered":"2023-10-09T06:31:03.216729+0000","last_unstale":"2023-10-09T06:31:16.175945+0000","last_undegraded":"2023-10-09T06:31:16.175945+0000","last_fullsized":"2023-10-09T06:31:16.175945+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":688,"log_dups_size":0,"ondisk_log_size":688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.232295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45088768,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":688,"num_write_kb":44032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":677,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114660+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:31:09.114660+0000","last_peered":"2023-10-09T06:31:09.114660+0000","last_clean":"2023-10-09T06:31:09.114660+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:31:09.114660+0000","last_undegraded":"2023-10-09T06:31:09.114660+0000","last_fullsized":"2023-10-09T06:31:09.114660+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":322,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219563+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:31:09.219563+0000","last_peered":"2023-10-09T06:31:09.219563+0000","last_clean":"2023-10-09T06:31:09.219563+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:31:09.219563+0000","last_undegraded":"2023-10-09T06:31:09.219563+0000","last_fullsized":"2023-10-09T06:31:09.219563+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220088+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:31:09.220088+0000","last_peered":"2023-10-09T06:31:09.220088+0000","last_clean":"2023-10-09T06:31:09.220088+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:31:09.220088+0000","last_undegraded":"2023-10-09T06:31:09.220088+0000","last_fullsized":"2023-10-09T06:31:09.220088+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"365'871","reported_seq":884,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:17.397044+0000","last_change":"2023-10-09T06:31:02.762312+0000","last_active":"2023-10-09T06:31:17.397044+0000","last_peered":"2023-10-09T06:31:17.397044+0000","last_clean":"2023-10-09T06:31:17.397044+0000","last_became_active":"2023-10-09T06:31:02.762007+0000","last_became_peered":"2023-10-09T06:31:02.762007+0000","last_unstale":"2023-10-09T06:31:17.397044+0000","last_undegraded":"2023-10-09T06:31:17.397044+0000","last_fullsized":"2023-10-09T06:31:17.397044+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":871,"log_dups_size":0,"ondisk_log_size":871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57081856,"num_objects":871,"num_object_clones":0,"num_object_copies":2613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":871,"num_write_kb":55744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":346,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219598+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:31:09.219598+0000","last_peered":"2023-10-09T06:31:09.219598+0000","last_clean":"2023-10-09T06:31:09.219598+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:31:09.219598+0000","last_undegraded":"2023-10-09T06:31:09.219598+0000","last_fullsized":"2023-10-09T06:31:09.219598+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":325,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220244+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:31:09.220244+0000","last_peered":"2023-10-09T06:31:09.220244+0000","last_clean":"2023-10-09T06:31:09.220244+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:31:09.220244+0000","last_undegraded":"2023-10-09T06:31:09.220244+0000","last_fullsized":"2023-10-09T06:31:09.220244+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"365'889","reported_seq":902,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:17.402474+0000","last_change":"2023-10-09T06:31:03.215765+0000","last_active":"2023-10-09T06:31:17.402474+0000","last_peered":"2023-10-09T06:31:17.402474+0000","last_clean":"2023-10-09T06:31:17.402474+0000","last_became_active":"2023-10-09T06:31:03.215425+0000","last_became_peered":"2023-10-09T06:31:03.215425+0000","last_unstale":"2023-10-09T06:31:17.402474+0000","last_undegraded":"2023-10-09T06:31:17.402474+0000","last_fullsized":"2023-10-09T06:31:17.402474+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":889,"log_dups_size":0,"ondisk_log_size":889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58261504,"num_objects":889,"num_object_clones":0,"num_object_copies":2667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":889,"num_write_kb":56896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":655,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114678+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:31:09.114678+0000","last_peered":"2023-10-09T06:31:09.114678+0000","last_clean":"2023-10-09T06:31:09.114678+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:31:09.114678+0000","last_undegraded":"2023-10-09T06:31:09.114678+0000","last_fullsized":"2023-10-09T06:31:09.114678+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217519+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:31:09.217519+0000","last_peered":"2023-10-09T06:31:09.217519+0000","last_clean":"2023-10-09T06:31:09.217519+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:31:09.217519+0000","last_undegraded":"2023-10-09T06:31:09.217519+0000","last_fullsized":"2023-10-09T06:31:09.217519+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":317,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766609+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:31:08.766609+0000","last_peered":"2023-10-09T06:31:08.766609+0000","last_clean":"2023-10-09T06:31:08.766609+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:31:08.766609+0000","last_undegraded":"2023-10-09T06:31:08.766609+0000","last_fullsized":"2023-10-09T06:31:08.766609+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"365'543","reported_seq":556,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:15.655969+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:31:15.655969+0000","last_peered":"2023-10-09T06:31:15.655969+0000","last_clean":"2023-10-09T06:31:15.655969+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:31:15.655969+0000","last_undegraded":"2023-10-09T06:31:15.655969+0000","last_fullsized":"2023-10-09T06:31:15.655969+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":543,"log_dups_size":0,"ondisk_log_size":543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35586048,"num_objects":543,"num_object_clones":0,"num_object_copies":1629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":543,"num_write_kb":34752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":627,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220671+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:31:09.220671+0000","last_peered":"2023-10-09T06:31:09.220671+0000","last_clean":"2023-10-09T06:31:09.220671+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:31:09.220671+0000","last_undegraded":"2023-10-09T06:31:09.220671+0000","last_fullsized":"2023-10-09T06:31:09.220671+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":368,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765908+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:31:08.765908+0000","last_peered":"2023-10-09T06:31:08.765908+0000","last_clean":"2023-10-09T06:31:08.765908+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:31:08.765908+0000","last_undegraded":"2023-10-09T06:31:08.765908+0000","last_fullsized":"2023-10-09T06:31:08.765908+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114987+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:31:09.114987+0000","last_peered":"2023-10-09T06:31:09.114987+0000","last_clean":"2023-10-09T06:31:09.114987+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:31:09.114987+0000","last_undegraded":"2023-10-09T06:31:09.114987+0000","last_fullsized":"2023-10-09T06:31:09.114987+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":302,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766093+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:31:08.766093+0000","last_peered":"2023-10-09T06:31:08.766093+0000","last_clean":"2023-10-09T06:31:08.766093+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:31:08.766093+0000","last_undegraded":"2023-10-09T06:31:08.766093+0000","last_fullsized":"2023-10-09T06:31:08.766093+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"365'612","reported_seq":625,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:15.657430+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:31:15.657430+0000","last_peered":"2023-10-09T06:31:15.657430+0000","last_clean":"2023-10-09T06:31:15.657430+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:31:15.657430+0000","last_undegraded":"2023-10-09T06:31:15.657430+0000","last_fullsized":"2023-10-09T06:31:15.657430+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":612,"log_dups_size":0,"ondisk_log_size":612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40108032,"num_objects":612,"num_object_clones":0,"num_object_copies":1836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":612,"num_write_kb":39168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":691,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114857+0000","last_change":"2023-10-09T06:29:15.532600+0000","last_active":"2023-10-09T06:31:09.114857+0000","last_peered":"2023-10-09T06:31:09.114857+0000","last_clean":"2023-10-09T06:31:09.114857+0000","last_became_active":"2023-10-09T06:29:15.532290+0000","last_became_peered":"2023-10-09T06:29:15.532290+0000","last_unstale":"2023-10-09T06:31:09.114857+0000","last_undegraded":"2023-10-09T06:31:09.114857+0000","last_fullsized":"2023-10-09T06:31:09.114857+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:55.742439+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114858+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:31:09.114858+0000","last_peered":"2023-10-09T06:31:09.114858+0000","last_clean":"2023-10-09T06:31:09.114858+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:31:09.114858+0000","last_undegraded":"2023-10-09T06:31:09.114858+0000","last_fullsized":"2023-10-09T06:31:09.114858+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":346,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220449+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:31:09.220449+0000","last_peered":"2023-10-09T06:31:09.220449+0000","last_clean":"2023-10-09T06:31:09.220449+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:31:09.220449+0000","last_undegraded":"2023-10-09T06:31:09.220449+0000","last_fullsized":"2023-10-09T06:31:09.220449+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":332,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765838+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:31:08.765838+0000","last_peered":"2023-10-09T06:31:08.765838+0000","last_clean":"2023-10-09T06:31:08.765838+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:31:08.765838+0000","last_undegraded":"2023-10-09T06:31:08.765838+0000","last_fullsized":"2023-10-09T06:31:08.765838+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"365'714","reported_seq":726,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:16.172008+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:31:16.172008+0000","last_peered":"2023-10-09T06:31:16.172008+0000","last_clean":"2023-10-09T06:31:16.172008+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:31:16.172008+0000","last_undegraded":"2023-10-09T06:31:16.172008+0000","last_fullsized":"2023-10-09T06:31:16.172008+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":714,"log_dups_size":0,"ondisk_log_size":714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46792704,"num_objects":714,"num_object_clones":0,"num_object_copies":2142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":714,"num_write_kb":45696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":687,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221206+0000","last_change":"2023-10-09T06:29:15.535685+0000","last_active":"2023-10-09T06:31:09.221206+0000","last_peered":"2023-10-09T06:31:09.221206+0000","last_clean":"2023-10-09T06:31:09.221206+0000","last_became_active":"2023-10-09T06:29:15.535396+0000","last_became_peered":"2023-10-09T06:29:15.535396+0000","last_unstale":"2023-10-09T06:31:09.221206+0000","last_undegraded":"2023-10-09T06:31:09.221206+0000","last_fullsized":"2023-10-09T06:31:09.221206+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:23.142031+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":354,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217534+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:31:09.217534+0000","last_peered":"2023-10-09T06:31:09.217534+0000","last_clean":"2023-10-09T06:31:09.217534+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:31:09.217534+0000","last_undegraded":"2023-10-09T06:31:09.217534+0000","last_fullsized":"2023-10-09T06:31:09.217534+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":341,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765081+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:31:08.765081+0000","last_peered":"2023-10-09T06:31:08.765081+0000","last_clean":"2023-10-09T06:31:08.765081+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:31:08.765081+0000","last_undegraded":"2023-10-09T06:31:08.765081+0000","last_fullsized":"2023-10-09T06:31:08.765081+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":363,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766681+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:31:08.766681+0000","last_peered":"2023-10-09T06:31:08.766681+0000","last_clean":"2023-10-09T06:31:08.766681+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:31:08.766681+0000","last_undegraded":"2023-10-09T06:31:08.766681+0000","last_fullsized":"2023-10-09T06:31:08.766681+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"365'1050","reported_seq":1062,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:18.297672+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:31:18.297672+0000","last_peered":"2023-10-09T06:31:18.297672+0000","last_clean":"2023-10-09T06:31:18.297672+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:31:18.297672+0000","last_undegraded":"2023-10-09T06:31:18.297672+0000","last_fullsized":"2023-10-09T06:31:18.297672+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":1050,"log_dups_size":0,"ondisk_log_size":1050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1050,"num_write_kb":67200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":52,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114722+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:31:09.114722+0000","last_peered":"2023-10-09T06:31:09.114722+0000","last_clean":"2023-10-09T06:31:09.114722+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:31:09.114722+0000","last_undegraded":"2023-10-09T06:31:09.114722+0000","last_fullsized":"2023-10-09T06:31:09.114722+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":668,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221172+0000","last_change":"2023-10-09T06:29:15.536561+0000","last_active":"2023-10-09T06:31:09.221172+0000","last_peered":"2023-10-09T06:31:09.221172+0000","last_clean":"2023-10-09T06:31:09.221172+0000","last_became_active":"2023-10-09T06:29:15.535869+0000","last_became_peered":"2023-10-09T06:29:15.535869+0000","last_unstale":"2023-10-09T06:31:09.221172+0000","last_undegraded":"2023-10-09T06:31:09.221172+0000","last_fullsized":"2023-10-09T06:31:09.221172+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:04.229310+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":72,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217743+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:31:09.217743+0000","last_peered":"2023-10-09T06:31:09.217743+0000","last_clean":"2023-10-09T06:31:09.217743+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:31:09.217743+0000","last_undegraded":"2023-10-09T06:31:09.217743+0000","last_fullsized":"2023-10-09T06:31:09.217743+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220477+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:31:09.220477+0000","last_peered":"2023-10-09T06:31:09.220477+0000","last_clean":"2023-10-09T06:31:09.220477+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:31:09.220477+0000","last_undegraded":"2023-10-09T06:31:09.220477+0000","last_fullsized":"2023-10-09T06:31:09.220477+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"365'635","reported_seq":648,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:15.598523+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:31:15.598523+0000","last_peered":"2023-10-09T06:31:15.598523+0000","last_clean":"2023-10-09T06:31:15.598523+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:31:15.598523+0000","last_undegraded":"2023-10-09T06:31:15.598523+0000","last_fullsized":"2023-10-09T06:31:15.598523+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":635,"log_dups_size":0,"ondisk_log_size":635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41615360,"num_objects":635,"num_object_clones":0,"num_object_copies":1905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":635,"num_write_kb":40640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":650,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765367+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:31:08.765367+0000","last_peered":"2023-10-09T06:31:08.765367+0000","last_clean":"2023-10-09T06:31:08.765367+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:31:08.765367+0000","last_undegraded":"2023-10-09T06:31:08.765367+0000","last_fullsized":"2023-10-09T06:31:08.765367+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":228,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220976+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:31:09.220976+0000","last_peered":"2023-10-09T06:31:09.220976+0000","last_clean":"2023-10-09T06:31:09.220976+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:31:09.220976+0000","last_undegraded":"2023-10-09T06:31:09.220976+0000","last_fullsized":"2023-10-09T06:31:09.220976+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764910+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:31:08.764910+0000","last_peered":"2023-10-09T06:31:08.764910+0000","last_clean":"2023-10-09T06:31:08.764910+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:31:08.764910+0000","last_undegraded":"2023-10-09T06:31:08.764910+0000","last_fullsized":"2023-10-09T06:31:08.764910+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":347,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766324+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:31:08.766324+0000","last_peered":"2023-10-09T06:31:08.766324+0000","last_clean":"2023-10-09T06:31:08.766324+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:31:08.766324+0000","last_undegraded":"2023-10-09T06:31:08.766324+0000","last_fullsized":"2023-10-09T06:31:08.766324+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"365'402","reported_seq":415,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:14.475725+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:31:14.475725+0000","last_peered":"2023-10-09T06:31:14.475725+0000","last_clean":"2023-10-09T06:31:14.475725+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:31:14.475725+0000","last_undegraded":"2023-10-09T06:31:14.475725+0000","last_fullsized":"2023-10-09T06:31:14.475725+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":402,"log_dups_size":0,"ondisk_log_size":402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26345472,"num_objects":402,"num_object_clones":0,"num_object_copies":1206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":402,"num_write_kb":25728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":657,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219907+0000","last_change":"2023-10-09T06:29:15.549866+0000","last_active":"2023-10-09T06:31:09.219907+0000","last_peered":"2023-10-09T06:31:09.219907+0000","last_clean":"2023-10-09T06:31:09.219907+0000","last_became_active":"2023-10-09T06:29:15.549495+0000","last_became_peered":"2023-10-09T06:29:15.549495+0000","last_unstale":"2023-10-09T06:31:09.219907+0000","last_undegraded":"2023-10-09T06:31:09.219907+0000","last_fullsized":"2023-10-09T06:31:09.219907+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:46.691754+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":228,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.221657+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:31:09.221657+0000","last_peered":"2023-10-09T06:31:09.221657+0000","last_clean":"2023-10-09T06:31:09.221657+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:31:09.221657+0000","last_undegraded":"2023-10-09T06:31:09.221657+0000","last_fullsized":"2023-10-09T06:31:09.221657+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":358,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765738+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:31:08.765738+0000","last_peered":"2023-10-09T06:31:08.765738+0000","last_clean":"2023-10-09T06:31:08.765738+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:31:08.765738+0000","last_undegraded":"2023-10-09T06:31:08.765738+0000","last_fullsized":"2023-10-09T06:31:08.765738+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":345,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764894+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:31:08.764894+0000","last_peered":"2023-10-09T06:31:08.764894+0000","last_clean":"2023-10-09T06:31:08.764894+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:31:08.764894+0000","last_undegraded":"2023-10-09T06:31:08.764894+0000","last_fullsized":"2023-10-09T06:31:08.764894+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"365'429","reported_seq":441,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:14.489434+0000","last_change":"2023-10-09T06:31:03.217038+0000","last_active":"2023-10-09T06:31:14.489434+0000","last_peered":"2023-10-09T06:31:14.489434+0000","last_clean":"2023-10-09T06:31:14.489434+0000","last_became_active":"2023-10-09T06:31:03.216747+0000","last_became_peered":"2023-10-09T06:31:03.216747+0000","last_unstale":"2023-10-09T06:31:14.489434+0000","last_undegraded":"2023-10-09T06:31:14.489434+0000","last_fullsized":"2023-10-09T06:31:14.489434+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":429,"log_dups_size":0,"ondisk_log_size":429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:22.026356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28114944,"num_objects":429,"num_object_clones":0,"num_object_copies":1287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":429,"num_write_kb":27456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":648,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219961+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:31:09.219961+0000","last_peered":"2023-10-09T06:31:09.219961+0000","last_clean":"2023-10-09T06:31:09.219961+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:31:09.219961+0000","last_undegraded":"2023-10-09T06:31:09.219961+0000","last_fullsized":"2023-10-09T06:31:09.219961+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":93,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.218383+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:31:09.218383+0000","last_peered":"2023-10-09T06:31:09.218383+0000","last_clean":"2023-10-09T06:31:09.218383+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:31:09.218383+0000","last_undegraded":"2023-10-09T06:31:09.218383+0000","last_fullsized":"2023-10-09T06:31:09.218383+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":326,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766128+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:31:08.766128+0000","last_peered":"2023-10-09T06:31:08.766128+0000","last_clean":"2023-10-09T06:31:08.766128+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:31:08.766128+0000","last_undegraded":"2023-10-09T06:31:08.766128+0000","last_fullsized":"2023-10-09T06:31:08.766128+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217501+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:31:09.217501+0000","last_peered":"2023-10-09T06:31:09.217501+0000","last_clean":"2023-10-09T06:31:09.217501+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:31:09.217501+0000","last_undegraded":"2023-10-09T06:31:09.217501+0000","last_fullsized":"2023-10-09T06:31:09.217501+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"365'695","reported_seq":707,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-10-09T06:31:16.166473+0000","last_change":"2023-10-09T06:31:03.215173+0000","last_active":"2023-10-09T06:31:16.166473+0000","last_peered":"2023-10-09T06:31:16.166473+0000","last_clean":"2023-10-09T06:31:16.166473+0000","last_became_active":"2023-10-09T06:31:03.214809+0000","last_became_peered":"2023-10-09T06:31:03.214809+0000","last_unstale":"2023-10-09T06:31:16.166473+0000","last_undegraded":"2023-10-09T06:31:16.166473+0000","last_fullsized":"2023-10-09T06:31:16.166473+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":695,"log_dups_size":0,"ondisk_log_size":695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:12.323433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45547520,"num_objects":695,"num_object_clones":0,"num_object_copies":2085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":695,"num_write_kb":44480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":694,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.218046+0000","last_change":"2023-10-09T06:28:14.734086+0000","last_active":"2023-10-09T06:31:09.218046+0000","last_peered":"2023-10-09T06:31:09.218046+0000","last_clean":"2023-10-09T06:31:09.218046+0000","last_became_active":"2023-10-09T06:28:14.733702+0000","last_became_peered":"2023-10-09T06:28:14.733702+0000","last_unstale":"2023-10-09T06:31:09.218046+0000","last_undegraded":"2023-10-09T06:31:09.218046+0000","last_fullsized":"2023-10-09T06:31:09.218046+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.831911+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":349,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220664+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:31:09.220664+0000","last_peered":"2023-10-09T06:31:09.220664+0000","last_clean":"2023-10-09T06:31:09.220664+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:31:09.220664+0000","last_undegraded":"2023-10-09T06:31:09.220664+0000","last_fullsized":"2023-10-09T06:31:09.220664+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":328,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766078+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:31:08.766078+0000","last_peered":"2023-10-09T06:31:08.766078+0000","last_clean":"2023-10-09T06:31:08.766078+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:31:08.766078+0000","last_undegraded":"2023-10-09T06:31:08.766078+0000","last_fullsized":"2023-10-09T06:31:08.766078+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":328,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219726+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:31:09.219726+0000","last_peered":"2023-10-09T06:31:09.219726+0000","last_clean":"2023-10-09T06:31:09.219726+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:31:09.219726+0000","last_undegraded":"2023-10-09T06:31:09.219726+0000","last_fullsized":"2023-10-09T06:31:09.219726+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":330,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.764822+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:31:08.764822+0000","last_peered":"2023-10-09T06:31:08.764822+0000","last_clean":"2023-10-09T06:31:08.764822+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:31:08.764822+0000","last_undegraded":"2023-10-09T06:31:08.764822+0000","last_fullsized":"2023-10-09T06:31:08.764822+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":351,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.765663+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:31:08.765663+0000","last_peered":"2023-10-09T06:31:08.765663+0000","last_clean":"2023-10-09T06:31:08.765663+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:31:08.765663+0000","last_undegraded":"2023-10-09T06:31:08.765663+0000","last_fullsized":"2023-10-09T06:31:08.765663+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":56,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114894+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:31:09.114894+0000","last_peered":"2023-10-09T06:31:09.114894+0000","last_clean":"2023-10-09T06:31:09.114894+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:31:09.114894+0000","last_undegraded":"2023-10-09T06:31:09.114894+0000","last_fullsized":"2023-10-09T06:31:09.114894+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":372,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220415+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:31:09.220415+0000","last_peered":"2023-10-09T06:31:09.220415+0000","last_clean":"2023-10-09T06:31:09.220415+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:31:09.220415+0000","last_undegraded":"2023-10-09T06:31:09.220415+0000","last_fullsized":"2023-10-09T06:31:09.220415+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":56,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.114935+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:31:09.114935+0000","last_peered":"2023-10-09T06:31:09.114935+0000","last_clean":"2023-10-09T06:31:09.114935+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:31:09.114935+0000","last_undegraded":"2023-10-09T06:31:09.114935+0000","last_fullsized":"2023-10-09T06:31:09.114935+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":382,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.220329+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:31:09.220329+0000","last_peered":"2023-10-09T06:31:09.220329+0000","last_clean":"2023-10-09T06:31:09.220329+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:31:09.220329+0000","last_undegraded":"2023-10-09T06:31:09.220329+0000","last_fullsized":"2023-10-09T06:31:09.220329+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":63,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.218175+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:31:09.218175+0000","last_peered":"2023-10-09T06:31:09.218175+0000","last_clean":"2023-10-09T06:31:09.218175+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:31:09.218175+0000","last_undegraded":"2023-10-09T06:31:09.218175+0000","last_fullsized":"2023-10-09T06:31:09.218175+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.217741+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:31:09.217741+0000","last_peered":"2023-10-09T06:31:09.217741+0000","last_clean":"2023-10-09T06:31:09.217741+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:31:09.217741+0000","last_undegraded":"2023-10-09T06:31:09.217741+0000","last_fullsized":"2023-10-09T06:31:09.217741+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":358,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:08.766793+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:31:08.766793+0000","last_peered":"2023-10-09T06:31:08.766793+0000","last_clean":"2023-10-09T06:31:08.766793+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:31:08.766793+0000","last_undegraded":"2023-10-09T06:31:08.766793+0000","last_fullsized":"2023-10-09T06:31:08.766793+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":325,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-10-09T06:31:09.219378+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:31:09.219378+0000","last_peered":"2023-10-09T06:31:09.219378+0000","last_clean":"2023-10-09T06:31:09.219378+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:31:09.219378+0000","last_undegraded":"2023-10-09T06:31:09.219378+0000","last_fullsized":"2023-10-09T06:31:09.219378+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":751501312,"num_objects":11467,"num_object_clones":0,"num_object_copies":34401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11467,"num_write_kb":733888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2273968128,"data_stored":2273968128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11467,"ondisk_log_size":11467,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234408,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332824,"kb_used_data":432592,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92412328,"statfs":{"total":95995035648,"available":94630223872,"internally_reserved":0,"allocated":442974208,"data_stored":442013913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":4.9039999999999999,"15min":4.9569999999999999},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":4.2549999999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":1.72},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.9370000000000003,"15min":5.0670000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":3.2930000000000001,"5min":215.577,"15min":219.399},"last":1.2110000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":7.7770000000000001,"5min":6.6429999999999998,"15min":4.2619999999999996},"min":{"1min":0.318,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":45.502000000000002,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.254},{"interface":"front","average":{"1min":5.8609999999999998,"5min":5.7720000000000002,"15min":3.794},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":45.447000000000003,"5min":216.22200000000001,"15min":216.22200000000001},"last":1.6839999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":4.5220000000000002,"15min":6.0960000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":6.3319999999999999,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.378},{"interface":"front","average":{"1min":3.1200000000000001,"5min":4.7999999999999998,"15min":6.2789999999999999},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":16.844000000000001,"5min":216.25200000000001,"15min":216.25200000000001},"last":2.1459999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":7.6580000000000004,"15min":3.899},"min":{"1min":0.307,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":187.34200000000001,"5min":215.922,"15min":215.922},"last":1.4119999999999999},{"interface":"front","average":{"1min":16.768000000000001,"5min":7.7969999999999997,"15min":3.9900000000000002},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":187.08199999999999,"5min":215.94900000000001,"15min":215.94900000000001},"last":1.9399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":4.1120000000000001,"15min":2.6160000000000001},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":6.1689999999999996,"5min":218.446,"15min":218.446},"last":1.752},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.9889999999999999,"15min":2.4079999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.23699999999999999},"max":{"1min":4.891,"5min":216.172,"15min":216.172},"last":1.395}]},{"osd":5,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":4.2350000000000003,"15min":2.661},"min":{"1min":0.31,"5min":0.31,"15min":0.22900000000000001},"max":{"1min":8.8170000000000002,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.603,"5min":4.1079999999999997,"15min":2.4809999999999999},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":6.3019999999999996,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.84299999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:30:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":4.343,"15min":2.3929999999999998},"min":{"1min":0.33300000000000002,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":15.162000000000001,"5min":218.19999999999999,"15min":218.19999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.5,"5min":3.9390000000000001,"15min":2.161},"min":{"1min":0.34699999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.2919999999999998,"5min":218.05500000000001,"15min":218.05500000000001},"last":1.3660000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496043,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169924,"kb_used_data":270908,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92575228,"statfs":{"total":95995035648,"available":94797033472,"internally_reserved":0,"allocated":277409792,"data_stored":276469243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.601,"15min":3.0369999999999999},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.7370000000000001,"5min":22.321999999999999,"15min":110.926},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.258,"5min":2.1259999999999999,"15min":3.3610000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.4340000000000002,"5min":22.338000000000001,"15min":110.911},"last":0.54200000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3760000000000003,"5min":6.9649999999999999,"15min":5.0010000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":82.159000000000006,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.40799999999999997},{"interface":"front","average":{"1min":7.1950000000000003,"5min":7.0590000000000002,"15min":4.8810000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":82.174000000000007,"5min":292.904,"15min":292.904},"last":0.35299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.109,"15min":2.6459999999999999},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.3670000000000009,"5min":20.331,"15min":75.191999999999993},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.335,"5min":1.748,"15min":2.351},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":8.2400000000000002,"5min":8.6069999999999993,"15min":75.120000000000005},"last":0.27100000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":4.1180000000000003,"15min":3.2120000000000002},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":88.763000000000005,"5min":91.174000000000007,"15min":110.358},"last":0.56200000000000006},{"interface":"front","average":{"1min":6.0300000000000002,"5min":4.2080000000000002,"15min":3.2850000000000001},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.311},"max":{"1min":86.831999999999994,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.361,"15min":1.3220000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":12.032,"5min":12.666,"15min":17.643999999999998},"last":0.377},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.97499999999999998,"15min":1.1739999999999999},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":5.1280000000000001,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.3169999999999999,"15min":1.429},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.9830000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.448,"5min":1.379,"15min":1.369},"min":{"1min":0.33300000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.8000000000000007,"5min":16.536999999999999,"15min":24.401},"last":0.438}]},{"osd":7,"last update":"Mon Oct 9 06:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":4.0529999999999999,"15min":2.528},"min":{"1min":0.314,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":7.8899999999999997,"5min":122.937,"15min":122.937},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.4350000000000001,"5min":4.3369999999999997,"15min":2.6709999999999998},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.32800000000000001},"max":{"1min":10.423,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.52500000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402781,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1179420,"kb_used_data":525780,"kb_used_omap":4,"kb_used_meta":653627,"kb_avail":92565732,"statfs":{"total":95995035648,"available":94787309568,"internally_reserved":0,"allocated":538398720,"data_stored":536733508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":669314958},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":9.8030000000000008,"5min":6.2869999999999999,"15min":5.2409999999999997},"min":{"1min":0.22,"5min":0.189,"15min":0.189},"max":{"1min":60.973999999999997,"5min":269.79199999999997,"15min":269.79199999999997},"last":0.20699999999999999},{"interface":"front","average":{"1min":8.8480000000000008,"5min":6.4539999999999997,"15min":5.6299999999999999},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.17899999999999999},"max":{"1min":60.201000000000001,"5min":271.20999999999998,"15min":271.20999999999998},"last":1.0269999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.2789999999999999,"15min":2.9540000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":6.9390000000000001,"5min":7.1449999999999996,"15min":139.75399999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3759999999999999,"15min":3.6970000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":6.673,"5min":12.077,"15min":140.29900000000001},"last":0.94299999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:30:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":3.2040000000000002,"15min":3.2679999999999998},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":13.214,"5min":31.062000000000001,"15min":153.18199999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":2.6389999999999998,"5min":2.2389999999999999,"15min":2.444},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.128,"5min":27.657,"15min":152.52000000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.883,"15min":3.3929999999999998},"min":{"1min":0.312,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":13.087999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.7050000000000001,"5min":1.823,"15min":3.2989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.1300000000000008,"5min":11.974,"15min":199.32900000000001},"last":0.747}]},{"osd":5,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":1.962,"15min":2.0699999999999998},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":52.512999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":4.665,"5min":2.3260000000000001,"15min":2.6000000000000001},"min":{"1min":0.39700000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":52.664999999999999,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.70799999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0220000000000002,"5min":2.7250000000000001,"15min":2.8969999999999998},"min":{"1min":0.45900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":52.488,"5min":52.488,"15min":77.292000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":6.556,"5min":2.952,"15min":2.843},"min":{"1min":0.38200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":52.558,"5min":52.558,"15min":77.378},"last":0.63700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:30:33 2023","interfaces":[{"interface":"back","average":{"1min":6.0129999999999999,"5min":3.8399999999999999,"15min":3.0470000000000002},"min":{"1min":0.40200000000000002,"5min":0.36499999999999999,"15min":0.32500000000000001},"max":{"1min":52.601999999999997,"5min":55.231000000000002,"15min":77.456999999999994},"last":0.66800000000000004},{"interface":"front","average":{"1min":4.8040000000000003,"5min":3.1800000000000002,"15min":2.5179999999999998},"min":{"1min":0.42199999999999999,"5min":0.35799999999999998,"15min":0.34399999999999997},"max":{"1min":52.579999999999998,"5min":55.253999999999998,"15min":77.359999999999999},"last":0.495}]}]},{"osd":0,"up_from":91,"seq":390842024214,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1293636,"kb_used_data":493308,"kb_used_omap":5,"kb_used_meta":800314,"kb_avail":92451516,"statfs":{"total":95995035648,"available":94670352384,"internally_reserved":0,"allocated":505147392,"data_stored":503214863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":819522170},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":3.6059999999999999,"15min":3.3719999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.16600000000000001},"max":{"1min":8.1389999999999993,"5min":191.446,"15min":191.446},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.6419999999999999,"5min":3.8159999999999998,"15min":3.6709999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.17499999999999999},"max":{"1min":12.959,"5min":191.393,"15min":191.393},"last":0.75700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.7309999999999999,"15min":2.3180000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":16.936,"5min":57.162999999999997,"15min":57.162999999999997},"last":2.008},{"interface":"front","average":{"1min":3.6419999999999999,"5min":3.524,"15min":2.6960000000000002},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":15.794,"5min":57.585000000000001,"15min":57.585000000000001},"last":2.1019999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:07 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":7.8419999999999996,"15min":5.4000000000000004},"min":{"1min":0.26800000000000002,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":187.172,"5min":208.11199999999999,"15min":208.11199999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.4950000000000001,"15min":5.391},"min":{"1min":0.31900000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":194.851,"5min":208.25899999999999,"15min":208.25899999999999},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.518,"15min":1.429},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":7.21,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.7470000000000001,"15min":1.798},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":10.903,"5min":18.081,"15min":18.081},"last":0.82499999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.7130000000000001,"15min":1.806},"min":{"1min":0.317,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":7.2409999999999997,"5min":10.865,"15min":62.984999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.9970000000000001,"15min":2.2469999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":10.603,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.91200000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":4.2619999999999996,"15min":2.8980000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":12.522,"5min":120.572,"15min":120.572},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.758,"5min":3.827,"15min":2.3330000000000002},"min":{"1min":0.371,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":11.093,"5min":120.64400000000001,"15min":120.64400000000001},"last":1.2}]},{"osd":7,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":3.1840000000000002,"15min":2.4969999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":8.8059999999999992,"5min":122.018,"15min":122.018},"last":0.79700000000000004},{"interface":"front","average":{"1min":2.415,"5min":3.6539999999999999,"15min":2.8730000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":10.563000000000001,"5min":122.32299999999999,"15min":122.32299999999999},"last":0.77400000000000002}]}]},{"osd":2,"up_from":128,"seq":549755814083,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886260,"kb_used_data":365036,"kb_used_omap":7,"kb_used_meta":521208,"kb_avail":92858892,"statfs":{"total":95995035648,"available":95087505408,"internally_reserved":0,"allocated":373796864,"data_stored":372166493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":533717073},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":4.2409999999999997,"15min":4.7370000000000001},"min":{"1min":0.19500000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":16.91,"5min":162.21100000000001,"15min":163.40100000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":3.4180000000000001,"5min":4.3630000000000004,"15min":4.4260000000000002},"min":{"1min":0.30099999999999999,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":16.888999999999999,"5min":168.49700000000001,"15min":168.49700000000001},"last":1.375}]},{"osd":1,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":3.3719999999999999,"15min":4.1740000000000004},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":24.312000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.714,"5min":4.423,"15min":4.5380000000000003},"min":{"1min":0.29399999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.259,"5min":91.721999999999994,"15min":222.19399999999999},"last":1.4379999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.3370000000000002,"15min":2.8090000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8499999999999996,"5min":91.644000000000005,"15min":91.644000000000005},"last":1.4119999999999999},{"interface":"front","average":{"1min":1.841,"5min":3.9809999999999999,"15min":3.056},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.9059999999999997,"5min":91.685000000000002,"15min":91.685000000000002},"last":1.464}]},{"osd":4,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":1.7589999999999999,"15min":2.081},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":13.023999999999999,"5min":17.776,"15min":90.938999999999993},"last":1.4890000000000001},{"interface":"front","average":{"1min":3.0880000000000001,"5min":2.1560000000000001,"15min":2.375},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":90.950000000000003},"last":0.33400000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.0339999999999998,"15min":1.605},"min":{"1min":0.315,"5min":0.315,"15min":0.309},"max":{"1min":13.102,"5min":17.800000000000001,"15min":17.800000000000001},"last":1.5129999999999999},{"interface":"front","average":{"1min":3.2930000000000001,"5min":2.2709999999999999,"15min":1.978},"min":{"1min":0.377,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":3.585,"5min":2.3650000000000002,"15min":2.5529999999999999},"min":{"1min":0.41399999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":12.387,"5min":12.387,"15min":138.291},"last":0.375},{"interface":"front","average":{"1min":1.639,"5min":1.6459999999999999,"15min":2.1829999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":9.1579999999999995,"5min":12.643000000000001,"15min":138.01599999999999},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.052,"15min":1.968},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":9.1859999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":0.75},{"interface":"front","average":{"1min":3.04,"5min":2.113,"15min":2.0110000000000001},"min":{"1min":0.42099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.167,"5min":17.734999999999999,"15min":55.731000000000002},"last":1.599}]}]},{"osd":3,"up_from":149,"seq":639950127280,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985836,"kb_used_data":495204,"kb_used_omap":2,"kb_used_meta":490621,"kb_avail":92759316,"statfs":{"total":95995035648,"available":94985539584,"internally_reserved":0,"allocated":507088896,"data_stored":505525380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":502395957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.806,"5min":3.3180000000000001,"15min":4.1609999999999996},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":47.847000000000001,"5min":47.847000000000001,"15min":187.852},"last":1.456},{"interface":"front","average":{"1min":4.0030000000000001,"5min":3.0419999999999998,"15min":4.4180000000000001},"min":{"1min":0.28199999999999997,"5min":0.20799999999999999,"15min":0.184},"max":{"1min":47.896999999999998,"5min":47.896999999999998,"15min":187.923},"last":0.52800000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0279999999999996,"5min":2.734,"15min":3.5449999999999999},"min":{"1min":0.29599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":47.960999999999999,"5min":47.960999999999999,"15min":187.89099999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":5.5490000000000004,"5min":4.0780000000000003,"15min":3.0939999999999999},"min":{"1min":0.26100000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.087000000000003,"5min":48.087000000000003,"15min":67.028000000000006},"last":1.25}]},{"osd":2,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.8809999999999998,"15min":3.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":48.055,"5min":48.055,"15min":187.76400000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":5.25,"5min":3.375,"15min":3.1200000000000001},"min":{"1min":0.33100000000000002,"5min":0.27900000000000003,"15min":0.22500000000000001},"max":{"1min":48.128999999999998,"5min":48.128999999999998,"15min":187.79499999999999},"last":1.1240000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":2.657,"15min":1.78},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":48.249000000000002,"5min":48.249000000000002,"15min":48.249000000000002},"last":1.1619999999999999},{"interface":"front","average":{"1min":4.4370000000000003,"5min":2.6930000000000001,"15min":2.4649999999999999},"min":{"1min":0.40899999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":48.031999999999996,"5min":48.031999999999996,"15min":187.67400000000001},"last":1.208}]},{"osd":5,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":2.4510000000000001,"15min":2.5169999999999999},"min":{"1min":0.40200000000000002,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":48.162999999999997,"5min":48.162999999999997,"15min":187.833},"last":0.98499999999999999},{"interface":"front","average":{"1min":3.9119999999999999,"5min":2.0409999999999999,"15min":2.3079999999999998},"min":{"1min":0.46300000000000002,"5min":0.34599999999999997,"15min":0.32100000000000001},"max":{"1min":48.106000000000002,"5min":48.106000000000002,"15min":187.952},"last":0.69799999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":4.1029999999999998,"15min":3.125},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":48.270000000000003,"5min":130.39599999999999,"15min":187.78200000000001},"last":1.173},{"interface":"front","average":{"1min":3.8370000000000002,"5min":3.6099999999999999,"15min":2.7949999999999999},"min":{"1min":0.40300000000000002,"5min":0.34399999999999997,"15min":0.32900000000000001},"max":{"1min":48.155000000000001,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.91400000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.2069999999999999,"5min":4.4710000000000001,"15min":3.109},"min":{"1min":0.41899999999999998,"5min":0.38500000000000001,"15min":0.34100000000000003},"max":{"1min":48.292000000000002,"5min":121.613,"15min":192.20400000000001},"last":1.581},{"interface":"front","average":{"1min":5.4050000000000002,"5min":4.7080000000000002,"15min":3.2570000000000001},"min":{"1min":0.44700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":48.200000000000003,"5min":121.559,"15min":187.86500000000001},"last":1.0249999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019317,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068236,"kb_used_data":173252,"kb_used_omap":0,"kb_used_meta":894976,"kb_avail":92676916,"statfs":{"total":95995035648,"available":94901161984,"internally_reserved":0,"allocated":177410048,"data_stored":176461684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916455424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.696,"15min":2.2210000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":18.625,"5min":37.485999999999997,"15min":76.944999999999993},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.962,"5min":1.77,"15min":2.4740000000000002},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.654,"5min":37.502000000000002,"15min":76.988},"last":0.79500000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":10.651999999999999,"5min":4.3789999999999996,"15min":2.968},"min":{"1min":0.37,"5min":0.26400000000000001,"15min":0.23100000000000001},"max":{"1min":128.24299999999999,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":10.84,"5min":4.4909999999999997,"15min":3.133},"min":{"1min":0.43099999999999999,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":128.21600000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":3.0659999999999998,"15min":2.48},"min":{"1min":0.309,"5min":0.309,"15min":0.255},"max":{"1min":7.101,"5min":47.125,"15min":69.640000000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.601,"5min":2.7080000000000002,"15min":2.508},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.9870000000000001,"5min":47.151000000000003,"15min":69.676000000000002},"last":1.6830000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7669999999999999,"5min":7.024,"15min":4.4420000000000002},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":35.018000000000001,"5min":133.38900000000001,"15min":133.38900000000001},"last":1.274},{"interface":"front","average":{"1min":3.1869999999999998,"5min":6.7530000000000001,"15min":4.173},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":35.036999999999999,"5min":132.57599999999999,"15min":132.57599999999999},"last":1.0189999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.0329999999999999,"15min":1.1200000000000001},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":6.0199999999999996,"5min":6.2670000000000003,"15min":10.673},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.0249999999999999,"15min":1.111},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":5.7000000000000002,"5min":7.2249999999999996,"15min":12.552},"last":1.101}]},{"osd":6,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.4660000000000002,"15min":1.3340000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.0629999999999997,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.8029999999999999,"15min":1.7989999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.7210000000000001,"5min":163.381,"15min":163.381},"last":1.0309999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.05,"15min":1.476},"min":{"1min":0.32700000000000001,"5min":0.312,"15min":0.308},"max":{"1min":3.6989999999999998,"5min":6.2679999999999998,"15min":62.658000000000001},"last":1.167},{"interface":"front","average":{"1min":1.734,"5min":0.995,"15min":1.462},"min":{"1min":0.35299999999999998,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":62.573},"last":0.54100000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757682,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1031620,"kb_used_data":138108,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":92713532,"statfs":{"total":95995035648,"available":94938656768,"internally_reserved":0,"allocated":141422592,"data_stored":140478672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.0990000000000002,"15min":3.4900000000000002},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":3.819,"5min":131.87200000000001,"15min":131.87200000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.8769999999999998,"15min":3.4260000000000002},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":3.6709999999999998,"5min":131.80199999999999,"15min":131.80199999999999},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":9.0109999999999992,"5min":4.9610000000000003,"15min":2.843},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23400000000000001},"max":{"1min":87.795000000000002,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":10.266999999999999,"5min":5.2930000000000001,"15min":3.0350000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":87.856999999999999,"5min":126.931,"15min":126.931},"last":0.71799999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3599999999999999,"5min":2.52,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":16.5,"5min":51.393000000000001,"15min":81.260000000000005},"last":0.28999999999999998},{"interface":"front","average":{"1min":2.282,"5min":2.1789999999999998,"15min":2.1840000000000002},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5679999999999996,"5min":51.432000000000002,"15min":81.245000000000005},"last":1.23}]},{"osd":3,"last update":"Mon Oct 9 06:31:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5049999999999999,"5min":9.5310000000000006,"15min":5.4580000000000002},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":92.521000000000001,"5min":144.56399999999999,"15min":190.411},"last":0.499},{"interface":"front","average":{"1min":5.4720000000000004,"5min":9.4909999999999997,"15min":5.3449999999999998},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":94.492000000000004,"5min":144.62799999999999,"15min":190.70699999999999},"last":1.119}]},{"osd":4,"last update":"Mon Oct 9 06:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":1.0649999999999999,"15min":1.3759999999999999},"min":{"1min":0.311,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.9119999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":1.149},{"interface":"front","average":{"1min":1.8300000000000001,"5min":0.90400000000000003,"15min":1.1619999999999999},"min":{"1min":0.33600000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2610000000000001,"5min":6.2610000000000001,"15min":32.82},"last":0.621}]},{"osd":6,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3080000000000001,"15min":1.5509999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":3.6859999999999999,"5min":6.2080000000000002,"15min":38.902000000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.395,"15min":1.5920000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.6340000000000003,"5min":8.3659999999999997,"15min":35.774000000000001},"last":0.33500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:31:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.363,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.226,"5min":10.461,"15min":30.658999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.1970000000000001,"15min":1.448},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":3.1930000000000001,"5min":6.2320000000000002,"15min":30.683},"last":0.58299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":235470848,"data_stored":235470848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":291766272,"data_stored":291766272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":396230656,"data_stored":396230656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":217317376,"data_stored":217317376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":200146944,"data_stored":200146944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":140836864,"data_stored":140836864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":312213504,"data_stored":312213504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":479985664,"data_stored":479985664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:31:24.032 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-10-09T06:31:24.049 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:24.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:24.157 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:24.269 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:24.279 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:24.394 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:24.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:24.509 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:24.582 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:24.684 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:24.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:24.795 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:24.847 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:24 smithi107 ceph-mon[212750]: pgmap v1135: 99 pgs: 99 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:31:24.860 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:24.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 28129 28113 135.127 99.875 0.0119624 0.00739521 2023-10-09T06:31:25.066 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:25.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:25.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:24 smithi150 ceph-mon[194178]: pgmap v1135: 99 pgs: 99 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:31:25.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:24 smithi107 ceph-mon[220921]: pgmap v1135: 99 pgs: 99 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T06:31:25.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:25.233 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:25.369 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:25.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:25.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:25.440 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:25.440 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:25.478 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:25.508 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:25.549 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:25.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:25.652 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:25.690 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:25.759 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:25.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:25.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:25.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:25.910 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:25.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 29499 29483 131.591 85.625 0.00202418 0.00756849 2023-10-09T06:31:26.054 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:26.065 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:31:26.071 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:26.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:26.176 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:26.267 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:26.543 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:26.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:26.603 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:31:26.650 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:26.697 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:26.769 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:26.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:26.855 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:26 smithi107 ceph-mon[220921]: pgmap v1136: 99 pgs: 99 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:31:26.877 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:26.908 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:26.918 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:26.940 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 30772 30756 128.123 79.5625 0.0136637 0.00779803 2023-10-09T06:31:26.944 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:26.978 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:26.990 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:26.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:26.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:27.015 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:27.055 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:27.091 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:27.096 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:27.132 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:27.149 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:27.150 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:26 smithi150 ceph-mon[194178]: pgmap v1136: 99 pgs: 99 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:31:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:26 smithi107 ceph-mon[212750]: pgmap v1136: 99 pgs: 99 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-10-09T06:31:27.154 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:27.172 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.175 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:31:27.227 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:27.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:27.338 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:27.383 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:27.501 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:27.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:27.610 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:27.639 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3943368929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:27.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3943368929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:27.940 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 32134 32118 125.435 85.125 0.00681906 0.00796669 2023-10-09T06:31:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3943368929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:28.562 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:28.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:28.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:28.671 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:28.727 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:28.739 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:28.892 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:28.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:28.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 33389 33373 122.67 78.4375 0.0105136 0.00814653 2023-10-09T06:31:28.998 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:29.075 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:29.097 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:28 smithi107 ceph-mon[212750]: pgmap v1137: 99 pgs: 99 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:31:29.098 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:28 smithi107 ceph-mon[220921]: pgmap v1137: 99 pgs: 99 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:31:29.117 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:28 smithi150 ceph-mon[194178]: pgmap v1137: 99 pgs: 99 active+clean; 1.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-09T06:31:29.242 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:29.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:29.347 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:29.391 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:31:29.391 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:31:29.391 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:31:29.391 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:31:29.392 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:31:29.405 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:29.708 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:29.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:29.819 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:29.884 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:29.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 34924 34908 121.184 95.9375 0.00561313 0.0082323 2023-10-09T06:31:29.944 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:29.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:29.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:30.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:30.078 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:30.131 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:30.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:30.236 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:30.298 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:30.494 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:31:30.518 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:30.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:30.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:30.625 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:30.667 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:30.754 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:30.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:30.863 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:30.869 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:31:30.897 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:30.922 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 1 pgp_num to 11 2023-10-09T06:31:30.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 36313 36297 119.375 86.8125 0.0145463 0.00837129 2023-10-09T06:31:30.963 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:30.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:31.023 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:31.072 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:30 smithi150 ceph-mon[194178]: pgmap v1138: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T06:31:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3408627925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:30 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[212750]: pgmap v1138: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3408627925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[220921]: pgmap v1138: 99 pgs: 99 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3408627925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:30 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T06:31:31.184 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:31.333 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:31.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:31.440 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:31.488 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:31.655 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:31.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:31.759 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:31.785 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:31.903 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:31:31.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:31:31.942003+0000 min lat: 0.00111887 max lat: 0.297668 avg lat: 0.00855005 2023-10-09T06:31:31.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:31:31.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 37241 37225 116.306 58 0.00211638 0.00855005 2023-10-09T06:31:32.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:32] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1768851310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1768851310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: osdmap e367: 8 total, 8 up, 8 in 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: failed to encode map e367 with expected crc 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1768851310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:31:32.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:32.324 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:32.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:32.432 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:32.474 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:32.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 38377 38361 114.148 71 0.011105 0.00875565 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: pgmap v1140: 99 pgs: 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:31:32.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.973 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.973 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[212750]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.973 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:32.974 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: pgmap v1140: 99 pgs: 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:31:32.974 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:32.974 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:31:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.975 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:32 smithi107 ceph-mon[220921]: failed to encode map e368 with expected crc 2023-10-09T06:31:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:32.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82048 10.1 10:87d8b6ab:::benchmark_data_smithi107_363200_object82047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:31:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.813958+0000", 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 487.20448379700002, 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99731987799999999, 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82048 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.813958+0000", 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.813958+0000", 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.813957+0000", 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814000+0000", 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3476000000000002e-05 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814014+0000", 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3893000000000001e-05 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814019+0000", 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3299999999999998e-06 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.944334+0000", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13031504499999999 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.944345+0000", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1054e-05 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.811178+0000", 2023-10-09T06:31:33.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.86683222999999998 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.811278+0000", 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100454 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.028 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82052 10.11 10:8f39933d:::benchmark_data_smithi107_363200_object82051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:25.814028+0000", 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 487.20441364099997, 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0211701980000001, 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82052 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814028+0000", 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814028+0000", 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814028+0000", 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814067+0000", 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9162999999999998e-05 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814076+0000", 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2839999999999995e-06 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.814081+0000", 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3120000000000004e-06 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.953122+0000", 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.139040944 2023-10-09T06:31:33.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:25.953134+0000", 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2599000000000001e-05 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.835096+0000", 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88196197499999995 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.835199+0000", 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010241199999999999 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:82071 10.11 10:8db38651:::benchmark_data_smithi107_363200_object82070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:26.129576+0000", 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 486.88886653399999, 2023-10-09T06:31:33.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71292027999999996, 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 82071 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129576+0000", 2023-10-09T06:31:33.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129576+0000", 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129574+0000", 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129621+0000", 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6462999999999999e-05 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129636+0000", 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5242000000000001e-05 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129642+0000", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5150000000000003e-06 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129695+0000", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2973999999999998e-05 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.129705+0000", 2023-10-09T06:31:33.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8169999999999994e-06 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.835222+0000", 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.70551679899999997 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.835343+0000", 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012065099999999999 2023-10-09T06:31:33.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.835449+0000", 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010599699999999999 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.837077+0000", 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001628338 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.837544+0000", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000467295 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.842371+0000", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048270880000000002 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.842449+0000", 2023-10-09T06:31:33.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7863999999999999e-05 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:26.842496+0000", 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6539000000000003e-05 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.042 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:105532 10.10 10:0bbe1a44:::benchmark_data_smithi107_363200_object1440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:57.486546+0000", 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 455.531896507, 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.67920663100000001, 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 105532 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486546+0000", 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486546+0000", 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486544+0000", 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486549+0000", 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9200000000000003e-06 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486562+0000", 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2325e-05 2023-10-09T06:31:33.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486567+0000", 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3650000000000003e-06 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486646+0000", 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9308999999999996e-05 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.486656+0000", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8009999999999998e-06 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.157540+0000", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.67088381100000005 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.157658+0000", 2023-10-09T06:31:33.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011807399999999999 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.157825+0000", 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016694 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.160916+0000", 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003091042 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165702+0000", 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004785905 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165718+0000", 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5719999999999999e-05 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165736+0000", 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8827000000000001e-05 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165752+0000", 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5854000000000001e-05 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:105574 10.10 10:0f6c8bab:::benchmark_data_smithi107_363200_object1482:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:57.569307+0000", 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 455.449134716, 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.59656459299999998, 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 105574 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.051 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569307+0000", 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569307+0000", 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569306+0000", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569313+0000", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0500000000000003e-06 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569329+0000", 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5673000000000001e-05 2023-10-09T06:31:33.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569336+0000", 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1289999999999998e-06 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569400+0000", 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4389999999999998e-05 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.569409+0000", 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9889999999999998e-06 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.158836+0000", 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.58942658299999995 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.158951+0000", 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115303 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.159137+0000", 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018595499999999999 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.163342+0000", 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042050220000000001 2023-10-09T06:31:33.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165769+0000", 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024268760000000001 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165834+0000", 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5030000000000006e-05 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165855+0000", 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0729000000000001e-05 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165872+0000", 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7158000000000001e-05 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:105669 10.10 10:084caf6d:::benchmark_data_smithi107_363200_object1577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:23:57.686743+0000", 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 455.33169874999999, 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.47919561999999999, 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 105669 2023-10-09T06:31:33.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686743+0000", 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686743+0000", 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686742+0000", 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686748+0000", 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7919999999999998e-06 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686759+0000", 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.133e-05 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686766+0000", 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9310000000000002e-06 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686822+0000", 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5566999999999998e-05 2023-10-09T06:31:33.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:57.686830+0000", 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9349999999999994e-06 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.159904+0000", 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.47307378300000003 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.160009+0000", 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105583 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 6,7", 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.160186+0000", 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000176706 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.164826+0000", 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046397529999999999 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165806+0000", 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00098019499999999989 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165905+0000", 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9029000000000003e-05 2023-10-09T06:31:33.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165923+0000", 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8456000000000001e-05 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:23:58.165939+0000", 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5553000000000002e-05 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.066 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:206697 10.e 10:748c2ac0:::benchmark_data_smithi107_363200_object102605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 329.74200178799998, 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29428934499999998, 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:31:33.067 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 206697 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:31:33.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276439+0000", 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276446+0000", 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4529999999999999e-06 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276460+0000", 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4163e-05 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276466+0000", 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276517+0000", 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1008999999999999e-05 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276641+0000", 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012383699999999999 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276760+0000", 2023-10-09T06:31:33.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119003 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.280387+0000", 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036266850000000002 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.283508+0000", 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031211300000000002 2023-10-09T06:31:33.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570636+0000", 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28712860299999998 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570683+0000", 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6950000000000003e-05 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570730+0000", 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6347000000000002e-05 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:49050 11.e 11:7f8fd6d3:::benchmark_data_smithi107_418363_object49049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 266.745682407, 2023-10-09T06:31:33.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31877694499999998, 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 49050 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272758+0000", 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272861+0000", 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010279 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272876+0000", 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.537e-05 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272881+0000", 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0259999999999998e-06 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272923+0000", 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1832000000000002e-05 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.273063+0000", 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013991399999999999 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.273209+0000", 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014560199999999999 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.277138+0000", 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003928941 2023-10-09T06:31:33.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.277154+0000", 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6611000000000001e-05 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591443+0000", 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31428866700000002 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591495+0000", 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1518999999999999e-05 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591537+0000", 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2082000000000001e-05 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117048 11.d 11:b97729d1:::benchmark_data_smithi107_418363_object26612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.360991762, 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.039298464, 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117048 2023-10-09T06:31:33.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657449+0000", 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657457+0000", 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8329999999999994e-06 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657470+0000", 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3081999999999999e-05 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657477+0000", 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8279999999999997e-06 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657537+0000", 2023-10-09T06:31:33.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0040000000000001e-05 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657546+0000", 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4749999999999999e-06 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.661510+0000", 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.003963478 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.663496+0000", 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019859529999999999 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.663578+0000", 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2823000000000003e-05 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676713+0000", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013134866 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696643+0000", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019930184 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696673+0000", 2023-10-09T06:31:33.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9291000000000001e-05 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696712+0000", 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9456000000000003e-05 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696749+0000", 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.663e-05 2023-10-09T06:31:33.091 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117050 11.d 11:b78e64bd:::benchmark_data_smithi107_418363_object26614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.35405628000001, 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0325316689999999, 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.092 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117050 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664384+0000", 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664391+0000", 2023-10-09T06:31:33.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.447e-06 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664417+0000", 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5969000000000002e-05 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664438+0000", 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0896000000000001e-05 2023-10-09T06:31:33.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664529+0000", 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0749999999999997e-05 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664539+0000", 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0315e-05 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666048+0000", 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0015085319999999 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666153+0000", 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105499 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666223+0000", 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9479000000000003e-05 2023-10-09T06:31:33.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676722+0000", 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010498802 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696799+0000", 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020077426999999998 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696845+0000", 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5605999999999997e-05 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696884+0000", 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8911000000000001e-05 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696917+0000", 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3878999999999999e-05 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117064 11.d 11:be7c3437:::benchmark_data_smithi107_418363_object26628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.33838732000001, 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0170446479999999, 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.100 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117064 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680053+0000", 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680063+0000", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.057e-05 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680083+0000", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9009999999999999e-05 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680091+0000", 2023-10-09T06:31:33.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6989999999999992e-06 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.682293+0000", 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022022460000000002 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.682300+0000", 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668425+0000", 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98612509199999998 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668545+0000", 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119864 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668673+0000", 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012771700000000001 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682177+0000", 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013504063 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696961+0000", 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014783918 2023-10-09T06:31:33.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697034+0000", 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3361000000000005e-05 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697068+0000", 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4257000000000001e-05 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697099+0000", 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1103999999999999e-05 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117099 11.d 11:b81849c2:::benchmark_data_smithi107_418363_object26663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.278608542, 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95753685700000002, 2023-10-09T06:31:33.108 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117099 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739832+0000", 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739840+0000", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4880000000000007e-06 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739857+0000", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7003e-05 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739865+0000", 2023-10-09T06:31:33.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3950000000000003e-06 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739924+0000", 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9728000000000001e-05 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739933+0000", 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2740000000000007e-06 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671421+0000", 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93148836300000004 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671546+0000", 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012488199999999999 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671666+0000", 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120091 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682184+0000", 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010518323 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696999+0000", 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014814475000000001 2023-10-09T06:31:33.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697307+0000", 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000308654 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697341+0000", 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3426999999999997e-05 2023-10-09T06:31:33.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697370+0000", 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9592999999999999e-05 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117109 11.d 11:bb11f552:::benchmark_data_smithi107_418363_object26673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.266067134, 2023-10-09T06:31:33.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94509762100000005, 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117109 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:31:33.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752373+0000", 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752383+0000", 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4390000000000006e-06 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752402+0000", 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9057999999999999e-05 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752409+0000", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8220000000000001e-06 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752463+0000", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4107000000000002e-05 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752471+0000", 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3499999999999997e-06 2023-10-09T06:31:33.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674412+0000", 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92194090200000001 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674531+0000", 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011925399999999999 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674654+0000", 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012284199999999999 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692198+0000", 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017543665999999999 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697138+0000", 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049405960000000002 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697408+0000", 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00026925000000000001 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697442+0000", 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4663000000000003e-05 2023-10-09T06:31:33.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.125 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697473+0000", 2023-10-09T06:31:33.125 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0385000000000001e-05 2023-10-09T06:31:33.125 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.125 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.125 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117124 11.d 11:b2d54711:::benchmark_data_smithi107_418363_object26688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.24103268799999, 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92015732900000002, 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117124 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777408+0000", 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777416+0000", 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7450000000000001e-06 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777446+0000", 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9723999999999999e-05 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777454+0000", 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.323e-06 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777510+0000", 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6020000000000002e-05 2023-10-09T06:31:33.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777517+0000", 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5700000000000004e-06 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676769+0000", 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89925138699999996 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676871+0000", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000102178 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676940+0000", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9088e-05 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692205+0000", 2023-10-09T06:31:33.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015265261 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697174+0000", 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049690460000000004 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697505+0000", 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00033072700000000001 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697537+0000", 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2613000000000001e-05 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697567+0000", 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9298e-05 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117132 11.d 11:b198dabb:::benchmark_data_smithi107_418363_object26696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.22904641900001, 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91594003000000002, 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117132 2023-10-09T06:31:33.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.135 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789394+0000", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789404+0000", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8099999999999992e-06 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789427+0000", 2023-10-09T06:31:33.136 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2948000000000001e-05 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789435+0000", 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789488+0000", 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2843999999999998e-05 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789498+0000", 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9620000000000005e-06 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679459+0000", 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88996140499999998 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679533+0000", 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4139000000000003e-05 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679616+0000", 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2577999999999994e-05 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696206+0000", 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016589967000000001 2023-10-09T06:31:33.139 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697210+0000", 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001004128 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705228+0000", 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080186179999999999 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705281+0000", 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2667000000000002e-05 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705336+0000", 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4673000000000003e-05 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117133 11.d 11:b5b48777:::benchmark_data_smithi107_418363_object26697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.22874904400001, 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91576685000000002, 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117133 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.142 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.143 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789692+0000", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789698+0000", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3300000000000004e-06 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789712+0000", 2023-10-09T06:31:33.144 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.379e-05 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789720+0000", 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3119999999999996e-06 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789756+0000", 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6071000000000001e-05 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.145 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789761+0000", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4310000000000002e-06 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682210+0000", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89244917300000004 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682327+0000", 2023-10-09T06:31:33.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011647800000000001 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682398+0000", 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1798000000000006e-05 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696212+0000", 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013813997 2023-10-09T06:31:33.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697245+0000", 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010323960000000001 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705380+0000", 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0081353640000000008 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.148 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705424+0000", 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4078e-05 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705460+0000", 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5547999999999999e-05 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.149 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117137 11.d 11:ba9f3cee:::benchmark_data_smithi107_418363_object26701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.22360153899999, 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91075980999999995, 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117137 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794840+0000", 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794845+0000", 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1250000000000001e-06 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794858+0000", 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3273e-05 2023-10-09T06:31:33.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794865+0000", 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.883e-06 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.795916+0000", 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010514109999999999 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.795929+0000", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.289e-05 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685114+0000", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88918451899999995 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685228+0000", 2023-10-09T06:31:33.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000114472 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685348+0000", 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011943300000000001 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696220+0000", 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010872527999999999 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697273+0000", 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010526730000000001 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705519+0000", 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082458070000000008 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705577+0000", 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7926000000000001e-05 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705600+0000", 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3782e-05 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.157 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117147 11.d 11:b7f258a5:::benchmark_data_smithi107_418363_object26711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.189261354, 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.87652914699999995, 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117147 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.158 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829179+0000", 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829189+0000", 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8519999999999999e-06 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829214+0000", 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4949000000000001e-05 2023-10-09T06:31:33.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829223+0000", 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4599999999999992e-06 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829291+0000", 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8280000000000004e-05 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.161 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829304+0000", 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2408e-05 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688474+0000", 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.85917075399999998 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688581+0000", 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106147 2023-10-09T06:31:33.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688687+0000", 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106715 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696345+0000", 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0076581649999999998 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.163 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.700894+0000", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004548837 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705634+0000", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047392659999999998 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705672+0000", 2023-10-09T06:31:33.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8198999999999998e-05 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705710+0000", 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8099999999999998e-05 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117155 11.d 11:bfe8c66f:::benchmark_data_smithi107_418363_object26719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:31:33.165 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 164.15274480100001, 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84011032600000002, 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117155 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.166 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865696+0000", 2023-10-09T06:31:33.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865714+0000", 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7901e-05 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865733+0000", 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9585999999999998e-05 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865743+0000", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6849999999999996e-06 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865819+0000", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6369e-05 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865827+0000", 2023-10-09T06:31:33.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7170000000000008e-06 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692234+0000", 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.82640717200000002 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692351+0000", 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117275 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692432+0000", 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0427999999999996e-05 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.699389+0000", 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0069568240000000003 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.171 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.700923+0000", 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015340040000000001 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705740+0000", 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048176399999999998 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705765+0000", 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4825e-05 2023-10-09T06:31:33.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705808+0000", 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2391000000000001e-05 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:32545 12.0 12:01fec15d:::benchmark_data_smithi107_479504_object32544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.861707902, 2023-10-09T06:31:33.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.280403281, 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 32545 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:31:33.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156733+0000", 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156799+0000", 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5947999999999999e-05 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156811+0000", 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2396e-05 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:33.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156818+0000", 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3509999999999998e-06 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156855+0000", 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7357000000000002e-05 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.177 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156935+0000", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9637999999999999e-05 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.157020+0000", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5840000000000005e-05 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158089+0000", 2023-10-09T06:31:33.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010683859999999999 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158849+0000", 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00076036400000000003 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437051+0000", 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27820162700000001 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437100+0000", 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9347e-05 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437137+0000", 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7234000000000003e-05 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.180 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:33.181 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:33.181 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:31:33.181 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:33.182 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: pgmap v1140: 99 pgs: 99 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: failed to encode map e368 with expected crc 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: osdmap e368: 8 total, 8 up, 8 in 2023-10-09T06:31:33.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: failed to encode map e368 with expected crc 2023-10-09T06:31:33.183 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: failed to encode map e368 with expected crc 2023-10-09T06:31:33.183 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:32 smithi150 ceph-mon[194178]: failed to encode map e368 with expected crc 2023-10-09T06:31:33.227 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:33.310 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:33.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:33.418 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:33.501 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:33.791 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:33.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:33.897 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:33.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 39841 39825 113.119 91.5 0.00272139 0.00883442 2023-10-09T06:31:33.973 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:34.072 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:34.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:34.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:34.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:31:34.179 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:34.206 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:34.247 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:34.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:34.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:34.351 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:34.377 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:34.434 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:34.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:34.487 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:34.542 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:34.817 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:34.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:34.928 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:34.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 40909 40893 111.102 66.75 0.0121581 0.00899226 2023-10-09T06:31:34.975 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:31:34.994 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:35.037 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:34 smithi107 ceph-mon[212750]: pgmap v1142: 99 pgs: 99 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-10-09T06:31:35.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:34 smithi107 ceph-mon[220921]: pgmap v1142: 99 pgs: 99 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-10-09T06:31:35.084 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:34 smithi150 ceph-mon[194178]: pgmap v1142: 99 pgs: 99 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-10-09T06:31:35.201 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:35.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:35.317 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:35.347 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:35.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:35.447 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:35.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:35.464 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:35.468 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:35.486 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:35.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:35.528 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:35.547 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:35.551 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:35.579 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:35.635 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:35.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:35.646 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:35.682 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:35.684 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.684 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.684 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.684 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.685 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.685 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.685 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.685 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:31:35.740 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:35.779 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:35.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 41886 41870 109.017 61.0625 0.0124802 0.00916798 2023-10-09T06:31:35.945 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:35.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:36.059 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:36.105 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2635563149' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:36.105 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2635563149' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:36.130 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:36.242 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2635563149' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:36.530 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:36.530 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:36.941 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 43082 43066 107.647 74.75 0.00228784 0.0092711 2023-10-09T06:31:37.103 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:37.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:37.207 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.248 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:36 smithi150 ceph-mon[194178]: pgmap v1143: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:37.253 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:37.323 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:36 smithi107 ceph-mon[212750]: pgmap v1143: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:37.323 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:36 smithi107 ceph-mon[220921]: pgmap v1143: 99 pgs: 99 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:37.453 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:37.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:37.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:37.513 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:31:37.559 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:37.630 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:37.720 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:37.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:37.737 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:31:37.737 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:31:37.737 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:31:37.737 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:31:37.827 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:37.869 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:37.945 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:37.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 44310 44294 106.445 76.75 0.00833539 0.0093891 2023-10-09T06:31:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:37.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:38.020 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:38.052 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:38.095 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:38.176 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:38.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:38.279 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:38.320 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:38.428 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:38.499 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:38.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:38.605 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:38.676 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:38.798 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:38.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:38.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:38.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:38.930 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:38.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 45597 45581 105.482 80.4375 0.00262824 0.00946137 2023-10-09T06:31:38.945 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:31:38.977 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:38.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:39.081 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:39.119 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:39.365 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:31:39.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[220921]: pgmap v1144: 99 pgs: 99 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:31:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/864804089' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[212750]: pgmap v1144: 99 pgs: 99 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:31:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/864804089' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:38 smithi150 ceph-mon[194178]: pgmap v1144: 99 pgs: 99 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T06:31:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/864804089' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:39.437 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:39.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:39.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:39.542 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:39.611 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:39.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 46534 46518 103.806 58.5625 0.00792523 0.00956453 2023-10-09T06:31:40.051 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:40.051 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:31:40.081 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:39 smithi150 ceph-mon[194178]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:31:40.249 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[212750]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.251 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:31:40.251 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:31:40.251 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.251 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:40.252 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: osdmap e369: 8 total, 8 up, 8 in 2023-10-09T06:31:40.252 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:39 smithi107 ceph-mon[220921]: failed to encode map e369 with expected crc 2023-10-09T06:31:40.874 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:40.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:40.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:40.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 47652 47636 102.636 69.875 0.0135899 0.00973713 2023-10-09T06:31:40.946 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:31:40.982 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:41.028 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:41.070 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:41.115 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:41.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:40 smithi107 ceph-mon[212750]: pgmap v1146: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T06:31:41.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:40 smithi107 ceph-mon[220921]: pgmap v1146: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T06:31:41.227 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:41.271 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:40 smithi150 ceph-mon[194178]: pgmap v1146: 99 pgs: 99 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-09T06:31:41.275 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:41.370 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:41.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:41.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:41.480 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:41.485 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:41.556 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:41.692 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:41.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:41.798 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:41.836 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:41.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 48903 48887 101.821 78.1875 0.0197207 0.00981579 2023-10-09T06:31:41.968 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:31:41.976 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:31:42.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:42.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:42.152 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:42.201 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:42.268 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:42.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:42.372 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:42.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:42] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T06:31:42.430 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:42.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 50122 50106 100.994 76.1875 0.0104083 0.00989604 2023-10-09T06:31:42.997 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:31:43.103 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:43.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:43.213 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:43.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:42 smithi150 ceph-mon[194178]: pgmap v1147: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:31:43.240 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:42 smithi150 ceph-mon[194178]: from='client.56263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:43.242 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:43.401 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:43.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:42 smithi107 ceph-mon[212750]: pgmap v1147: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:31:43.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:42 smithi107 ceph-mon[212750]: from='client.56263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:43.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:42 smithi107 ceph-mon[220921]: pgmap v1147: 99 pgs: 99 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T06:31:43.407 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:42 smithi107 ceph-mon[220921]: from='client.56263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:31:43.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:43.508 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:43.604 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:43.663 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:43.663 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:43.756 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1147,"stamp":"2023-10-09T06:31:41.324269+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3041204128,"num_objects":46420,"num_object_clones":0,"num_object_copies":139260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46457,"num_write_kb":2969373,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":600556,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48222,"ondisk_log_size":48222,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15678636,"kb_used_data":9527532,"kb_used_omap":20,"kb_used_meta":6151019,"kb_avail":734282580,"statfs":{"total":767960285184,"available":751905361920,"internally_reserved":0,"allocated":9756192768,"data_stored":9745570115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6298644110},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":763297792,"num_objects":11647,"num_object_clones":0,"num_object_copies":34941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11647,"num_write_kb":745408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.427701"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":347,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908494+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:31:30.908494+0000","last_peered":"2023-10-09T06:31:30.908494+0000","last_clean":"2023-10-09T06:31:30.908494+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:31:30.908494+0000","last_undegraded":"2023-10-09T06:31:30.908494+0000","last_fullsized":"2023-10-09T06:31:30.908494+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898821+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:31:31.898821+0000","last_peered":"2023-10-09T06:31:31.898821+0000","last_clean":"2023-10-09T06:31:31.898821+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:31:31.898821+0000","last_undegraded":"2023-10-09T06:31:31.898821+0000","last_fullsized":"2023-10-09T06:31:31.898821+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":360,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.898851+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:31:30.898851+0000","last_peered":"2023-10-09T06:31:30.898851+0000","last_clean":"2023-10-09T06:31:30.898851+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:31:30.898851+0000","last_undegraded":"2023-10-09T06:31:30.898851+0000","last_fullsized":"2023-10-09T06:31:30.898851+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":365,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900900+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:31:31.900900+0000","last_peered":"2023-10-09T06:31:31.900900+0000","last_clean":"2023-10-09T06:31:31.900900+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:31:31.900900+0000","last_undegraded":"2023-10-09T06:31:31.900900+0000","last_fullsized":"2023-10-09T06:31:31.900900+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898477+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:31:31.898477+0000","last_peered":"2023-10-09T06:31:31.898477+0000","last_clean":"2023-10-09T06:31:31.898477+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:31:31.898477+0000","last_undegraded":"2023-10-09T06:31:31.898477+0000","last_fullsized":"2023-10-09T06:31:31.898477+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":358,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108836+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:31:32.108836+0000","last_peered":"2023-10-09T06:31:32.108836+0000","last_clean":"2023-10-09T06:31:32.108836+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:31:32.108836+0000","last_undegraded":"2023-10-09T06:31:32.108836+0000","last_fullsized":"2023-10-09T06:31:32.108836+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":41,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:38.937719+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:31:38.937719+0000","last_peered":"2023-10-09T06:31:38.937719+0000","last_clean":"2023-10-09T06:31:38.937719+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:31:38.937719+0000","last_undegraded":"2023-10-09T06:31:38.937719+0000","last_fullsized":"2023-10-09T06:31:38.937719+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898922+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:31:31.898922+0000","last_peered":"2023-10-09T06:31:31.898922+0000","last_clean":"2023-10-09T06:31:31.898922+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:31:31.898922+0000","last_undegraded":"2023-10-09T06:31:31.898922+0000","last_fullsized":"2023-10-09T06:31:31.898922+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900224+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:31:30.900224+0000","last_peered":"2023-10-09T06:31:30.900224+0000","last_clean":"2023-10-09T06:31:30.900224+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:31:30.900224+0000","last_undegraded":"2023-10-09T06:31:30.900224+0000","last_fullsized":"2023-10-09T06:31:30.900224+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898825+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:31:31.898825+0000","last_peered":"2023-10-09T06:31:31.898825+0000","last_clean":"2023-10-09T06:31:31.898825+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:31:31.898825+0000","last_undegraded":"2023-10-09T06:31:31.898825+0000","last_fullsized":"2023-10-09T06:31:31.898825+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":249,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.907198+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:31:30.907198+0000","last_peered":"2023-10-09T06:31:30.907198+0000","last_clean":"2023-10-09T06:31:30.907198+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:31:30.907198+0000","last_undegraded":"2023-10-09T06:31:30.907198+0000","last_fullsized":"2023-10-09T06:31:30.907198+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":301,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899623+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:31:30.899623+0000","last_peered":"2023-10-09T06:31:30.899623+0000","last_clean":"2023-10-09T06:31:30.899623+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:31:30.899623+0000","last_undegraded":"2023-10-09T06:31:30.899623+0000","last_fullsized":"2023-10-09T06:31:30.899623+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":374,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908745+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:31:30.908745+0000","last_peered":"2023-10-09T06:31:30.908745+0000","last_clean":"2023-10-09T06:31:30.908745+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:31:30.908745+0000","last_undegraded":"2023-10-09T06:31:30.908745+0000","last_fullsized":"2023-10-09T06:31:30.908745+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":338,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908505+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:31:30.908505+0000","last_peered":"2023-10-09T06:31:30.908505+0000","last_clean":"2023-10-09T06:31:30.908505+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:31:30.908505+0000","last_undegraded":"2023-10-09T06:31:30.908505+0000","last_fullsized":"2023-10-09T06:31:30.908505+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":217,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906964+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:31:30.906964+0000","last_peered":"2023-10-09T06:31:30.906964+0000","last_clean":"2023-10-09T06:31:30.906964+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:31:30.906964+0000","last_undegraded":"2023-10-09T06:31:30.906964+0000","last_fullsized":"2023-10-09T06:31:30.906964+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.899937+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:31:31.899937+0000","last_peered":"2023-10-09T06:31:31.899937+0000","last_clean":"2023-10-09T06:31:31.899937+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:31:31.899937+0000","last_undegraded":"2023-10-09T06:31:31.899937+0000","last_fullsized":"2023-10-09T06:31:31.899937+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":364,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:35.673296+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:31:35.673296+0000","last_peered":"2023-10-09T06:31:35.673296+0000","last_clean":"2023-10-09T06:31:35.673296+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:31:35.673296+0000","last_undegraded":"2023-10-09T06:31:35.673296+0000","last_fullsized":"2023-10-09T06:31:35.673296+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900607+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:31:30.900607+0000","last_peered":"2023-10-09T06:31:30.900607+0000","last_clean":"2023-10-09T06:31:30.900607+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:31:30.900607+0000","last_undegraded":"2023-10-09T06:31:30.900607+0000","last_fullsized":"2023-10-09T06:31:30.900607+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906617+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:31:30.906617+0000","last_peered":"2023-10-09T06:31:30.906617+0000","last_clean":"2023-10-09T06:31:30.906617+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:31:30.906617+0000","last_undegraded":"2023-10-09T06:31:30.906617+0000","last_fullsized":"2023-10-09T06:31:30.906617+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":41,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:38.937768+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:31:38.937768+0000","last_peered":"2023-10-09T06:31:38.937768+0000","last_clean":"2023-10-09T06:31:38.937768+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:31:38.937768+0000","last_undegraded":"2023-10-09T06:31:38.937768+0000","last_fullsized":"2023-10-09T06:31:38.937768+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":447,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108204+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:31:32.108204+0000","last_peered":"2023-10-09T06:31:32.108204+0000","last_clean":"2023-10-09T06:31:32.108204+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:31:32.108204+0000","last_undegraded":"2023-10-09T06:31:32.108204+0000","last_fullsized":"2023-10-09T06:31:32.108204+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.899167+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:31:31.899167+0000","last_peered":"2023-10-09T06:31:31.899167+0000","last_clean":"2023-10-09T06:31:31.899167+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:31:31.899167+0000","last_undegraded":"2023-10-09T06:31:31.899167+0000","last_fullsized":"2023-10-09T06:31:31.899167+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"368'2789","reported_seq":2804,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:38.219628+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:31:38.219628+0000","last_peered":"2023-10-09T06:31:38.219628+0000","last_clean":"2023-10-09T06:31:38.219628+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:31:38.219628+0000","last_undegraded":"2023-10-09T06:31:38.219628+0000","last_fullsized":"2023-10-09T06:31:38.219628+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2789,"log_dups_size":0,"ondisk_log_size":2789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2789,"num_write_kb":178496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":253,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.907024+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:31:30.907024+0000","last_peered":"2023-10-09T06:31:30.907024+0000","last_clean":"2023-10-09T06:31:30.907024+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:31:30.907024+0000","last_undegraded":"2023-10-09T06:31:30.907024+0000","last_fullsized":"2023-10-09T06:31:30.907024+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900042+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:31:30.900042+0000","last_peered":"2023-10-09T06:31:30.900042+0000","last_clean":"2023-10-09T06:31:30.900042+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:31:30.900042+0000","last_undegraded":"2023-10-09T06:31:30.900042+0000","last_fullsized":"2023-10-09T06:31:30.900042+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"369'3054","reported_seq":3070,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:41.161890+0000","last_change":"2023-10-09T06:31:02.763465+0000","last_active":"2023-10-09T06:31:41.161890+0000","last_peered":"2023-10-09T06:31:41.161890+0000","last_clean":"2023-10-09T06:31:41.161890+0000","last_became_active":"2023-10-09T06:31:02.763153+0000","last_became_peered":"2023-10-09T06:31:02.763153+0000","last_unstale":"2023-10-09T06:31:41.161890+0000","last_undegraded":"2023-10-09T06:31:41.161890+0000","last_fullsized":"2023-10-09T06:31:41.161890+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3054,"log_dups_size":0,"ondisk_log_size":3054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:01.874473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200146944,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3054,"num_write_kb":195456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":215,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.907308+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:31:30.907308+0000","last_peered":"2023-10-09T06:31:30.907308+0000","last_clean":"2023-10-09T06:31:30.907308+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:31:30.907308+0000","last_undegraded":"2023-10-09T06:31:30.907308+0000","last_fullsized":"2023-10-09T06:31:30.907308+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":365,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.909401+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:31:30.909401+0000","last_peered":"2023-10-09T06:31:30.909401+0000","last_clean":"2023-10-09T06:31:30.909401+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:31:30.909401+0000","last_undegraded":"2023-10-09T06:31:30.909401+0000","last_fullsized":"2023-10-09T06:31:30.909401+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"368'2806","reported_seq":2820,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:38.222901+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:31:38.222901+0000","last_peered":"2023-10-09T06:31:38.222901+0000","last_clean":"2023-10-09T06:31:38.222901+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:31:38.222901+0000","last_undegraded":"2023-10-09T06:31:38.222901+0000","last_fullsized":"2023-10-09T06:31:38.222901+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2806,"log_dups_size":0,"ondisk_log_size":2806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183894016,"num_objects":2806,"num_object_clones":0,"num_object_copies":8418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2806,"num_write_kb":179584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":271,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906987+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:31:30.906987+0000","last_peered":"2023-10-09T06:31:30.906987+0000","last_clean":"2023-10-09T06:31:30.906987+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:31:30.906987+0000","last_undegraded":"2023-10-09T06:31:30.906987+0000","last_fullsized":"2023-10-09T06:31:30.906987+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":345,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108448+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:31:32.108448+0000","last_peered":"2023-10-09T06:31:32.108448+0000","last_clean":"2023-10-09T06:31:32.108448+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:31:32.108448+0000","last_undegraded":"2023-10-09T06:31:32.108448+0000","last_fullsized":"2023-10-09T06:31:32.108448+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"369'2925","reported_seq":2942,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:40.659495+0000","last_change":"2023-10-09T06:31:02.761953+0000","last_active":"2023-10-09T06:31:40.659495+0000","last_peered":"2023-10-09T06:31:40.659495+0000","last_clean":"2023-10-09T06:31:40.659495+0000","last_became_active":"2023-10-09T06:31:02.761625+0000","last_became_peered":"2023-10-09T06:31:02.761625+0000","last_unstale":"2023-10-09T06:31:40.659495+0000","last_undegraded":"2023-10-09T06:31:40.659495+0000","last_fullsized":"2023-10-09T06:31:40.659495+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2925,"log_dups_size":0,"ondisk_log_size":2925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:31.403447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191692800,"num_objects":2925,"num_object_clones":0,"num_object_copies":8775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2925,"num_write_kb":187200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900907+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:31:30.900907+0000","last_peered":"2023-10-09T06:31:30.900907+0000","last_clean":"2023-10-09T06:31:30.900907+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:31:30.900907+0000","last_undegraded":"2023-10-09T06:31:30.900907+0000","last_fullsized":"2023-10-09T06:31:30.900907+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":328,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900890+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:31:31.900890+0000","last_peered":"2023-10-09T06:31:31.900890+0000","last_clean":"2023-10-09T06:31:31.900890+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:31:31.900890+0000","last_undegraded":"2023-10-09T06:31:31.900890+0000","last_fullsized":"2023-10-09T06:31:31.900890+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"369'2968","reported_seq":2984,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:41.177474+0000","last_change":"2023-10-09T06:31:02.762571+0000","last_active":"2023-10-09T06:31:41.177474+0000","last_peered":"2023-10-09T06:31:41.177474+0000","last_clean":"2023-10-09T06:31:41.177474+0000","last_became_active":"2023-10-09T06:31:02.762287+0000","last_became_peered":"2023-10-09T06:31:02.762287+0000","last_unstale":"2023-10-09T06:31:41.177474+0000","last_undegraded":"2023-10-09T06:31:41.177474+0000","last_fullsized":"2023-10-09T06:31:41.177474+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2968,"log_dups_size":0,"ondisk_log_size":2968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:15.365778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194510848,"num_objects":2968,"num_object_clones":0,"num_object_copies":8904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2968,"num_write_kb":189952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":700,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906455+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:31:30.906455+0000","last_peered":"2023-10-09T06:31:30.906455+0000","last_clean":"2023-10-09T06:31:30.906455+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:31:30.906455+0000","last_undegraded":"2023-10-09T06:31:30.906455+0000","last_fullsized":"2023-10-09T06:31:30.906455+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":93,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898869+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:31:31.898869+0000","last_peered":"2023-10-09T06:31:31.898869+0000","last_clean":"2023-10-09T06:31:31.898869+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:31:31.898869+0000","last_undegraded":"2023-10-09T06:31:31.898869+0000","last_fullsized":"2023-10-09T06:31:31.898869+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":348,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908597+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:31:30.908597+0000","last_peered":"2023-10-09T06:31:30.908597+0000","last_clean":"2023-10-09T06:31:30.908597+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:31:30.908597+0000","last_undegraded":"2023-10-09T06:31:30.908597+0000","last_fullsized":"2023-10-09T06:31:30.908597+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"369'3023","reported_seq":3039,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:41.169926+0000","last_change":"2023-10-09T06:31:03.217042+0000","last_active":"2023-10-09T06:31:41.169926+0000","last_peered":"2023-10-09T06:31:41.169926+0000","last_clean":"2023-10-09T06:31:41.169926+0000","last_became_active":"2023-10-09T06:31:03.216729+0000","last_became_peered":"2023-10-09T06:31:03.216729+0000","last_unstale":"2023-10-09T06:31:41.169926+0000","last_undegraded":"2023-10-09T06:31:41.169926+0000","last_fullsized":"2023-10-09T06:31:41.169926+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3023,"log_dups_size":0,"ondisk_log_size":3023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.232295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198115328,"num_objects":3023,"num_object_clones":0,"num_object_copies":9069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3023,"num_write_kb":193472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":680,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900056+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:31:30.900056+0000","last_peered":"2023-10-09T06:31:30.900056+0000","last_clean":"2023-10-09T06:31:30.900056+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:31:30.900056+0000","last_undegraded":"2023-10-09T06:31:30.900056+0000","last_fullsized":"2023-10-09T06:31:30.900056+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":326,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108350+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:31:32.108350+0000","last_peered":"2023-10-09T06:31:32.108350+0000","last_clean":"2023-10-09T06:31:32.108350+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:31:32.108350+0000","last_undegraded":"2023-10-09T06:31:32.108350+0000","last_fullsized":"2023-10-09T06:31:32.108350+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":345,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.909113+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:31:30.909113+0000","last_peered":"2023-10-09T06:31:30.909113+0000","last_clean":"2023-10-09T06:31:30.909113+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:31:30.909113+0000","last_undegraded":"2023-10-09T06:31:30.909113+0000","last_fullsized":"2023-10-09T06:31:30.909113+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"368'2666","reported_seq":2682,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:37.403512+0000","last_change":"2023-10-09T06:31:02.762312+0000","last_active":"2023-10-09T06:31:37.403512+0000","last_peered":"2023-10-09T06:31:37.403512+0000","last_clean":"2023-10-09T06:31:37.403512+0000","last_became_active":"2023-10-09T06:31:02.762007+0000","last_became_peered":"2023-10-09T06:31:02.762007+0000","last_unstale":"2023-10-09T06:31:37.403512+0000","last_undegraded":"2023-10-09T06:31:37.403512+0000","last_fullsized":"2023-10-09T06:31:37.403512+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2666,"log_dups_size":0,"ondisk_log_size":2666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174718976,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2666,"num_write_kb":170624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":349,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.909173+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:31:30.909173+0000","last_peered":"2023-10-09T06:31:30.909173+0000","last_clean":"2023-10-09T06:31:30.909173+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:31:30.909173+0000","last_undegraded":"2023-10-09T06:31:30.909173+0000","last_fullsized":"2023-10-09T06:31:30.909173+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":328,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.909669+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:31:30.909669+0000","last_peered":"2023-10-09T06:31:30.909669+0000","last_clean":"2023-10-09T06:31:30.909669+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:31:30.909669+0000","last_undegraded":"2023-10-09T06:31:30.909669+0000","last_fullsized":"2023-10-09T06:31:30.909669+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"368'2674","reported_seq":2690,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:37.407162+0000","last_change":"2023-10-09T06:31:03.215765+0000","last_active":"2023-10-09T06:31:37.407162+0000","last_peered":"2023-10-09T06:31:37.407162+0000","last_clean":"2023-10-09T06:31:37.407162+0000","last_became_active":"2023-10-09T06:31:03.215425+0000","last_became_peered":"2023-10-09T06:31:03.215425+0000","last_unstale":"2023-10-09T06:31:37.407162+0000","last_undegraded":"2023-10-09T06:31:37.407162+0000","last_fullsized":"2023-10-09T06:31:37.407162+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2674,"log_dups_size":0,"ondisk_log_size":2674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175243264,"num_objects":2674,"num_object_clones":0,"num_object_copies":8022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2674,"num_write_kb":171136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":658,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900481+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:31:30.900481+0000","last_peered":"2023-10-09T06:31:30.900481+0000","last_clean":"2023-10-09T06:31:30.900481+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:31:30.900481+0000","last_undegraded":"2023-10-09T06:31:30.900481+0000","last_fullsized":"2023-10-09T06:31:30.900481+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898590+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:31:31.898590+0000","last_peered":"2023-10-09T06:31:31.898590+0000","last_clean":"2023-10-09T06:31:31.898590+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:31:31.898590+0000","last_undegraded":"2023-10-09T06:31:31.898590+0000","last_fullsized":"2023-10-09T06:31:31.898590+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":320,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900652+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:31:30.900652+0000","last_peered":"2023-10-09T06:31:30.900652+0000","last_clean":"2023-10-09T06:31:30.900652+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:31:30.900652+0000","last_undegraded":"2023-10-09T06:31:30.900652+0000","last_fullsized":"2023-10-09T06:31:30.900652+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"369'2937","reported_seq":2954,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:40.591557+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:31:40.591557+0000","last_peered":"2023-10-09T06:31:40.591557+0000","last_clean":"2023-10-09T06:31:40.591557+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:31:40.591557+0000","last_undegraded":"2023-10-09T06:31:40.591557+0000","last_fullsized":"2023-10-09T06:31:40.591557+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2937,"log_dups_size":0,"ondisk_log_size":2937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2937,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":631,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108936+0000","last_change":"2023-10-09T06:27:39.657881+0000","last_active":"2023-10-09T06:31:32.108936+0000","last_peered":"2023-10-09T06:31:32.108936+0000","last_clean":"2023-10-09T06:31:32.108936+0000","last_became_active":"2023-10-09T06:27:39.657522+0000","last_became_peered":"2023-10-09T06:27:39.657522+0000","last_unstale":"2023-10-09T06:31:32.108936+0000","last_undegraded":"2023-10-09T06:31:32.108936+0000","last_fullsized":"2023-10-09T06:31:32.108936+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:14:44.592371+0000","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":371,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899341+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:31:30.899341+0000","last_peered":"2023-10-09T06:31:30.899341+0000","last_clean":"2023-10-09T06:31:30.899341+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:31:30.899341+0000","last_undegraded":"2023-10-09T06:31:30.899341+0000","last_fullsized":"2023-10-09T06:31:30.899341+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900525+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:31:30.900525+0000","last_peered":"2023-10-09T06:31:30.900525+0000","last_clean":"2023-10-09T06:31:30.900525+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:31:30.900525+0000","last_undegraded":"2023-10-09T06:31:30.900525+0000","last_fullsized":"2023-10-09T06:31:30.900525+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":305,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899306+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:31:30.899306+0000","last_peered":"2023-10-09T06:31:30.899306+0000","last_clean":"2023-10-09T06:31:30.899306+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:31:30.899306+0000","last_undegraded":"2023-10-09T06:31:30.899306+0000","last_fullsized":"2023-10-09T06:31:30.899306+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"369'2941","reported_seq":2958,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:40.660127+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:31:40.660127+0000","last_peered":"2023-10-09T06:31:40.660127+0000","last_clean":"2023-10-09T06:31:40.660127+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:31:40.660127+0000","last_undegraded":"2023-10-09T06:31:40.660127+0000","last_fullsized":"2023-10-09T06:31:40.660127+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2941,"log_dups_size":0,"ondisk_log_size":2941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192741376,"num_objects":2941,"num_object_clones":0,"num_object_copies":8823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2941,"num_write_kb":188224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":694,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900173+0000","last_change":"2023-10-09T06:29:15.532600+0000","last_active":"2023-10-09T06:31:30.900173+0000","last_peered":"2023-10-09T06:31:30.900173+0000","last_clean":"2023-10-09T06:31:30.900173+0000","last_became_active":"2023-10-09T06:29:15.532290+0000","last_became_peered":"2023-10-09T06:29:15.532290+0000","last_unstale":"2023-10-09T06:31:30.900173+0000","last_undegraded":"2023-10-09T06:31:30.900173+0000","last_fullsized":"2023-10-09T06:31:30.900173+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:19:55.742439+0000","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900663+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:31:30.900663+0000","last_peered":"2023-10-09T06:31:30.900663+0000","last_clean":"2023-10-09T06:31:30.900663+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:31:30.900663+0000","last_undegraded":"2023-10-09T06:31:30.900663+0000","last_fullsized":"2023-10-09T06:31:30.900663+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":349,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908358+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:31:30.908358+0000","last_peered":"2023-10-09T06:31:30.908358+0000","last_clean":"2023-10-09T06:31:30.908358+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:31:30.908358+0000","last_undegraded":"2023-10-09T06:31:30.908358+0000","last_fullsized":"2023-10-09T06:31:30.908358+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":335,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899235+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:31:30.899235+0000","last_peered":"2023-10-09T06:31:30.899235+0000","last_clean":"2023-10-09T06:31:30.899235+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:31:30.899235+0000","last_undegraded":"2023-10-09T06:31:30.899235+0000","last_fullsized":"2023-10-09T06:31:30.899235+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"369'3096","reported_seq":3112,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:41.182114+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:31:41.182114+0000","last_peered":"2023-10-09T06:31:41.182114+0000","last_clean":"2023-10-09T06:31:41.182114+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:31:41.182114+0000","last_undegraded":"2023-10-09T06:31:41.182114+0000","last_fullsized":"2023-10-09T06:31:41.182114+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3096,"log_dups_size":0,"ondisk_log_size":3096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202899456,"num_objects":3096,"num_object_clones":0,"num_object_copies":9288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3096,"num_write_kb":198144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":690,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906785+0000","last_change":"2023-10-09T06:29:15.535685+0000","last_active":"2023-10-09T06:31:30.906785+0000","last_peered":"2023-10-09T06:31:30.906785+0000","last_clean":"2023-10-09T06:31:30.906785+0000","last_became_active":"2023-10-09T06:29:15.535396+0000","last_became_peered":"2023-10-09T06:29:15.535396+0000","last_unstale":"2023-10-09T06:31:30.906785+0000","last_undegraded":"2023-10-09T06:31:30.906785+0000","last_fullsized":"2023-10-09T06:31:30.906785+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:23.142031+0000","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":358,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898598+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:31:31.898598+0000","last_peered":"2023-10-09T06:31:31.898598+0000","last_clean":"2023-10-09T06:31:31.898598+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:31:31.898598+0000","last_undegraded":"2023-10-09T06:31:31.898598+0000","last_fullsized":"2023-10-09T06:31:31.898598+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":344,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899168+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:31:30.899168+0000","last_peered":"2023-10-09T06:31:30.899168+0000","last_clean":"2023-10-09T06:31:30.899168+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:31:30.899168+0000","last_undegraded":"2023-10-09T06:31:30.899168+0000","last_fullsized":"2023-10-09T06:31:30.899168+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":366,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900721+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:31:30.900721+0000","last_peered":"2023-10-09T06:31:30.900721+0000","last_clean":"2023-10-09T06:31:30.900721+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:31:30.900721+0000","last_undegraded":"2023-10-09T06:31:30.900721+0000","last_fullsized":"2023-10-09T06:31:30.900721+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"368'2867","reported_seq":2882,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:38.323309+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:31:38.323309+0000","last_peered":"2023-10-09T06:31:38.323309+0000","last_clean":"2023-10-09T06:31:38.323309+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:31:38.323309+0000","last_undegraded":"2023-10-09T06:31:38.323309+0000","last_fullsized":"2023-10-09T06:31:38.323309+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2867,"log_dups_size":0,"ondisk_log_size":2867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891712,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2867,"num_write_kb":183488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":55,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900235+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:31:30.900235+0000","last_peered":"2023-10-09T06:31:30.900235+0000","last_clean":"2023-10-09T06:31:30.900235+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:31:30.900235+0000","last_undegraded":"2023-10-09T06:31:30.900235+0000","last_fullsized":"2023-10-09T06:31:30.900235+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":671,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906323+0000","last_change":"2023-10-09T06:29:15.536561+0000","last_active":"2023-10-09T06:31:30.906323+0000","last_peered":"2023-10-09T06:31:30.906323+0000","last_clean":"2023-10-09T06:31:30.906323+0000","last_became_active":"2023-10-09T06:29:15.535869+0000","last_became_peered":"2023-10-09T06:29:15.535869+0000","last_unstale":"2023-10-09T06:31:30.906323+0000","last_undegraded":"2023-10-09T06:31:30.906323+0000","last_fullsized":"2023-10-09T06:31:30.906323+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:04.229310+0000","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":76,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.899004+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:31:31.899004+0000","last_peered":"2023-10-09T06:31:31.899004+0000","last_clean":"2023-10-09T06:31:31.899004+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:31:31.899004+0000","last_undegraded":"2023-10-09T06:31:31.899004+0000","last_fullsized":"2023-10-09T06:31:31.899004+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108895+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:31:32.108895+0000","last_peered":"2023-10-09T06:31:32.108895+0000","last_clean":"2023-10-09T06:31:32.108895+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:31:32.108895+0000","last_undegraded":"2023-10-09T06:31:32.108895+0000","last_fullsized":"2023-10-09T06:31:32.108895+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"369'3011","reported_seq":3028,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:40.668889+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:31:40.668889+0000","last_peered":"2023-10-09T06:31:40.668889+0000","last_clean":"2023-10-09T06:31:40.668889+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:31:40.668889+0000","last_undegraded":"2023-10-09T06:31:40.668889+0000","last_fullsized":"2023-10-09T06:31:40.668889+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3011,"log_dups_size":0,"ondisk_log_size":3011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197328896,"num_objects":3011,"num_object_clones":0,"num_object_copies":9033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3011,"num_write_kb":192704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":653,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900005+0000","last_change":"2023-10-09T06:21:30.014606+0000","last_active":"2023-10-09T06:31:30.900005+0000","last_peered":"2023-10-09T06:31:30.900005+0000","last_clean":"2023-10-09T06:31:30.900005+0000","last_became_active":"2023-10-09T06:21:17.851379+0000","last_became_peered":"2023-10-09T06:21:17.851379+0000","last_unstale":"2023-10-09T06:31:30.900005+0000","last_undegraded":"2023-10-09T06:31:30.900005+0000","last_fullsized":"2023-10-09T06:31:30.900005+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:15.794756+0000","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":231,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.906138+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:31:30.906138+0000","last_peered":"2023-10-09T06:31:30.906138+0000","last_clean":"2023-10-09T06:31:30.906138+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:31:30.906138+0000","last_undegraded":"2023-10-09T06:31:30.906138+0000","last_fullsized":"2023-10-09T06:31:30.906138+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":41,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900507+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:31:31.900507+0000","last_peered":"2023-10-09T06:31:31.900507+0000","last_clean":"2023-10-09T06:31:31.900507+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:31:31.900507+0000","last_undegraded":"2023-10-09T06:31:31.900507+0000","last_fullsized":"2023-10-09T06:31:31.900507+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":351,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900992+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:31:31.900992+0000","last_peered":"2023-10-09T06:31:31.900992+0000","last_clean":"2023-10-09T06:31:31.900992+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:31:31.900992+0000","last_undegraded":"2023-10-09T06:31:31.900992+0000","last_fullsized":"2023-10-09T06:31:31.900992+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"369'2840","reported_seq":2856,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:39.471280+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:31:39.471280+0000","last_peered":"2023-10-09T06:31:39.471280+0000","last_clean":"2023-10-09T06:31:39.471280+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:31:39.471280+0000","last_undegraded":"2023-10-09T06:31:39.471280+0000","last_fullsized":"2023-10-09T06:31:39.471280+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2840,"log_dups_size":0,"ondisk_log_size":2840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2840,"num_write_kb":181760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":661,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.109136+0000","last_change":"2023-10-09T06:29:15.549866+0000","last_active":"2023-10-09T06:31:32.109136+0000","last_peered":"2023-10-09T06:31:32.109136+0000","last_clean":"2023-10-09T06:31:32.109136+0000","last_became_active":"2023-10-09T06:29:15.549495+0000","last_became_peered":"2023-10-09T06:29:15.549495+0000","last_unstale":"2023-10-09T06:31:32.109136+0000","last_undegraded":"2023-10-09T06:31:32.109136+0000","last_fullsized":"2023-10-09T06:31:32.109136+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:46.691754+0000","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.907110+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:31:30.907110+0000","last_peered":"2023-10-09T06:31:30.907110+0000","last_clean":"2023-10-09T06:31:30.907110+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:31:30.907110+0000","last_undegraded":"2023-10-09T06:31:30.907110+0000","last_fullsized":"2023-10-09T06:31:30.907110+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":361,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899161+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:31:30.899161+0000","last_peered":"2023-10-09T06:31:30.899161+0000","last_clean":"2023-10-09T06:31:30.899161+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:31:30.899161+0000","last_undegraded":"2023-10-09T06:31:30.899161+0000","last_fullsized":"2023-10-09T06:31:30.899161+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":350,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.901019+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:31:31.901019+0000","last_peered":"2023-10-09T06:31:31.901019+0000","last_clean":"2023-10-09T06:31:31.901019+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:31:31.901019+0000","last_undegraded":"2023-10-09T06:31:31.901019+0000","last_fullsized":"2023-10-09T06:31:31.901019+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"369'2824","reported_seq":2841,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:39.484996+0000","last_change":"2023-10-09T06:31:03.217038+0000","last_active":"2023-10-09T06:31:39.484996+0000","last_peered":"2023-10-09T06:31:39.484996+0000","last_clean":"2023-10-09T06:31:39.484996+0000","last_became_active":"2023-10-09T06:31:03.216747+0000","last_became_peered":"2023-10-09T06:31:03.216747+0000","last_unstale":"2023-10-09T06:31:39.484996+0000","last_undegraded":"2023-10-09T06:31:39.484996+0000","last_fullsized":"2023-10-09T06:31:39.484996+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2824,"log_dups_size":0,"ondisk_log_size":2824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:22.026356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185073664,"num_objects":2824,"num_object_clones":0,"num_object_copies":8472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2824,"num_write_kb":180736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":651,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908915+0000","last_change":"2023-10-09T06:21:39.977997+0000","last_active":"2023-10-09T06:31:30.908915+0000","last_peered":"2023-10-09T06:31:30.908915+0000","last_clean":"2023-10-09T06:31:30.908915+0000","last_became_active":"2023-10-09T06:21:17.841754+0000","last_became_peered":"2023-10-09T06:21:17.841754+0000","last_unstale":"2023-10-09T06:31:30.908915+0000","last_undegraded":"2023-10-09T06:31:30.908915+0000","last_fullsized":"2023-10-09T06:31:30.908915+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:26.521119+0000","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":97,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.899235+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:31:31.899235+0000","last_peered":"2023-10-09T06:31:31.899235+0000","last_clean":"2023-10-09T06:31:31.899235+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:31:31.899235+0000","last_undegraded":"2023-10-09T06:31:31.899235+0000","last_fullsized":"2023-10-09T06:31:31.899235+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":329,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900187+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:31:30.900187+0000","last_peered":"2023-10-09T06:31:30.900187+0000","last_clean":"2023-10-09T06:31:30.900187+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:31:30.900187+0000","last_undegraded":"2023-10-09T06:31:30.900187+0000","last_fullsized":"2023-10-09T06:31:30.900187+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:39.112957+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:31:39.112957+0000","last_peered":"2023-10-09T06:31:39.112957+0000","last_clean":"2023-10-09T06:31:39.112957+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:31:39.112957+0000","last_undegraded":"2023-10-09T06:31:39.112957+0000","last_fullsized":"2023-10-09T06:31:39.112957+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"369'2975","reported_seq":2991,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:41.171850+0000","last_change":"2023-10-09T06:31:03.215173+0000","last_active":"2023-10-09T06:31:41.171850+0000","last_peered":"2023-10-09T06:31:41.171850+0000","last_clean":"2023-10-09T06:31:41.171850+0000","last_became_active":"2023-10-09T06:31:03.214809+0000","last_became_peered":"2023-10-09T06:31:03.214809+0000","last_unstale":"2023-10-09T06:31:41.171850+0000","last_undegraded":"2023-10-09T06:31:41.171850+0000","last_fullsized":"2023-10-09T06:31:41.171850+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":2975,"log_dups_size":0,"ondisk_log_size":2975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:12.323433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194969600,"num_objects":2975,"num_object_clones":0,"num_object_copies":8925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2975,"num_write_kb":190400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":698,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900083+0000","last_change":"2023-10-09T06:28:14.734086+0000","last_active":"2023-10-09T06:31:31.900083+0000","last_peered":"2023-10-09T06:31:31.900083+0000","last_clean":"2023-10-09T06:31:31.900083+0000","last_became_active":"2023-10-09T06:28:14.733702+0000","last_became_peered":"2023-10-09T06:28:14.733702+0000","last_unstale":"2023-10-09T06:31:31.900083+0000","last_undegraded":"2023-10-09T06:31:31.900083+0000","last_fullsized":"2023-10-09T06:31:31.900083+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:01.831911+0000","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":353,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:32.108923+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:31:32.108923+0000","last_peered":"2023-10-09T06:31:32.108923+0000","last_clean":"2023-10-09T06:31:32.108923+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:31:32.108923+0000","last_undegraded":"2023-10-09T06:31:32.108923+0000","last_fullsized":"2023-10-09T06:31:32.108923+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":332,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900842+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:31:31.900842+0000","last_peered":"2023-10-09T06:31:31.900842+0000","last_clean":"2023-10-09T06:31:31.900842+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:31:31.900842+0000","last_undegraded":"2023-10-09T06:31:31.900842+0000","last_fullsized":"2023-10-09T06:31:31.900842+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":331,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.909323+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:31:30.909323+0000","last_peered":"2023-10-09T06:31:30.909323+0000","last_clean":"2023-10-09T06:31:30.909323+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:31:30.909323+0000","last_undegraded":"2023-10-09T06:31:30.909323+0000","last_fullsized":"2023-10-09T06:31:30.909323+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":334,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900977+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:31:31.900977+0000","last_peered":"2023-10-09T06:31:31.900977+0000","last_clean":"2023-10-09T06:31:31.900977+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:31:31.900977+0000","last_undegraded":"2023-10-09T06:31:31.900977+0000","last_fullsized":"2023-10-09T06:31:31.900977+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":355,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.899072+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:31:30.899072+0000","last_peered":"2023-10-09T06:31:30.899072+0000","last_clean":"2023-10-09T06:31:30.899072+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:31:30.899072+0000","last_undegraded":"2023-10-09T06:31:30.899072+0000","last_fullsized":"2023-10-09T06:31:30.899072+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":59,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900863+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:31:30.900863+0000","last_peered":"2023-10-09T06:31:30.900863+0000","last_clean":"2023-10-09T06:31:30.900863+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:31:30.900863+0000","last_undegraded":"2023-10-09T06:31:30.900863+0000","last_fullsized":"2023-10-09T06:31:30.900863+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":375,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908263+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:31:30.908263+0000","last_peered":"2023-10-09T06:31:30.908263+0000","last_clean":"2023-10-09T06:31:30.908263+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:31:30.908263+0000","last_undegraded":"2023-10-09T06:31:30.908263+0000","last_fullsized":"2023-10-09T06:31:30.908263+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":59,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.900447+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:31:30.900447+0000","last_peered":"2023-10-09T06:31:30.900447+0000","last_clean":"2023-10-09T06:31:30.900447+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:31:30.900447+0000","last_undegraded":"2023-10-09T06:31:30.900447+0000","last_fullsized":"2023-10-09T06:31:30.900447+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":385,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908583+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:31:30.908583+0000","last_peered":"2023-10-09T06:31:30.908583+0000","last_clean":"2023-10-09T06:31:30.908583+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:31:30.908583+0000","last_undegraded":"2023-10-09T06:31:30.908583+0000","last_fullsized":"2023-10-09T06:31:30.908583+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":68,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T06:31:39.111910+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:31:39.111910+0000","last_peered":"2023-10-09T06:31:39.111910+0000","last_clean":"2023-10-09T06:31:39.111910+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:31:39.111910+0000","last_undegraded":"2023-10-09T06:31:39.111910+0000","last_fullsized":"2023-10-09T06:31:39.111910+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":85,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.898746+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:31:31.898746+0000","last_peered":"2023-10-09T06:31:31.898746+0000","last_clean":"2023-10-09T06:31:31.898746+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:31:31.898746+0000","last_undegraded":"2023-10-09T06:31:31.898746+0000","last_fullsized":"2023-10-09T06:31:31.898746+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":363,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T06:31:31.900825+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:31:31.900825+0000","last_peered":"2023-10-09T06:31:31.900825+0000","last_clean":"2023-10-09T06:31:31.900825+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:31:31.900825+0000","last_undegraded":"2023-10-09T06:31:31.900825+0000","last_fullsized":"2023-10-09T06:31:31.900825+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":328,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T06:31:30.908305+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:31:30.908305+0000","last_peered":"2023-10-09T06:31:30.908305+0000","last_clean":"2023-10-09T06:31:30.908305+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:31:30.908305+0000","last_undegraded":"2023-10-09T06:31:30.908305+0000","last_fullsized":"2023-10-09T06:31:30.908305+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3040608256,"num_objects":46396,"num_object_clones":0,"num_object_copies":139188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46396,"num_write_kb":2969344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9083813888,"data_stored":9083813888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46396,"ondisk_log_size":46396,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234412,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2133872,"kb_used_data":1237736,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":91611280,"statfs":{"total":95995035648,"available":93809950720,"internally_reserved":0,"allocated":1267441664,"data_stored":1266481186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":4.9039999999999999,"15min":4.9569999999999999},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":4.2549999999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":1.5549999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.9370000000000003,"15min":5.0670000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":3.2930000000000001,"5min":215.577,"15min":219.399},"last":0.22500000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":5.9080000000000004,"5min":6.9850000000000003,"15min":4.4669999999999996},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":107.87,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":6.0869999999999997,"5min":6.1299999999999999,"15min":4.0709999999999997},"min":{"1min":0.38600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.904,"5min":216.22200000000001,"15min":216.22200000000001},"last":2.5960000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":10.584,"5min":6.4859999999999998,"15min":6.665},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":198.94200000000001,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":10.708,"5min":6.5149999999999997,"15min":6.8470000000000004},"min":{"1min":0.30299999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":198.92099999999999,"5min":216.25200000000001,"15min":216.25200000000001},"last":1.577}]},{"osd":3,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":7.4969999999999999,"15min":3.931},"min":{"1min":0.37,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":3.4460000000000002,"5min":215.922,"15min":215.922},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0269999999999999,"5min":7.6669999999999998,"15min":4.0170000000000003},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":3.3210000000000002,"5min":215.94900000000001,"15min":215.94900000000001},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":4.0579999999999998,"15min":2.5419999999999998},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":4.4589999999999996,"5min":218.446,"15min":218.446},"last":1.5189999999999999},{"interface":"front","average":{"1min":0.755,"5min":3.9969999999999999,"15min":2.395},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.23699999999999999},"max":{"1min":3.2210000000000001,"5min":216.172,"15min":216.172},"last":0.32600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":4.2270000000000003,"15min":2.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":3.7349999999999999,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.436},{"interface":"front","average":{"1min":1.1839999999999999,"5min":4.1559999999999997,"15min":2.4369999999999998},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.5670000000000002,"5min":218.15899999999999,"15min":218.15899999999999},"last":1.615}]},{"osd":6,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":4.2880000000000003,"15min":2.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":4.548,"5min":218.19999999999999,"15min":218.19999999999999},"last":1.597},{"interface":"front","average":{"1min":0.80100000000000005,"5min":3.944,"15min":2.149},"min":{"1min":0.40799999999999997,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":3.2650000000000001,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.41599999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496048,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2235300,"kb_used_data":1332188,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91509852,"statfs":{"total":95995035648,"available":93706088448,"internally_reserved":0,"allocated":1364160512,"data_stored":1363219719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.601,"15min":3.0369999999999999},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.7370000000000001,"5min":22.321999999999999,"15min":110.926},"last":3.0899999999999999},{"interface":"front","average":{"1min":1.258,"5min":2.1259999999999999,"15min":3.3610000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.4340000000000002,"5min":22.338000000000001,"15min":110.911},"last":3.7440000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3760000000000003,"5min":6.9649999999999999,"15min":5.0010000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":82.159000000000006,"5min":292.92200000000003,"15min":292.92200000000003},"last":2.375},{"interface":"front","average":{"1min":7.1950000000000003,"5min":7.0590000000000002,"15min":4.8810000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":82.174000000000007,"5min":292.904,"15min":292.904},"last":0.496}]},{"osd":2,"last update":"Mon Oct 9 06:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":2.109,"15min":2.6459999999999999},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.3670000000000009,"5min":20.331,"15min":75.191999999999993},"last":2.9009999999999998},{"interface":"front","average":{"1min":2.335,"5min":1.748,"15min":2.351},"min":{"1min":0.33400000000000002,"5min":0.32000000000000001,"15min":0.27300000000000002},"max":{"1min":8.2400000000000002,"5min":8.6069999999999993,"15min":75.120000000000005},"last":3.3479999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":4.1180000000000003,"15min":3.2120000000000002},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":88.763000000000005,"5min":91.174000000000007,"15min":110.358},"last":3.1640000000000001},{"interface":"front","average":{"1min":6.0300000000000002,"5min":4.2080000000000002,"15min":3.2850000000000001},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.311},"max":{"1min":86.831999999999994,"5min":91.438999999999993,"15min":110.43899999999999},"last":3.9990000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":1.361,"15min":1.3220000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":12.032,"5min":12.666,"15min":17.643999999999998},"last":2.0790000000000002},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.97499999999999998,"15min":1.1739999999999999},"min":{"1min":0.28699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":5.1280000000000001,"5min":8.6059999999999999,"15min":10.563000000000001},"last":2.2869999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:30:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.3169999999999999,"15min":1.429},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28499999999999998},"max":{"1min":3.9830000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":2.048},{"interface":"front","average":{"1min":1.448,"5min":1.379,"15min":1.369},"min":{"1min":0.33300000000000002,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":8.8000000000000007,"5min":16.536999999999999,"15min":24.401},"last":2.2490000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":4.1630000000000003,"15min":2.569},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":19.295999999999999,"5min":122.937,"15min":122.937},"last":3.1309999999999998},{"interface":"front","average":{"1min":1.647,"5min":4.444,"15min":2.6850000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.584,"5min":122.92100000000001,"15min":122.92100000000001},"last":2.3159999999999998}]}]},{"osd":1,"up_from":110,"seq":472446402785,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776512,"kb_used_data":1104824,"kb_used_omap":4,"kb_used_meta":671675,"kb_avail":91968640,"statfs":{"total":95995035648,"available":94175887360,"internally_reserved":0,"allocated":1131339776,"data_stored":1129663543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":687796110},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":10.044,"5min":8.0340000000000007,"15min":5.7519999999999998},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.189},"max":{"1min":143.465,"5min":269.79199999999997,"15min":269.79199999999997},"last":1.1679999999999999},{"interface":"front","average":{"1min":10.407,"5min":8.1820000000000004,"15min":6.1429999999999998},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.17899999999999999},"max":{"1min":143.66399999999999,"5min":271.20999999999998,"15min":271.20999999999998},"last":0.38200000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.2789999999999999,"15min":2.9540000000000002},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":6.9390000000000001,"5min":7.1449999999999996,"15min":139.75399999999999},"last":0.318},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3759999999999999,"15min":3.6970000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":6.673,"5min":12.077,"15min":140.29900000000001},"last":0.40400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:31:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.5409999999999999,"15min":3.3050000000000002},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":4.452,"5min":31.062000000000001,"15min":153.18199999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.437,"5min":1.952,"15min":2.4460000000000002},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.95999999999999996,"5min":27.657,"15min":152.52000000000001},"last":0.23000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.7569999999999999,"15min":3.3610000000000002},"min":{"1min":0.317,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":2.5409999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.655,"15min":3.1699999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.605,"5min":11.974,"15min":199.32900000000001},"last":0.45800000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":1.8779999999999999,"15min":2.0049999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.9289999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.1779999999999999,"15min":2.4670000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":2.5779999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.42399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":2.5680000000000001,"15min":2.6930000000000001},"min":{"1min":0.35599999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.4980000000000002,"5min":52.488,"15min":77.292000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.8109999999999999,"15min":2.758},"min":{"1min":0.38700000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.8280000000000001,"5min":52.558,"15min":77.378},"last":0.63500000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.681,"15min":2.9660000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":0.79500000000000004,"5min":55.231000000000002,"15min":77.456999999999994},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":3.0539999999999998,"15min":2.4289999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":0.81499999999999995,"5min":55.253999999999998,"15min":77.359999999999999},"last":0.34599999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024219,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2186292,"kb_used_data":1372012,"kb_used_omap":5,"kb_used_meta":814266,"kb_avail":91558860,"statfs":{"total":95995035648,"available":93756272640,"internally_reserved":0,"allocated":1404940288,"data_stored":1402992941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":833809018},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:31:29 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":3.2789999999999999,"15min":3.3290000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":3.2839999999999998,"5min":191.446,"15min":191.446},"last":1.698},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.3929999999999998,"15min":3.5579999999999998},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":4.0609999999999999,"5min":191.393,"15min":191.393},"last":1.6299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.7309999999999999,"15min":2.3180000000000001},"min":{"1min":0.20399999999999999,"5min":0.19900000000000001,"15min":0.17799999999999999},"max":{"1min":16.936,"5min":57.162999999999997,"15min":57.162999999999997},"last":2.8919999999999999},{"interface":"front","average":{"1min":3.6419999999999999,"5min":3.524,"15min":2.6960000000000002},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":15.794,"5min":57.585000000000001,"15min":57.585000000000001},"last":2.2869999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:07 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":7.8419999999999996,"15min":5.4000000000000004},"min":{"1min":0.26800000000000002,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":187.172,"5min":208.11199999999999,"15min":208.11199999999999},"last":2.0539999999999998},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.4950000000000001,"15min":5.391},"min":{"1min":0.31900000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":194.851,"5min":208.25899999999999,"15min":208.25899999999999},"last":1.831}]},{"osd":4,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.518,"15min":1.429},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":7.21,"5min":9.7159999999999993,"15min":11.162000000000001},"last":1.74},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.7470000000000001,"15min":1.798},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":10.903,"5min":18.081,"15min":18.081},"last":1.8049999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.7130000000000001,"15min":1.806},"min":{"1min":0.317,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":7.2409999999999997,"5min":10.865,"15min":62.984999999999999},"last":1.7709999999999999},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.9970000000000001,"15min":2.2469999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":10.603,"5min":15.747999999999999,"15min":64.983999999999995},"last":2.0910000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":4.2619999999999996,"15min":2.8980000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":12.522,"5min":120.572,"15min":120.572},"last":2.194},{"interface":"front","average":{"1min":1.758,"5min":3.827,"15min":2.3330000000000002},"min":{"1min":0.371,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":11.093,"5min":120.64400000000001,"15min":120.64400000000001},"last":5.1790000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":3.1840000000000002,"15min":2.4969999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":8.8059999999999992,"5min":122.018,"15min":122.018},"last":1.9870000000000001},{"interface":"front","average":{"1min":2.415,"5min":3.6539999999999999,"15min":2.8730000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":10.563000000000001,"5min":122.32299999999999,"15min":122.32299999999999},"last":2.1309999999999998}]}]},{"osd":2,"up_from":128,"seq":549755814088,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2417304,"kb_used_data":1864080,"kb_used_omap":7,"kb_used_meta":553208,"kb_avail":91327848,"statfs":{"total":95995035648,"available":93519716352,"internally_reserved":0,"allocated":1908817920,"data_stored":1907183569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":566485073},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":4.2409999999999997,"15min":4.7370000000000001},"min":{"1min":0.19500000000000001,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":16.91,"5min":162.21100000000001,"15min":163.40100000000001},"last":4.133},{"interface":"front","average":{"1min":3.4180000000000001,"5min":4.3630000000000004,"15min":4.4260000000000002},"min":{"1min":0.30099999999999999,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":16.888999999999999,"5min":168.49700000000001,"15min":168.49700000000001},"last":2.827}]},{"osd":1,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.762,"5min":3.3719999999999999,"15min":4.1740000000000004},"min":{"1min":0.221,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":24.312000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.714,"5min":4.423,"15min":4.5380000000000003},"min":{"1min":0.29399999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":20.259,"5min":91.721999999999994,"15min":222.19399999999999},"last":2.7629999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.3370000000000002,"15min":2.8090000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8499999999999996,"5min":91.644000000000005,"15min":91.644000000000005},"last":2.8540000000000001},{"interface":"front","average":{"1min":1.841,"5min":3.9809999999999999,"15min":3.056},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.9059999999999997,"5min":91.685000000000002,"15min":91.685000000000002},"last":4.1619999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":1.7589999999999999,"15min":2.081},"min":{"1min":0.33300000000000002,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":13.023999999999999,"5min":17.776,"15min":90.938999999999993},"last":3.0150000000000001},{"interface":"front","average":{"1min":3.0880000000000001,"5min":2.1560000000000001,"15min":2.375},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.433999999999999,"5min":12.433999999999999,"15min":90.950000000000003},"last":4.0679999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.0339999999999998,"15min":1.605},"min":{"1min":0.315,"5min":0.315,"15min":0.309},"max":{"1min":13.102,"5min":17.800000000000001,"15min":17.800000000000001},"last":2.8879999999999999},{"interface":"front","average":{"1min":3.2930000000000001,"5min":2.2709999999999999,"15min":1.978},"min":{"1min":0.377,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.44400000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":3.585,"5min":2.3650000000000002,"15min":2.5529999999999999},"min":{"1min":0.41399999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":12.387,"5min":12.387,"15min":138.291},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.639,"5min":1.6459999999999999,"15min":2.1829999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":9.1579999999999995,"5min":12.643000000000001,"15min":138.01599999999999},"last":4.1779999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.052,"15min":1.968},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":9.1859999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":4.0339999999999998},{"interface":"front","average":{"1min":3.04,"5min":2.113,"15min":2.0110000000000001},"min":{"1min":0.42099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.167,"5min":17.734999999999999,"15min":55.731000000000002},"last":5.8220000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127285,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2246732,"kb_used_data":1722564,"kb_used_omap":2,"kb_used_meta":524157,"kb_avail":91498420,"statfs":{"total":95995035648,"available":93694382080,"internally_reserved":0,"allocated":1763905536,"data_stored":1762334735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":536736821},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.806,"5min":3.3180000000000001,"15min":4.1609999999999996},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":47.847000000000001,"5min":47.847000000000001,"15min":187.852},"last":6.7759999999999998},{"interface":"front","average":{"1min":4.0030000000000001,"5min":3.0419999999999998,"15min":4.4180000000000001},"min":{"1min":0.28199999999999997,"5min":0.20799999999999999,"15min":0.184},"max":{"1min":47.896999999999998,"5min":47.896999999999998,"15min":187.923},"last":1.6579999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0279999999999996,"5min":2.734,"15min":3.5449999999999999},"min":{"1min":0.29599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":47.960999999999999,"5min":47.960999999999999,"15min":187.89099999999999},"last":1.45},{"interface":"front","average":{"1min":5.5490000000000004,"5min":4.0780000000000003,"15min":3.0939999999999999},"min":{"1min":0.26100000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.087000000000003,"5min":48.087000000000003,"15min":67.028000000000006},"last":3.7719999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.8809999999999998,"15min":3.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":48.055,"5min":48.055,"15min":187.76400000000001},"last":1.6879999999999999},{"interface":"front","average":{"1min":5.25,"5min":3.375,"15min":3.1200000000000001},"min":{"1min":0.33100000000000002,"5min":0.27900000000000003,"15min":0.22500000000000001},"max":{"1min":48.128999999999998,"5min":48.128999999999998,"15min":187.79499999999999},"last":6.8600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":2.657,"15min":1.78},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":48.249000000000002,"5min":48.249000000000002,"15min":48.249000000000002},"last":3.6640000000000001},{"interface":"front","average":{"1min":4.4370000000000003,"5min":2.6930000000000001,"15min":2.4649999999999999},"min":{"1min":0.40899999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":48.031999999999996,"5min":48.031999999999996,"15min":187.67400000000001},"last":1.411}]},{"osd":5,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":2.4510000000000001,"15min":2.5169999999999999},"min":{"1min":0.40200000000000002,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":48.162999999999997,"5min":48.162999999999997,"15min":187.833},"last":3.625},{"interface":"front","average":{"1min":3.9119999999999999,"5min":2.0409999999999999,"15min":2.3079999999999998},"min":{"1min":0.46300000000000002,"5min":0.34599999999999997,"15min":0.32100000000000001},"max":{"1min":48.106000000000002,"5min":48.106000000000002,"15min":187.952},"last":4.1749999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":4.1029999999999998,"15min":3.125},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":48.270000000000003,"5min":130.39599999999999,"15min":187.78200000000001},"last":1.514},{"interface":"front","average":{"1min":3.8370000000000002,"5min":3.6099999999999999,"15min":2.7949999999999999},"min":{"1min":0.40300000000000002,"5min":0.34399999999999997,"15min":0.32900000000000001},"max":{"1min":48.155000000000001,"5min":130.67699999999999,"15min":187.81800000000001},"last":1.48}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.2069999999999999,"5min":4.4710000000000001,"15min":3.109},"min":{"1min":0.41899999999999998,"5min":0.38500000000000001,"15min":0.34100000000000003},"max":{"1min":48.292000000000002,"5min":121.613,"15min":192.20400000000001},"last":3.6989999999999998},{"interface":"front","average":{"1min":5.4050000000000002,"5min":4.7080000000000002,"15min":3.2570000000000001},"min":{"1min":0.44700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":48.200000000000003,"5min":121.559,"15min":187.86500000000001},"last":1.548}]}]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428452,"kb_used_data":529372,"kb_used_omap":0,"kb_used_meta":899072,"kb_avail":92316700,"statfs":{"total":95995035648,"available":94532300800,"internally_reserved":0,"allocated":542076928,"data_stored":541128381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920649728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.696,"15min":2.2210000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":18.625,"5min":37.485999999999997,"15min":76.944999999999993},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.962,"5min":1.77,"15min":2.4740000000000002},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.654,"5min":37.502000000000002,"15min":76.988},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":4.2999999999999998,"15min":2.891},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23100000000000001},"max":{"1min":0.67700000000000005,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":4.3200000000000003,"15min":3.0590000000000002},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":1.0660000000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":2.1280000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":2.0950000000000002,"5min":32.613,"15min":69.640000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.2,"5min":1.7629999999999999,"15min":2.4540000000000002},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.9320000000000004,"5min":34.188000000000002,"15min":69.676000000000002},"last":4.5179999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7669999999999999,"5min":7.024,"15min":4.4420000000000002},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":35.018000000000001,"5min":133.38900000000001,"15min":133.38900000000001},"last":4.0949999999999998},{"interface":"front","average":{"1min":3.1869999999999998,"5min":6.7530000000000001,"15min":4.173},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":35.036999999999999,"5min":132.57599999999999,"15min":132.57599999999999},"last":1.5669999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.93999999999999995,"15min":1.0249999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.7829999999999999,"5min":6.25,"15min":10.673},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.0760000000000001,"15min":1.0309999999999999},"min":{"1min":0.32200000000000001,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":4.9589999999999996,"5min":7.2249999999999996,"15min":12.552},"last":0.41399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.4660000000000002,"15min":1.3340000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.0629999999999997,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.8029999999999999,"15min":1.7989999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.7210000000000001,"5min":163.381,"15min":163.381},"last":0.34999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.05,"15min":1.476},"min":{"1min":0.32700000000000001,"5min":0.312,"15min":0.308},"max":{"1min":3.6989999999999998,"5min":6.2679999999999998,"15min":62.658000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.734,"5min":0.995,"15min":1.462},"min":{"1min":0.35299999999999998,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":62.573},"last":0.53000000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757686,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254172,"kb_used_data":364756,"kb_used_omap":0,"kb_used_meta":889408,"kb_avail":92490980,"statfs":{"total":95995035648,"available":94710763520,"internally_reserved":0,"allocated":373510144,"data_stored":372566041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910753792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.0990000000000002,"15min":3.4900000000000002},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":3.819,"5min":131.87200000000001,"15min":131.87200000000001},"last":294.89699999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.8769999999999998,"15min":3.4260000000000002},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":3.6709999999999998,"5min":131.80199999999999,"15min":131.80199999999999},"last":294.30399999999997}]},{"osd":1,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":9.0109999999999992,"5min":4.9610000000000003,"15min":2.843},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23400000000000001},"max":{"1min":87.795000000000002,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":10.266999999999999,"5min":5.2930000000000001,"15min":3.0350000000000001},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.25900000000000001},"max":{"1min":87.856999999999999,"5min":126.931,"15min":126.931},"last":0.41399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.3599999999999999,"5min":2.52,"15min":2.294},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":16.5,"5min":51.393000000000001,"15min":81.260000000000005},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.282,"5min":2.1789999999999998,"15min":2.1840000000000002},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5679999999999996,"5min":51.432000000000002,"15min":81.245000000000005},"last":0.45500000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:31:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5049999999999999,"5min":9.5310000000000006,"15min":5.4580000000000002},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":92.521000000000001,"5min":144.56399999999999,"15min":190.411},"last":0.39400000000000002},{"interface":"front","average":{"1min":5.4720000000000004,"5min":9.4909999999999997,"15min":5.3449999999999998},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":94.492000000000004,"5min":144.62799999999999,"15min":190.70699999999999},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.0209999999999999,"15min":1.236},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.6279999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.872,"15min":1.089},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0760000000000001,"5min":6.2610000000000001,"15min":32.82},"last":0.34999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3080000000000001,"15min":1.5509999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":3.6859999999999999,"5min":6.2080000000000002,"15min":38.902000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.395,"15min":1.5920000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.6340000000000003,"5min":8.3659999999999997,"15min":35.774000000000001},"last":0.36899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:31:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.363,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.226,"5min":10.461,"15min":30.658999999999999},"last":0.441},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.1970000000000001,"15min":1.448},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":3.1930000000000001,"5min":6.2320000000000002,"15min":30.683},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1155334144,"data_stored":1155334144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":891158528,"data_stored":891158528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1936064512,"data_stored":1936064512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1505492992,"data_stored":1505492992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":553320448,"data_stored":553320448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":374079488,"data_stored":374079488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1380122624,"data_stored":1380122624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1288241152,"data_stored":1288241152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:31:43.764 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:31:43.764 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-10-09T06:31:43.941 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:43.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:43.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 51244 51228 100.03 70.125 0.0217539 0.00999204 2023-10-09T06:31:44.053 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:44.082 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:44.109 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:44.140 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:44.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:44.226 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:44.246 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:44.264 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:44.307 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:44.317 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:44.351 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:44.382 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:44.422 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:44.432 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:44.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:44.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:44.460 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:44.498 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.502 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:31:44.546 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:44.596 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:44.630 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:31:44.681 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:44.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:44.789 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:44.822 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:44.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 52042 52026 98.5094 49.875 0.163498 0.0101174 2023-10-09T06:31:44.966 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:44 smithi107 ceph-mon[212750]: pgmap v1148: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:44.966 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2166213686' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:45.068 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:45.298 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:44 smithi150 ceph-mon[194178]: pgmap v1148: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:45.298 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2166213686' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:45.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:44 smithi107 ceph-mon[220921]: pgmap v1148: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:31:45.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2166213686' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:45.719 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:45.737 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:45.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:45.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:45.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:45.876 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:45.919 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:45.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:45.945 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 53019 53003 97.408 61.0625 0.00467021 0.010229 2023-10-09T06:31:46.022 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.088 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:46.417 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:46.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:46.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:46.565 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:46.739 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:46.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:46.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:46.907 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:46.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 54137 54121 96.6215 69.875 0.0122787 0.010343 2023-10-09T06:31:46.972 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:31:46.972 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54154 12.f 12:fbe0b36a:::benchmark_data_smithi107_479504_object54153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.958114+0000", 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.013291159, 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013326411, 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:46.973 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54154 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:46.974 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958114+0000", 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958114+0000", 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958114+0000", 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958151+0000", 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7441999999999998e-05 2023-10-09T06:31:46.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958168+0000", 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7065e-05 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958173+0000", 2023-10-09T06:31:46.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.48e-06 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958206+0000", 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2394000000000002e-05 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958340+0000", 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013439800000000001 2023-10-09T06:31:46.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958472+0000", 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013172100000000001 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963069+0000", 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045968780000000004 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.964792+0000", 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017234489999999999 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54156 12.5 12:a3c949a5:::benchmark_data_smithi107_479504_object54155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.958183+0000", 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.013222276, 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013357693, 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:46.981 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54156 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958183+0000", 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958183+0000", 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958182+0000", 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958239+0000", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6793000000000003e-05 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958247+0000", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.537e-06 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958251+0000", 2023-10-09T06:31:46.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3499999999999999e-06 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958338+0000", 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6640999999999999e-05 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958474+0000", 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013602800000000001 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958621+0000", 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000147105 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963054+0000", 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0044329240000000004 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:46.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.966455+0000", 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003401528 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54162 12.f 12:f67c9154:::benchmark_data_smithi107_479504_object54161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.958335+0000", 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.013070297999999999, 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013292597, 2023-10-09T06:31:46.987 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54162 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958335+0000", 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958335+0000", 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958334+0000", 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958366+0000", 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1659000000000003e-05 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958373+0000", 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2649999999999999e-06 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.958378+0000", 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.617e-06 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.960427+0000", 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002049014 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.960546+0000", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119165 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.960693+0000", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014730900000000001 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963075+0000", 2023-10-09T06:31:46.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023814000000000001 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.964824+0000", 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017491989999999999 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.993 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54167 12.4 12:2ed5a83b:::benchmark_data_smithi107_479504_object54166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.963943+0000", 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.0074619619999999999, 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0077709349999999996, 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54167 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:46.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963943+0000", 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963943+0000", 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:46.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.963943+0000", 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.964017+0000", 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4474999999999994e-05 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.964027+0000", 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8600000000000005e-06 2023-10-09T06:31:46.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.964031+0000", 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3749999999999996e-06 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965459+0000", 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001427619 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965584+0000", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000125621 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965736+0000", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000151469 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.967303+0000", 2023-10-09T06:31:46.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001567089 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970916+0000", 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003612625 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54168 12.f 12:f5e4ce7f:::benchmark_data_smithi107_479504_object54167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:46.999 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.965000+0000", 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.0064048849999999999, 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068109090000000004, 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:47.000 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54168 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965000+0000", 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965000+0000", 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:47.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:47.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965000+0000", 2023-10-09T06:31:47.002 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965074+0000", 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4464999999999999e-05 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965084+0000", 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7899999999999994e-06 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965088+0000", 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0099999999999997e-06 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965114+0000", 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6432999999999999e-05 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:47.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965211+0000", 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6296000000000001e-05 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.965320+0000", 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010896 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.967027+0000", 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001707358 2023-10-09T06:31:47.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.967505+0000", 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047822000000000001 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:54170 12.9 12:96e28158:::benchmark_data_smithi107_479504_object54169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T06:31:47.006 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:46.970756+0000", 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00064888399999999996, 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011394930000000001, 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 54170 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970756+0000", 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970756+0000", 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:31:47.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970755+0000", 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970811+0000", 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5881999999999998e-05 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970821+0000", 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0033e-05 2023-10-09T06:31:47.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970825+0000", 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9500000000000003e-06 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970864+0000", 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8268e-05 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.970963+0000", 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9402999999999997e-05 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:46.971051+0000", 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8573999999999996e-05 2023-10-09T06:31:47.011 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout: ], 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 6 2023-10-09T06:31:47.012 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:31:47.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[212750]: pgmap v1149: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:31:47.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:47.057 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:47.058 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[220921]: pgmap v1149: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:31:47.058 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:47.058 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:46 smithi150 ceph-mon[194178]: pgmap v1149: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T06:31:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:31:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:31:47.751 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:31:47.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:47.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:47.903 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:31:47 smithi150 conmon[174357]: logger=cleanup t=2023-10-09T06:31:47.589127568Z level=info msg="Completed cleanup jobs" duration=442.02447ms 2023-10-09T06:31:47.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 55213 55197 95.8055 67.25 0.0243863 0.0104324 2023-10-09T06:31:48.002 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:48.090 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:31:48.108 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:48.199 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:48.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:48.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:48.310 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:48.378 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:48 smithi150 ceph-mon[194178]: pgmap v1150: 99 pgs: 99 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:31:48.379 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:48.398 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:48.439 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:31:48.608 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:48.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:48.615 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:48 smithi107 ceph-mon[212750]: pgmap v1150: 99 pgs: 99 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:31:48.615 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:48.615 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:48 smithi107 ceph-mon[220921]: pgmap v1150: 99 pgs: 99 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T06:31:48.616 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:31:48.638 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:48.713 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:48.783 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:48.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 56297 56281 95.0472 67.75 0.00329282 0.0105036 2023-10-09T06:31:49.406 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 1 pg_num to 21 2023-10-09T06:31:49.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:49.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/90846468' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: osdmap e370: 8 total, 8 up, 8 in 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: failed to encode map e370 with expected crc 2023-10-09T06:31:49.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-10-09T06:31:49.946 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 57188 57172 94.0114 55.6875 0.00235237 0.0106299 2023-10-09T06:31:50.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:50.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:50.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:50.159 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:50.216 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:50.368 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:50.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:50.451 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:50.471 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:50.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[212750]: pgmap v1152: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:31:50.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[212750]: failed to encode map e371 with expected crc 2023-10-09T06:31:50.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T06:31:50.484 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[212750]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:31:50.484 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[220921]: pgmap v1152: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:31:50.485 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[220921]: failed to encode map e371 with expected crc 2023-10-09T06:31:50.485 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T06:31:50.485 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:50 smithi107 ceph-mon[220921]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:31:50.516 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:50 smithi150 ceph-mon[194178]: pgmap v1152: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T06:31:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:50 smithi150 ceph-mon[194178]: failed to encode map e371 with expected crc 2023-10-09T06:31:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1224719500' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-10-09T06:31:50.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:50 smithi150 ceph-mon[194178]: osdmap e371: 8 total, 8 up, 8 in 2023-10-09T06:31:50.698 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:50.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:50.773 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:31:50.773 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:31:50.773 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:31:50.773 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:31:50.773 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:31:50.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:50.835 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:50.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 58210 58194 93.235 63.875 0.0111849 0.0107201 2023-10-09T06:31:51.046 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:51.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:51.153 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:51.180 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:51.220 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:51.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:51.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:51.368 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:51.414 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:51.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:51.519 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:51.581 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:51.666 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:51.667 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:31:51.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:51.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:51.894 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:51.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:51.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:31:51.949005+0000 min lat: 0.00111887 max lat: 0.343589 avg lat: 0.0108403 2023-10-09T06:31:51.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:31:51.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 59038 59022 92.197 51.75 0.0206294 0.0108403 2023-10-09T06:31:52.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:52.065 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:52.143 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:52.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:31:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:31:52] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:31:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:52.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:52.195 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:52.248 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:31:52.256 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:31:52.301 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:52.358 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:31:52.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:52.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:31:52.465 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:52.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: pgmap v1154: 99 pgs: 99 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:31:52.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.547 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T06:31:52.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:31:52.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[212750]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: pgmap v1154: 99 pgs: 99 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.554 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:52 smithi107 ceph-mon[220921]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.559 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:52.613 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:31:52.651 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: pgmap v1154: 99 pgs: 99 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: osdmap e372: 8 total, 8 up, 8 in 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:52 smithi150 ceph-mon[194178]: failed to encode map e372 with expected crc 2023-10-09T06:31:52.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:31:52.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:52.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:52.728 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:31:52.774 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:31:52.812 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:52.824 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:31:52.869 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:31:52.889 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:52.912 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:31:52.934 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.935 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:31:52.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 60068 60052 91.5184 64.375 0.0153227 0.0109198 2023-10-09T06:31:53.252 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:53.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:53.356 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:53.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2089231298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2089231298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:31:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: osdmap e373: 8 total, 8 up, 8 in 2023-10-09T06:31:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: failed to encode map e373 with expected crc 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2089231298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:31:53.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:31:53.693 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:53.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:53.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:53.798 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:53.865 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:53.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 60776 60760 90.393 44.25 0.00774898 0.0110574 2023-10-09T06:31:54.043 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:54.117 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:54.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:54.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:54.291 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:54.368 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:54.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:54.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:54.471 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:54.471 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:54.473 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:31:54.554 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:54.770 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:54 smithi107 ceph-mon[212750]: pgmap v1157: 109 pgs: 1 peering, 7 unknown, 101 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:31:54.771 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:54 smithi107 ceph-mon[220921]: pgmap v1157: 109 pgs: 1 peering, 7 unknown, 101 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:31:54.781 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:54 smithi150 ceph-mon[194178]: pgmap v1157: 109 pgs: 1 peering, 7 unknown, 101 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2023-10-09T06:31:54.870 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:31:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:54.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:31:54.948 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 61882 61866 89.8984 69.125 0.0142405 0.0111181 2023-10-09T06:31:54.979 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:31:55.025 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:55.485 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:31:55.485 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:31:55.486 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:31:55.486 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:31:55.777 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:31:55.802 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:31:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:55.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:31:55.910 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:55.949 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 63011 62995 89.4587 70.5625 0.0088411 0.011173 2023-10-09T06:31:55.959 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:56.055 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:56.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:56.071 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:31:56.170 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:56.209 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:56.324 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:56.340 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:31:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:56.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:31:56.444 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:31:56.542 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:56.575 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:31:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:56 smithi150 ceph-mon[194178]: pgmap v1158: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:31:56.688 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:31:56.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:56.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:31:56.788 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:56 smithi107 ceph-mon[212750]: pgmap v1158: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:31:56.789 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:56 smithi107 ceph-mon[220921]: pgmap v1158: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-09T06:31:56.795 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:56.864 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:56.950 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 63912 63896 88.7202 56.3125 0.0254962 0.0112661 2023-10-09T06:31:56.969 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:56.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:57.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.158 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:57.333 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:57.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:57.442 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:31:57.468 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3119846402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.468 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.511 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:31:57.513 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:57.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3119846402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3119846402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:31:57.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 64681 64665 87.8319 48.0625 0.00735972 0.0113804 2023-10-09T06:31:58.232 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:31:58.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:58.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:31:58.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:31:58.369 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:58.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: pgmap v1159: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:31:58.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:58.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.641 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[220921]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: pgmap v1159: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.643 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:58 smithi107 ceph-mon[212750]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.840 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:31:58.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:58.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:31:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: pgmap v1159: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-10-09T06:31:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:31:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:31:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: osdmap e374: 8 total, 8 up, 8 in 2023-10-09T06:31:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:58 smithi150 ceph-mon[194178]: failed to encode map e374 with expected crc 2023-10-09T06:31:58.937 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:31:58.938 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:31:58.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 65617 65601 87.2079 58.5 0.0052293 0.0114618 2023-10-09T06:31:58.952 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:31:58.972 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:31:59.000 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:59.342 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:31:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:31:59.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:31:59.449 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:31:59.477 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:31:59.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:31:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:59.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:31:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:59.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:31:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:31:59.686 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:31:59.710 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:31:59.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 66396 66380 86.4055 48.6875 0.0128905 0.011568 2023-10-09T06:32:00.071 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:00.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:00.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:00.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:32:00.177 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:00.249 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:00.368 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:00.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:00.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:00.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:00.532 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: pgmap v1161: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:32:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:00 smithi150 ceph-mon[194178]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:00.664 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: pgmap v1161: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:32:00.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[212750]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: pgmap v1161: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: failed to encode map e375 with expected crc 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: osdmap e375: 8 total, 8 up, 8 in 2023-10-09T06:32:00.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:00 smithi107 ceph-mon[220921]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:00.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:32:00.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 67201 67185 85.669 50.3125 0.0104542 0.0116679 2023-10-09T06:32:01.337 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1162,"stamp":"2023-10-09T06:31:59.478665+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4182906784,"num_objects":63841,"num_object_clones":0,"num_object_copies":191523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63878,"num_write_kb":4084317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":600556,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67253,"ondisk_log_size":67253,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":326,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19056912,"kb_used_data":12859920,"kb_used_omap":20,"kb_used_meta":6196907,"kb_avail":730904304,"statfs":{"total":767960285184,"available":748446007296,"internally_reserved":0,"allocated":13168558080,"data_stored":13157900681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6345633422},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":419233792,"num_objects":6397,"num_object_clones":0,"num_object_copies":19191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6397,"num_write_kb":409408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.043901"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":353,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437580+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:31:52.437580+0000","last_peered":"2023-10-09T06:31:52.437580+0000","last_clean":"2023-10-09T06:31:52.437580+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:31:52.437580+0000","last_undegraded":"2023-10-09T06:31:52.437580+0000","last_fullsized":"2023-10-09T06:31:52.437580+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.442890+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:31:52.442890+0000","last_peered":"2023-10-09T06:31:52.442890+0000","last_clean":"2023-10-09T06:31:52.442890+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:31:52.442890+0000","last_undegraded":"2023-10-09T06:31:52.442890+0000","last_fullsized":"2023-10-09T06:31:52.442890+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":365,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.509950+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:31:51.509950+0000","last_peered":"2023-10-09T06:31:51.509950+0000","last_clean":"2023-10-09T06:31:51.509950+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:31:51.509950+0000","last_undegraded":"2023-10-09T06:31:51.509950+0000","last_fullsized":"2023-10-09T06:31:51.509950+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":369,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.410809+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:31:51.410809+0000","last_peered":"2023-10-09T06:31:51.410809+0000","last_clean":"2023-10-09T06:31:51.410809+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:31:51.410809+0000","last_undegraded":"2023-10-09T06:31:51.410809+0000","last_fullsized":"2023-10-09T06:31:51.410809+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.445968+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:31:52.445968+0000","last_peered":"2023-10-09T06:31:52.445968+0000","last_clean":"2023-10-09T06:31:52.445968+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:31:52.445968+0000","last_undegraded":"2023-10-09T06:31:52.445968+0000","last_fullsized":"2023-10-09T06:31:52.445968+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":362,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.414292+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:31:51.414292+0000","last_peered":"2023-10-09T06:31:51.414292+0000","last_clean":"2023-10-09T06:31:51.414292+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:31:51.414292+0000","last_undegraded":"2023-10-09T06:31:51.414292+0000","last_fullsized":"2023-10-09T06:31:51.414292+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":45,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409461+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:31:51.409461+0000","last_peered":"2023-10-09T06:31:51.409461+0000","last_clean":"2023-10-09T06:31:51.409461+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:31:51.409461+0000","last_undegraded":"2023-10-09T06:31:51.409461+0000","last_fullsized":"2023-10-09T06:31:51.409461+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.448779+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:31:52.448779+0000","last_peered":"2023-10-09T06:31:52.448779+0000","last_clean":"2023-10-09T06:31:52.448779+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:31:52.448779+0000","last_undegraded":"2023-10-09T06:31:52.448779+0000","last_fullsized":"2023-10-09T06:31:52.448779+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":712,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.956866+0000","last_change":"2023-10-09T06:31:52.457819+0000","last_active":"2023-10-09T06:31:52.956866+0000","last_peered":"2023-10-09T06:31:52.956866+0000","last_clean":"2023-10-09T06:31:52.956866+0000","last_became_active":"2023-10-09T06:31:52.457504+0000","last_became_peered":"2023-10-09T06:31:52.457504+0000","last_unstale":"2023-10-09T06:31:52.956866+0000","last_undegraded":"2023-10-09T06:31:52.956866+0000","last_fullsized":"2023-10-09T06:31:52.956866+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.440788+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:31:52.440788+0000","last_peered":"2023-10-09T06:31:52.440788+0000","last_clean":"2023-10-09T06:31:52.440788+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:31:52.440788+0000","last_undegraded":"2023-10-09T06:31:52.440788+0000","last_fullsized":"2023-10-09T06:31:52.440788+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.442770+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:31:52.442770+0000","last_peered":"2023-10-09T06:31:52.442770+0000","last_clean":"2023-10-09T06:31:52.442770+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:31:52.442770+0000","last_undegraded":"2023-10-09T06:31:52.442770+0000","last_fullsized":"2023-10-09T06:31:52.442770+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":255,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444394+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:31:52.444394+0000","last_peered":"2023-10-09T06:31:52.444394+0000","last_clean":"2023-10-09T06:31:52.444394+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:31:52.444394+0000","last_undegraded":"2023-10-09T06:31:52.444394+0000","last_fullsized":"2023-10-09T06:31:52.444394+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":307,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510221+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:31:51.510221+0000","last_peered":"2023-10-09T06:31:51.510221+0000","last_clean":"2023-10-09T06:31:51.510221+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:31:51.510221+0000","last_undegraded":"2023-10-09T06:31:51.510221+0000","last_fullsized":"2023-10-09T06:31:51.510221+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":380,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437644+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:31:52.437644+0000","last_peered":"2023-10-09T06:31:52.437644+0000","last_clean":"2023-10-09T06:31:52.437644+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:31:52.437644+0000","last_undegraded":"2023-10-09T06:31:52.437644+0000","last_fullsized":"2023-10-09T06:31:52.437644+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438372+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:31:52.438372+0000","last_peered":"2023-10-09T06:31:52.438372+0000","last_clean":"2023-10-09T06:31:52.438372+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:31:52.438372+0000","last_undegraded":"2023-10-09T06:31:52.438372+0000","last_fullsized":"2023-10-09T06:31:52.438372+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":709,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.454241+0000","last_change":"2023-10-09T06:31:52.452870+0000","last_active":"2023-10-09T06:31:52.454241+0000","last_peered":"2023-10-09T06:31:52.454241+0000","last_clean":"2023-10-09T06:31:52.454241+0000","last_became_active":"2023-10-09T06:31:52.452182+0000","last_became_peered":"2023-10-09T06:31:52.452182+0000","last_unstale":"2023-10-09T06:31:52.454241+0000","last_undegraded":"2023-10-09T06:31:52.454241+0000","last_fullsized":"2023-10-09T06:31:52.454241+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":223,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.443720+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:31:52.443720+0000","last_peered":"2023-10-09T06:31:52.443720+0000","last_clean":"2023-10-09T06:31:52.443720+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:31:52.443720+0000","last_undegraded":"2023-10-09T06:31:52.443720+0000","last_fullsized":"2023-10-09T06:31:52.443720+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.445184+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:31:52.445184+0000","last_peered":"2023-10-09T06:31:52.445184+0000","last_clean":"2023-10-09T06:31:52.445184+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:31:52.445184+0000","last_undegraded":"2023-10-09T06:31:52.445184+0000","last_fullsized":"2023-10-09T06:31:52.445184+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":686,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.463580+0000","last_change":"2023-10-09T06:31:52.463020+0000","last_active":"2023-10-09T06:31:52.463580+0000","last_peered":"2023-10-09T06:31:52.463580+0000","last_clean":"2023-10-09T06:31:52.463580+0000","last_became_active":"2023-10-09T06:31:52.462664+0000","last_became_peered":"2023-10-09T06:31:52.462664+0000","last_unstale":"2023-10-09T06:31:52.463580+0000","last_undegraded":"2023-10-09T06:31:52.463580+0000","last_fullsized":"2023-10-09T06:31:52.463580+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":368,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.410175+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:31:51.410175+0000","last_peered":"2023-10-09T06:31:51.410175+0000","last_clean":"2023-10-09T06:31:51.410175+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:31:51.410175+0000","last_undegraded":"2023-10-09T06:31:51.410175+0000","last_fullsized":"2023-10-09T06:31:51.410175+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441138+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:31:52.441138+0000","last_peered":"2023-10-09T06:31:52.441138+0000","last_clean":"2023-10-09T06:31:52.441138+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:31:52.441138+0000","last_undegraded":"2023-10-09T06:31:52.441138+0000","last_fullsized":"2023-10-09T06:31:52.441138+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":668,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.460987+0000","last_change":"2023-10-09T06:31:52.460987+0000","last_active":"2023-10-09T06:31:52.460987+0000","last_peered":"2023-10-09T06:31:52.460987+0000","last_clean":"2023-10-09T06:31:52.460987+0000","last_became_active":"2023-10-09T06:31:52.460041+0000","last_became_peered":"2023-10-09T06:31:52.460041+0000","last_unstale":"2023-10-09T06:31:52.460987+0000","last_undegraded":"2023-10-09T06:31:52.460987+0000","last_fullsized":"2023-10-09T06:31:52.460987+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.443993+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:31:52.443993+0000","last_peered":"2023-10-09T06:31:52.443993+0000","last_clean":"2023-10-09T06:31:52.443993+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:31:52.443993+0000","last_undegraded":"2023-10-09T06:31:52.443993+0000","last_fullsized":"2023-10-09T06:31:52.443993+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":45,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409879+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:31:51.409879+0000","last_peered":"2023-10-09T06:31:51.409879+0000","last_clean":"2023-10-09T06:31:51.409879+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:31:51.409879+0000","last_undegraded":"2023-10-09T06:31:51.409879+0000","last_fullsized":"2023-10-09T06:31:51.409879+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":705,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.457846+0000","last_change":"2023-10-09T06:31:52.456058+0000","last_active":"2023-10-09T06:31:52.457846+0000","last_peered":"2023-10-09T06:31:52.457846+0000","last_clean":"2023-10-09T06:31:52.457846+0000","last_became_active":"2023-10-09T06:31:52.455692+0000","last_became_peered":"2023-10-09T06:31:52.455692+0000","last_unstale":"2023-10-09T06:31:52.457846+0000","last_undegraded":"2023-10-09T06:31:52.457846+0000","last_fullsized":"2023-10-09T06:31:52.457846+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":451,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.413692+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:31:51.413692+0000","last_peered":"2023-10-09T06:31:51.413692+0000","last_clean":"2023-10-09T06:31:51.413692+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:31:51.413692+0000","last_undegraded":"2023-10-09T06:31:51.413692+0000","last_fullsized":"2023-10-09T06:31:51.413692+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444442+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:31:52.444442+0000","last_peered":"2023-10-09T06:31:52.444442+0000","last_clean":"2023-10-09T06:31:52.444442+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:31:52.444442+0000","last_undegraded":"2023-10-09T06:31:52.444442+0000","last_fullsized":"2023-10-09T06:31:52.444442+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":666,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.454587+0000","last_change":"2023-10-09T06:31:52.454228+0000","last_active":"2023-10-09T06:31:52.454587+0000","last_peered":"2023-10-09T06:31:52.454587+0000","last_clean":"2023-10-09T06:31:52.454587+0000","last_became_active":"2023-10-09T06:31:52.453965+0000","last_became_peered":"2023-10-09T06:31:52.453965+0000","last_unstale":"2023-10-09T06:31:52.454587+0000","last_undegraded":"2023-10-09T06:31:52.454587+0000","last_fullsized":"2023-10-09T06:31:52.454587+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.0","version":"374'4070","reported_seq":4091,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:58.222353+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:31:58.222353+0000","last_peered":"2023-10-09T06:31:58.222353+0000","last_clean":"2023-10-09T06:31:58.222353+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:31:58.222353+0000","last_undegraded":"2023-10-09T06:31:58.222353+0000","last_fullsized":"2023-10-09T06:31:58.222353+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4070,"log_dups_size":0,"ondisk_log_size":4070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266731520,"num_objects":4070,"num_object_clones":0,"num_object_copies":12210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4070,"num_write_kb":260480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":259,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.446175+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:31:52.446175+0000","last_peered":"2023-10-09T06:31:52.446175+0000","last_clean":"2023-10-09T06:31:52.446175+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:31:52.446175+0000","last_undegraded":"2023-10-09T06:31:52.446175+0000","last_fullsized":"2023-10-09T06:31:52.446175+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.440871+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:31:52.440871+0000","last_peered":"2023-10-09T06:31:52.440871+0000","last_clean":"2023-10-09T06:31:52.440871+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:31:52.440871+0000","last_undegraded":"2023-10-09T06:31:52.440871+0000","last_fullsized":"2023-10-09T06:31:52.440871+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":713,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.956890+0000","last_change":"2023-10-09T06:31:52.459390+0000","last_active":"2023-10-09T06:31:52.956890+0000","last_peered":"2023-10-09T06:31:52.956890+0000","last_clean":"2023-10-09T06:31:52.956890+0000","last_became_active":"2023-10-09T06:31:52.459008+0000","last_became_peered":"2023-10-09T06:31:52.459008+0000","last_unstale":"2023-10-09T06:31:52.956890+0000","last_undegraded":"2023-10-09T06:31:52.956890+0000","last_fullsized":"2023-10-09T06:31:52.956890+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":109227,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"373'3991","reported_seq":4011,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:56.308756+0000","last_change":"2023-10-09T06:31:02.763465+0000","last_active":"2023-10-09T06:31:56.308756+0000","last_peered":"2023-10-09T06:31:56.308756+0000","last_clean":"2023-10-09T06:31:56.308756+0000","last_became_active":"2023-10-09T06:31:02.763153+0000","last_became_peered":"2023-10-09T06:31:02.763153+0000","last_unstale":"2023-10-09T06:31:56.308756+0000","last_undegraded":"2023-10-09T06:31:56.308756+0000","last_fullsized":"2023-10-09T06:31:56.308756+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:01.874473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3991,"num_write_kb":255424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":221,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444724+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:31:52.444724+0000","last_peered":"2023-10-09T06:31:52.444724+0000","last_clean":"2023-10-09T06:31:52.444724+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:31:52.444724+0000","last_undegraded":"2023-10-09T06:31:52.444724+0000","last_fullsized":"2023-10-09T06:31:52.444724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437341+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:31:52.437341+0000","last_peered":"2023-10-09T06:31:52.437341+0000","last_clean":"2023-10-09T06:31:52.437341+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:31:52.437341+0000","last_undegraded":"2023-10-09T06:31:52.437341+0000","last_fullsized":"2023-10-09T06:31:52.437341+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":644,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.450666+0000","last_change":"2023-10-09T06:31:52.450666+0000","last_active":"2023-10-09T06:31:52.450666+0000","last_peered":"2023-10-09T06:31:52.450666+0000","last_clean":"2023-10-09T06:31:52.450666+0000","last_became_active":"2023-10-09T06:31:52.450370+0000","last_became_peered":"2023-10-09T06:31:52.450370+0000","last_unstale":"2023-10-09T06:31:52.450666+0000","last_undegraded":"2023-10-09T06:31:52.450666+0000","last_fullsized":"2023-10-09T06:31:52.450666+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"374'4123","reported_seq":4144,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:58.204145+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:31:58.204145+0000","last_peered":"2023-10-09T06:31:58.204145+0000","last_clean":"2023-10-09T06:31:58.204145+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:31:58.204145+0000","last_undegraded":"2023-10-09T06:31:58.204145+0000","last_fullsized":"2023-10-09T06:31:58.204145+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4123,"log_dups_size":0,"ondisk_log_size":4123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270204928,"num_objects":4123,"num_object_clones":0,"num_object_copies":12369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4123,"num_write_kb":263872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":277,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.445827+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:31:52.445827+0000","last_peered":"2023-10-09T06:31:52.445827+0000","last_clean":"2023-10-09T06:31:52.445827+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:31:52.445827+0000","last_undegraded":"2023-10-09T06:31:52.445827+0000","last_fullsized":"2023-10-09T06:31:52.445827+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":349,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.414035+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:31:51.414035+0000","last_peered":"2023-10-09T06:31:51.414035+0000","last_clean":"2023-10-09T06:31:51.414035+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:31:51.414035+0000","last_undegraded":"2023-10-09T06:31:51.414035+0000","last_fullsized":"2023-10-09T06:31:51.414035+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":674,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.463455+0000","last_change":"2023-10-09T06:31:52.463455+0000","last_active":"2023-10-09T06:31:52.463455+0000","last_peered":"2023-10-09T06:31:52.463455+0000","last_clean":"2023-10-09T06:31:52.463455+0000","last_became_active":"2023-10-09T06:31:52.463110+0000","last_became_peered":"2023-10-09T06:31:52.463110+0000","last_unstale":"2023-10-09T06:31:52.463455+0000","last_undegraded":"2023-10-09T06:31:52.463455+0000","last_fullsized":"2023-10-09T06:31:52.463455+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"373'3905","reported_seq":3926,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:55.666362+0000","last_change":"2023-10-09T06:31:02.761953+0000","last_active":"2023-10-09T06:31:55.666362+0000","last_peered":"2023-10-09T06:31:55.666362+0000","last_clean":"2023-10-09T06:31:55.666362+0000","last_became_active":"2023-10-09T06:31:02.761625+0000","last_became_peered":"2023-10-09T06:31:02.761625+0000","last_unstale":"2023-10-09T06:31:55.666362+0000","last_undegraded":"2023-10-09T06:31:55.666362+0000","last_fullsized":"2023-10-09T06:31:55.666362+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3905,"log_dups_size":0,"ondisk_log_size":3905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:31.403447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255918080,"num_objects":3905,"num_object_clones":0,"num_object_copies":11715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3905,"num_write_kb":249920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441555+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:31:52.441555+0000","last_peered":"2023-10-09T06:31:52.441555+0000","last_clean":"2023-10-09T06:31:52.441555+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:31:52.441555+0000","last_undegraded":"2023-10-09T06:31:52.441555+0000","last_fullsized":"2023-10-09T06:31:52.441555+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":332,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409462+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:31:51.409462+0000","last_peered":"2023-10-09T06:31:51.409462+0000","last_clean":"2023-10-09T06:31:51.409462+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:31:51.409462+0000","last_undegraded":"2023-10-09T06:31:51.409462+0000","last_fullsized":"2023-10-09T06:31:51.409462+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"373'3913","reported_seq":3933,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:56.294099+0000","last_change":"2023-10-09T06:31:02.762571+0000","last_active":"2023-10-09T06:31:56.294099+0000","last_peered":"2023-10-09T06:31:56.294099+0000","last_clean":"2023-10-09T06:31:56.294099+0000","last_became_active":"2023-10-09T06:31:02.762287+0000","last_became_peered":"2023-10-09T06:31:02.762287+0000","last_unstale":"2023-10-09T06:31:56.294099+0000","last_undegraded":"2023-10-09T06:31:56.294099+0000","last_fullsized":"2023-10-09T06:31:56.294099+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3913,"log_dups_size":0,"ondisk_log_size":3913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:15.365778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256442368,"num_objects":3913,"num_object_clones":0,"num_object_copies":11739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3913,"num_write_kb":250432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":706,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.443854+0000","last_change":"2023-10-09T06:27:39.662163+0000","last_active":"2023-10-09T06:31:52.443854+0000","last_peered":"2023-10-09T06:31:52.443854+0000","last_clean":"2023-10-09T06:31:52.443854+0000","last_became_active":"2023-10-09T06:27:39.661811+0000","last_became_peered":"2023-10-09T06:27:39.661811+0000","last_unstale":"2023-10-09T06:31:52.443854+0000","last_undegraded":"2023-10-09T06:31:52.443854+0000","last_fullsized":"2023-10-09T06:31:52.443854+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:50.382498+0000","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":98,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.442690+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:31:52.442690+0000","last_peered":"2023-10-09T06:31:52.442690+0000","last_clean":"2023-10-09T06:31:52.442690+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:31:52.442690+0000","last_undegraded":"2023-10-09T06:31:52.442690+0000","last_fullsized":"2023-10-09T06:31:52.442690+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":354,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437492+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:31:52.437492+0000","last_peered":"2023-10-09T06:31:52.437492+0000","last_clean":"2023-10-09T06:31:52.437492+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:31:52.437492+0000","last_undegraded":"2023-10-09T06:31:52.437492+0000","last_fullsized":"2023-10-09T06:31:52.437492+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"373'3935","reported_seq":3954,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:56.313030+0000","last_change":"2023-10-09T06:31:03.217042+0000","last_active":"2023-10-09T06:31:56.313030+0000","last_peered":"2023-10-09T06:31:56.313030+0000","last_clean":"2023-10-09T06:31:56.313030+0000","last_became_active":"2023-10-09T06:31:03.216729+0000","last_became_peered":"2023-10-09T06:31:03.216729+0000","last_unstale":"2023-10-09T06:31:56.313030+0000","last_undegraded":"2023-10-09T06:31:56.313030+0000","last_fullsized":"2023-10-09T06:31:56.313030+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3935,"log_dups_size":0,"ondisk_log_size":3935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.232295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3935,"num_write_kb":251840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":686,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.440866+0000","last_change":"2023-10-09T06:28:14.733366+0000","last_active":"2023-10-09T06:31:52.440866+0000","last_peered":"2023-10-09T06:31:52.440866+0000","last_clean":"2023-10-09T06:31:52.440866+0000","last_became_active":"2023-10-09T06:28:14.732959+0000","last_became_peered":"2023-10-09T06:28:14.732959+0000","last_unstale":"2023-10-09T06:31:52.440866+0000","last_undegraded":"2023-10-09T06:31:52.440866+0000","last_fullsized":"2023-10-09T06:31:52.440866+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:20.117980+0000","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":330,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.414195+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:31:51.414195+0000","last_peered":"2023-10-09T06:31:51.414195+0000","last_clean":"2023-10-09T06:31:51.414195+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:31:51.414195+0000","last_undegraded":"2023-10-09T06:31:51.414195+0000","last_fullsized":"2023-10-09T06:31:51.414195+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":351,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438477+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:31:52.438477+0000","last_peered":"2023-10-09T06:31:52.438477+0000","last_clean":"2023-10-09T06:31:52.438477+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:31:52.438477+0000","last_undegraded":"2023-10-09T06:31:52.438477+0000","last_fullsized":"2023-10-09T06:31:52.438477+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":668,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.461171+0000","last_change":"2023-10-09T06:31:52.461171+0000","last_active":"2023-10-09T06:31:52.461171+0000","last_peered":"2023-10-09T06:31:52.461171+0000","last_clean":"2023-10-09T06:31:52.461171+0000","last_became_active":"2023-10-09T06:31:52.460483+0000","last_became_peered":"2023-10-09T06:31:52.460483+0000","last_unstale":"2023-10-09T06:31:52.461171+0000","last_undegraded":"2023-10-09T06:31:52.461171+0000","last_fullsized":"2023-10-09T06:31:52.461171+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"373'4018","reported_seq":4039,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.408711+0000","last_change":"2023-10-09T06:31:02.762312+0000","last_active":"2023-10-09T06:31:57.408711+0000","last_peered":"2023-10-09T06:31:57.408711+0000","last_clean":"2023-10-09T06:31:57.408711+0000","last_became_active":"2023-10-09T06:31:02.762007+0000","last_became_peered":"2023-10-09T06:31:02.762007+0000","last_unstale":"2023-10-09T06:31:57.408711+0000","last_undegraded":"2023-10-09T06:31:57.408711+0000","last_fullsized":"2023-10-09T06:31:57.408711+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4018,"log_dups_size":0,"ondisk_log_size":4018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263323648,"num_objects":4018,"num_object_clones":0,"num_object_copies":12054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4018,"num_write_kb":257152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437221+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:31:52.437221+0000","last_peered":"2023-10-09T06:31:52.437221+0000","last_clean":"2023-10-09T06:31:52.437221+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:31:52.437221+0000","last_undegraded":"2023-10-09T06:31:52.437221+0000","last_fullsized":"2023-10-09T06:31:52.437221+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":334,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438666+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:31:52.438666+0000","last_peered":"2023-10-09T06:31:52.438666+0000","last_clean":"2023-10-09T06:31:52.438666+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:31:52.438666+0000","last_undegraded":"2023-10-09T06:31:52.438666+0000","last_fullsized":"2023-10-09T06:31:52.438666+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"373'3927","reported_seq":3947,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.403794+0000","last_change":"2023-10-09T06:31:03.215765+0000","last_active":"2023-10-09T06:31:57.403794+0000","last_peered":"2023-10-09T06:31:57.403794+0000","last_clean":"2023-10-09T06:31:57.403794+0000","last_became_active":"2023-10-09T06:31:03.215425+0000","last_became_peered":"2023-10-09T06:31:03.215425+0000","last_unstale":"2023-10-09T06:31:57.403794+0000","last_undegraded":"2023-10-09T06:31:57.403794+0000","last_fullsized":"2023-10-09T06:31:57.403794+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3927,"log_dups_size":0,"ondisk_log_size":3927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257359872,"num_objects":3927,"num_object_clones":0,"num_object_copies":11781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3927,"num_write_kb":251328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":664,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441555+0000","last_change":"2023-10-09T06:28:14.737320+0000","last_active":"2023-10-09T06:31:52.441555+0000","last_peered":"2023-10-09T06:31:52.441555+0000","last_clean":"2023-10-09T06:31:52.441555+0000","last_became_active":"2023-10-09T06:28:14.736920+0000","last_became_peered":"2023-10-09T06:28:14.736920+0000","last_unstale":"2023-10-09T06:31:52.441555+0000","last_undegraded":"2023-10-09T06:31:52.441555+0000","last_fullsized":"2023-10-09T06:31:52.441555+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:09.743234+0000","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.490095+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:31:57.490095+0000","last_peered":"2023-10-09T06:31:57.490095+0000","last_clean":"2023-10-09T06:31:57.490095+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:31:57.490095+0000","last_undegraded":"2023-10-09T06:31:57.490095+0000","last_fullsized":"2023-10-09T06:31:57.490095+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":325,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.512606+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:31:51.512606+0000","last_peered":"2023-10-09T06:31:51.512606+0000","last_clean":"2023-10-09T06:31:51.512606+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:31:51.512606+0000","last_undegraded":"2023-10-09T06:31:51.512606+0000","last_fullsized":"2023-10-09T06:31:51.512606+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"373'3904","reported_seq":3925,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:55.646670+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:31:55.646670+0000","last_peered":"2023-10-09T06:31:55.646670+0000","last_clean":"2023-10-09T06:31:55.646670+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:31:55.646670+0000","last_undegraded":"2023-10-09T06:31:55.646670+0000","last_fullsized":"2023-10-09T06:31:55.646670+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3904,"log_dups_size":0,"ondisk_log_size":3904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255852544,"num_objects":3904,"num_object_clones":0,"num_object_copies":11712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3904,"num_write_kb":249856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":645,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.446286+0000","last_change":"2023-10-09T06:31:52.446286+0000","last_active":"2023-10-09T06:31:52.446286+0000","last_peered":"2023-10-09T06:31:52.446286+0000","last_clean":"2023-10-09T06:31:52.446286+0000","last_became_active":"2023-10-09T06:31:52.446006+0000","last_became_peered":"2023-10-09T06:31:52.446006+0000","last_unstale":"2023-10-09T06:31:52.446286+0000","last_undegraded":"2023-10-09T06:31:52.446286+0000","last_fullsized":"2023-10-09T06:31:52.446286+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":377,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510228+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:31:51.510228+0000","last_peered":"2023-10-09T06:31:51.510228+0000","last_clean":"2023-10-09T06:31:51.510228+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:31:51.510228+0000","last_undegraded":"2023-10-09T06:31:51.510228+0000","last_fullsized":"2023-10-09T06:31:51.510228+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441062+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:31:52.441062+0000","last_peered":"2023-10-09T06:31:52.441062+0000","last_clean":"2023-10-09T06:31:52.441062+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:31:52.441062+0000","last_undegraded":"2023-10-09T06:31:52.441062+0000","last_fullsized":"2023-10-09T06:31:52.441062+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":311,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510079+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:31:51.510079+0000","last_peered":"2023-10-09T06:31:51.510079+0000","last_clean":"2023-10-09T06:31:51.510079+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:31:51.510079+0000","last_undegraded":"2023-10-09T06:31:51.510079+0000","last_fullsized":"2023-10-09T06:31:51.510079+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"373'3918","reported_seq":3939,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:55.642971+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:31:55.642971+0000","last_peered":"2023-10-09T06:31:55.642971+0000","last_clean":"2023-10-09T06:31:55.642971+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:31:55.642971+0000","last_undegraded":"2023-10-09T06:31:55.642971+0000","last_fullsized":"2023-10-09T06:31:55.642971+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3918,"log_dups_size":0,"ondisk_log_size":3918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256770048,"num_objects":3918,"num_object_clones":0,"num_object_copies":11754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3918,"num_write_kb":250752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":710,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.454262+0000","last_change":"2023-10-09T06:31:52.453039+0000","last_active":"2023-10-09T06:31:52.454262+0000","last_peered":"2023-10-09T06:31:52.454262+0000","last_clean":"2023-10-09T06:31:52.454262+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:31:52.454262+0000","last_undegraded":"2023-10-09T06:31:52.454262+0000","last_fullsized":"2023-10-09T06:31:52.454262+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:02.875686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:02.875686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441693+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:31:52.441693+0000","last_peered":"2023-10-09T06:31:52.441693+0000","last_clean":"2023-10-09T06:31:52.441693+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:31:52.441693+0000","last_undegraded":"2023-10-09T06:31:52.441693+0000","last_fullsized":"2023-10-09T06:31:52.441693+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438124+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:31:52.438124+0000","last_peered":"2023-10-09T06:31:52.438124+0000","last_clean":"2023-10-09T06:31:52.438124+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:31:52.438124+0000","last_undegraded":"2023-10-09T06:31:52.438124+0000","last_fullsized":"2023-10-09T06:31:52.438124+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":341,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510254+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:31:51.510254+0000","last_peered":"2023-10-09T06:31:51.510254+0000","last_clean":"2023-10-09T06:31:51.510254+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:31:51.510254+0000","last_undegraded":"2023-10-09T06:31:51.510254+0000","last_fullsized":"2023-10-09T06:31:51.510254+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"373'4018","reported_seq":4038,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:56.303576+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:31:56.303576+0000","last_peered":"2023-10-09T06:31:56.303576+0000","last_clean":"2023-10-09T06:31:56.303576+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:31:56.303576+0000","last_undegraded":"2023-10-09T06:31:56.303576+0000","last_fullsized":"2023-10-09T06:31:56.303576+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4018,"log_dups_size":0,"ondisk_log_size":4018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263323648,"num_objects":4018,"num_object_clones":0,"num_object_copies":12054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4018,"num_write_kb":257152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":706,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.449952+0000","last_change":"2023-10-09T06:31:52.448623+0000","last_active":"2023-10-09T06:31:52.449952+0000","last_peered":"2023-10-09T06:31:52.449952+0000","last_clean":"2023-10-09T06:31:52.449952+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:31:52.449952+0000","last_undegraded":"2023-10-09T06:31:52.449952+0000","last_fullsized":"2023-10-09T06:31:52.449952+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:29.742229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:29.742229+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069915519999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":363,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.442486+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:31:52.442486+0000","last_peered":"2023-10-09T06:31:52.442486+0000","last_clean":"2023-10-09T06:31:52.442486+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:31:52.442486+0000","last_undegraded":"2023-10-09T06:31:52.442486+0000","last_fullsized":"2023-10-09T06:31:52.442486+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":349,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510032+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:31:51.510032+0000","last_peered":"2023-10-09T06:31:51.510032+0000","last_clean":"2023-10-09T06:31:51.510032+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:31:51.510032+0000","last_undegraded":"2023-10-09T06:31:51.510032+0000","last_fullsized":"2023-10-09T06:31:51.510032+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":371,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.511570+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:31:51.511570+0000","last_peered":"2023-10-09T06:31:51.511570+0000","last_clean":"2023-10-09T06:31:51.511570+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:31:51.511570+0000","last_undegraded":"2023-10-09T06:31:51.511570+0000","last_fullsized":"2023-10-09T06:31:51.511570+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"374'4154","reported_seq":4175,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:58.340193+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:31:58.340193+0000","last_peered":"2023-10-09T06:31:58.340193+0000","last_clean":"2023-10-09T06:31:58.340193+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:31:58.340193+0000","last_undegraded":"2023-10-09T06:31:58.340193+0000","last_fullsized":"2023-10-09T06:31:58.340193+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272236544,"num_objects":4154,"num_object_clones":0,"num_object_copies":12462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4154,"num_write_kb":265856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441699+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:31:52.441699+0000","last_peered":"2023-10-09T06:31:52.441699+0000","last_clean":"2023-10-09T06:31:52.441699+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:31:52.441699+0000","last_undegraded":"2023-10-09T06:31:52.441699+0000","last_fullsized":"2023-10-09T06:31:52.441699+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":687,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.457980+0000","last_change":"2023-10-09T06:31:52.456193+0000","last_active":"2023-10-09T06:31:52.457980+0000","last_peered":"2023-10-09T06:31:52.457980+0000","last_clean":"2023-10-09T06:31:52.457980+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:31:52.457980+0000","last_undegraded":"2023-10-09T06:31:52.457980+0000","last_fullsized":"2023-10-09T06:31:52.457980+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:27.787446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:27.787446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":82,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444068+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:31:52.444068+0000","last_peered":"2023-10-09T06:31:52.444068+0000","last_clean":"2023-10-09T06:31:52.444068+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:31:52.444068+0000","last_undegraded":"2023-10-09T06:31:52.444068+0000","last_fullsized":"2023-10-09T06:31:52.444068+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":335,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.414522+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:31:51.414522+0000","last_peered":"2023-10-09T06:31:51.414522+0000","last_clean":"2023-10-09T06:31:51.414522+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:31:51.414522+0000","last_undegraded":"2023-10-09T06:31:51.414522+0000","last_fullsized":"2023-10-09T06:31:51.414522+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"373'4008","reported_seq":4029,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:55.652177+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:31:55.652177+0000","last_peered":"2023-10-09T06:31:55.652177+0000","last_clean":"2023-10-09T06:31:55.652177+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:31:55.652177+0000","last_undegraded":"2023-10-09T06:31:55.652177+0000","last_fullsized":"2023-10-09T06:31:55.652177+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4008,"log_dups_size":0,"ondisk_log_size":4008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262668288,"num_objects":4008,"num_object_clones":0,"num_object_copies":12024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4008,"num_write_kb":256512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":668,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.460798+0000","last_change":"2023-10-09T06:31:52.460798+0000","last_active":"2023-10-09T06:31:52.460798+0000","last_peered":"2023-10-09T06:31:52.460798+0000","last_clean":"2023-10-09T06:31:52.460798+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:31:52.460798+0000","last_undegraded":"2023-10-09T06:31:52.460798+0000","last_fullsized":"2023-10-09T06:31:52.460798+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":237,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444278+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:31:52.444278+0000","last_peered":"2023-10-09T06:31:52.444278+0000","last_clean":"2023-10-09T06:31:52.444278+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:31:52.444278+0000","last_undegraded":"2023-10-09T06:31:52.444278+0000","last_fullsized":"2023-10-09T06:31:52.444278+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409029+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:31:51.409029+0000","last_peered":"2023-10-09T06:31:51.409029+0000","last_clean":"2023-10-09T06:31:51.409029+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:31:51.409029+0000","last_undegraded":"2023-10-09T06:31:51.409029+0000","last_fullsized":"2023-10-09T06:31:51.409029+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":355,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.410946+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:31:51.410946+0000","last_peered":"2023-10-09T06:31:51.410946+0000","last_clean":"2023-10-09T06:31:51.410946+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:31:51.410946+0000","last_undegraded":"2023-10-09T06:31:51.410946+0000","last_fullsized":"2023-10-09T06:31:51.410946+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"374'3988","reported_seq":4010,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.946470+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:31:57.946470+0000","last_peered":"2023-10-09T06:31:57.946470+0000","last_clean":"2023-10-09T06:31:57.946470+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:31:57.946470+0000","last_undegraded":"2023-10-09T06:31:57.946470+0000","last_fullsized":"2023-10-09T06:31:57.946470+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3988,"log_dups_size":0,"ondisk_log_size":3988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261357568,"num_objects":3988,"num_object_clones":0,"num_object_copies":11964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3988,"num_write_kb":255232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":675,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.456892+0000","last_change":"2023-10-09T06:31:52.456892+0000","last_active":"2023-10-09T06:31:52.456892+0000","last_peered":"2023-10-09T06:31:52.456892+0000","last_clean":"2023-10-09T06:31:52.456892+0000","last_became_active":"2023-10-09T06:31:52.456527+0000","last_became_peered":"2023-10-09T06:31:52.456527+0000","last_unstale":"2023-10-09T06:31:52.456892+0000","last_undegraded":"2023-10-09T06:31:52.456892+0000","last_fullsized":"2023-10-09T06:31:52.456892+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":237,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444308+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:31:52.444308+0000","last_peered":"2023-10-09T06:31:52.444308+0000","last_clean":"2023-10-09T06:31:52.444308+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:31:52.444308+0000","last_undegraded":"2023-10-09T06:31:52.444308+0000","last_fullsized":"2023-10-09T06:31:52.444308+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":367,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510152+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:31:51.510152+0000","last_peered":"2023-10-09T06:31:51.510152+0000","last_clean":"2023-10-09T06:31:51.510152+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:31:51.510152+0000","last_undegraded":"2023-10-09T06:31:51.510152+0000","last_fullsized":"2023-10-09T06:31:51.510152+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":354,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409684+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:31:51.409684+0000","last_peered":"2023-10-09T06:31:51.409684+0000","last_clean":"2023-10-09T06:31:51.409684+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:31:51.409684+0000","last_undegraded":"2023-10-09T06:31:51.409684+0000","last_fullsized":"2023-10-09T06:31:51.409684+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"374'4013","reported_seq":4034,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.963955+0000","last_change":"2023-10-09T06:31:03.217038+0000","last_active":"2023-10-09T06:31:57.963955+0000","last_peered":"2023-10-09T06:31:57.963955+0000","last_clean":"2023-10-09T06:31:57.963955+0000","last_became_active":"2023-10-09T06:31:03.216747+0000","last_became_peered":"2023-10-09T06:31:03.216747+0000","last_unstale":"2023-10-09T06:31:57.963955+0000","last_undegraded":"2023-10-09T06:31:57.963955+0000","last_fullsized":"2023-10-09T06:31:57.963955+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4013,"log_dups_size":0,"ondisk_log_size":4013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:22.026356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262995968,"num_objects":4013,"num_object_clones":0,"num_object_copies":12039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4013,"num_write_kb":256832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":667,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.446538+0000","last_change":"2023-10-09T06:31:52.446308+0000","last_active":"2023-10-09T06:31:52.446538+0000","last_peered":"2023-10-09T06:31:52.446538+0000","last_clean":"2023-10-09T06:31:52.446538+0000","last_became_active":"2023-10-09T06:31:52.446011+0000","last_became_peered":"2023-10-09T06:31:52.446011+0000","last_unstale":"2023-10-09T06:31:52.446538+0000","last_undegraded":"2023-10-09T06:31:52.446538+0000","last_fullsized":"2023-10-09T06:31:52.446538+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017933129999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":103,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-09T06:31:57.491528+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:31:57.491528+0000","last_peered":"2023-10-09T06:31:57.491528+0000","last_clean":"2023-10-09T06:31:57.491528+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:31:57.491528+0000","last_undegraded":"2023-10-09T06:31:57.491528+0000","last_fullsized":"2023-10-09T06:31:57.491528+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":334,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510482+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:31:51.510482+0000","last_peered":"2023-10-09T06:31:51.510482+0000","last_clean":"2023-10-09T06:31:51.510482+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:31:51.510482+0000","last_undegraded":"2023-10-09T06:31:51.510482+0000","last_fullsized":"2023-10-09T06:31:51.510482+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.448925+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:31:52.448925+0000","last_peered":"2023-10-09T06:31:52.448925+0000","last_clean":"2023-10-09T06:31:52.448925+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:31:52.448925+0000","last_undegraded":"2023-10-09T06:31:52.448925+0000","last_fullsized":"2023-10-09T06:31:52.448925+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"373'3932","reported_seq":3952,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:56.279699+0000","last_change":"2023-10-09T06:31:03.215173+0000","last_active":"2023-10-09T06:31:56.279699+0000","last_peered":"2023-10-09T06:31:56.279699+0000","last_clean":"2023-10-09T06:31:56.279699+0000","last_became_active":"2023-10-09T06:31:03.214809+0000","last_became_peered":"2023-10-09T06:31:03.214809+0000","last_unstale":"2023-10-09T06:31:56.279699+0000","last_undegraded":"2023-10-09T06:31:56.279699+0000","last_fullsized":"2023-10-09T06:31:56.279699+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":3932,"log_dups_size":0,"ondisk_log_size":3932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:12.323433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257687552,"num_objects":3932,"num_object_clones":0,"num_object_copies":11796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3932,"num_write_kb":251648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":713,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.956919+0000","last_change":"2023-10-09T06:31:52.455164+0000","last_active":"2023-10-09T06:31:52.956919+0000","last_peered":"2023-10-09T06:31:52.956919+0000","last_clean":"2023-10-09T06:31:52.956919+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:31:52.956919+0000","last_undegraded":"2023-10-09T06:31:52.956919+0000","last_fullsized":"2023-10-09T06:31:52.956919+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":109226,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":357,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.415030+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:31:51.415030+0000","last_peered":"2023-10-09T06:31:51.415030+0000","last_clean":"2023-10-09T06:31:51.415030+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:31:51.415030+0000","last_undegraded":"2023-10-09T06:31:51.415030+0000","last_fullsized":"2023-10-09T06:31:51.415030+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":336,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.410740+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:31:51.410740+0000","last_peered":"2023-10-09T06:31:51.410740+0000","last_clean":"2023-10-09T06:31:51.410740+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:31:51.410740+0000","last_undegraded":"2023-10-09T06:31:51.410740+0000","last_fullsized":"2023-10-09T06:31:51.410740+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":337,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437316+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:31:52.437316+0000","last_peered":"2023-10-09T06:31:52.437316+0000","last_clean":"2023-10-09T06:31:52.437316+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:31:52.437316+0000","last_undegraded":"2023-10-09T06:31:52.437316+0000","last_fullsized":"2023-10-09T06:31:52.437316+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":338,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409608+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:31:51.409608+0000","last_peered":"2023-10-09T06:31:51.409608+0000","last_clean":"2023-10-09T06:31:51.409608+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:31:51.409608+0000","last_undegraded":"2023-10-09T06:31:51.409608+0000","last_fullsized":"2023-10-09T06:31:51.409608+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":360,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.510101+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:31:51.510101+0000","last_peered":"2023-10-09T06:31:51.510101+0000","last_clean":"2023-10-09T06:31:51.510101+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:31:51.510101+0000","last_undegraded":"2023-10-09T06:31:51.510101+0000","last_fullsized":"2023-10-09T06:31:51.510101+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":65,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.441640+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:31:52.441640+0000","last_peered":"2023-10-09T06:31:52.441640+0000","last_clean":"2023-10-09T06:31:52.441640+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:31:52.441640+0000","last_undegraded":"2023-10-09T06:31:52.441640+0000","last_fullsized":"2023-10-09T06:31:52.441640+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":381,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438045+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:31:52.438045+0000","last_peered":"2023-10-09T06:31:52.438045+0000","last_clean":"2023-10-09T06:31:52.438045+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:31:52.438045+0000","last_undegraded":"2023-10-09T06:31:52.438045+0000","last_fullsized":"2023-10-09T06:31:52.438045+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":65,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.440976+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:31:52.440976+0000","last_peered":"2023-10-09T06:31:52.440976+0000","last_clean":"2023-10-09T06:31:52.440976+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:31:52.440976+0000","last_undegraded":"2023-10-09T06:31:52.440976+0000","last_fullsized":"2023-10-09T06:31:52.440976+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":391,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.437657+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:31:52.437657+0000","last_peered":"2023-10-09T06:31:52.437657+0000","last_clean":"2023-10-09T06:31:52.437657+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:31:52.437657+0000","last_undegraded":"2023-10-09T06:31:52.437657+0000","last_fullsized":"2023-10-09T06:31:52.437657+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":73,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.444298+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:31:52.444298+0000","last_peered":"2023-10-09T06:31:52.444298+0000","last_clean":"2023-10-09T06:31:52.444298+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:31:52.444298+0000","last_undegraded":"2023-10-09T06:31:52.444298+0000","last_fullsized":"2023-10-09T06:31:52.444298+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":90,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.442727+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:31:52.442727+0000","last_peered":"2023-10-09T06:31:52.442727+0000","last_clean":"2023-10-09T06:31:52.442727+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:31:52.442727+0000","last_undegraded":"2023-10-09T06:31:52.442727+0000","last_fullsized":"2023-10-09T06:31:52.442727+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":367,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-10-09T06:31:51.409318+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:31:51.409318+0000","last_peered":"2023-10-09T06:31:51.409318+0000","last_clean":"2023-10-09T06:31:51.409318+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:31:51.409318+0000","last_undegraded":"2023-10-09T06:31:51.409318+0000","last_fullsized":"2023-10-09T06:31:51.409318+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":334,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-10-09T06:31:52.438121+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:31:52.438121+0000","last_peered":"2023-10-09T06:31:52.438121+0000","last_clean":"2023-10-09T06:31:52.438121+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:31:52.438121+0000","last_undegraded":"2023-10-09T06:31:52.438121+0000","last_fullsized":"2023-10-09T06:31:52.438121+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4182310912,"num_objects":63817,"num_object_clones":0,"num_object_copies":191451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63817,"num_write_kb":4084288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12569673728,"data_stored":12569673728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63817,"ondisk_log_size":63817,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234416,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2716448,"kb_used_data":1820312,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":91028704,"statfs":{"total":95995035648,"available":93213392896,"internally_reserved":0,"allocated":1863999488,"data_stored":1863038652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":4.9039999999999999,"15min":4.9569999999999999},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":4.2549999999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":2.2890000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.9370000000000003,"15min":5.0670000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.218},"max":{"1min":3.2930000000000001,"5min":215.577,"15min":219.399},"last":2.2669999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":5.9080000000000004,"5min":6.9850000000000003,"15min":4.4669999999999996},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":107.87,"5min":215.61699999999999,"15min":215.61699999999999},"last":2.0059999999999998},{"interface":"front","average":{"1min":6.0869999999999997,"5min":6.1299999999999999,"15min":4.0709999999999997},"min":{"1min":0.38600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.904,"5min":216.22200000000001,"15min":216.22200000000001},"last":1.71}]},{"osd":2,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":10.584,"5min":6.4859999999999998,"15min":6.665},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":198.94200000000001,"5min":216.34299999999999,"15min":216.34299999999999},"last":4.3460000000000001},{"interface":"front","average":{"1min":10.708,"5min":6.5149999999999997,"15min":6.8470000000000004},"min":{"1min":0.30299999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":198.92099999999999,"5min":216.25200000000001,"15min":216.25200000000001},"last":2.3620000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":7.4969999999999999,"15min":3.931},"min":{"1min":0.37,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":3.4460000000000002,"5min":215.922,"15min":215.922},"last":1.9870000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":7.6669999999999998,"15min":4.0170000000000003},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":3.3210000000000002,"5min":215.94900000000001,"15min":215.94900000000001},"last":6.7160000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":4.0579999999999998,"15min":2.5419999999999998},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":4.4589999999999996,"5min":218.446,"15min":218.446},"last":4.9329999999999998},{"interface":"front","average":{"1min":0.755,"5min":3.9969999999999999,"15min":2.395},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.23699999999999999},"max":{"1min":3.2210000000000001,"5min":216.172,"15min":216.172},"last":0.29799999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":4.2270000000000003,"15min":2.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":3.7349999999999999,"5min":216.37200000000001,"15min":216.37200000000001},"last":1.972},{"interface":"front","average":{"1min":1.1839999999999999,"5min":4.1559999999999997,"15min":2.4369999999999998},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.5670000000000002,"5min":218.15899999999999,"15min":218.15899999999999},"last":4.8970000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":4.2880000000000003,"15min":2.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":4.548,"5min":218.19999999999999,"15min":218.19999999999999},"last":2.3300000000000001},{"interface":"front","average":{"1min":0.80100000000000005,"5min":3.944,"15min":2.149},"min":{"1min":0.40799999999999997,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":3.2650000000000001,"5min":218.05500000000001,"15min":218.05500000000001},"last":0.47399999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496051,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2656708,"kb_used_data":1761788,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91088444,"statfs":{"total":95995035648,"available":93274566656,"internally_reserved":0,"allocated":1804070912,"data_stored":1803129883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.601,"15min":3.0369999999999999},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.7370000000000001,"5min":22.321999999999999,"15min":110.926},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.258,"5min":2.1259999999999999,"15min":3.3610000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.4340000000000002,"5min":22.338000000000001,"15min":110.911},"last":0.98599999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":6.9020000000000001,"15min":4.8869999999999996},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":2.375,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":6.9800000000000004,"15min":4.7400000000000002},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":4.4359999999999999,"5min":292.904,"15min":292.904},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.9930000000000001,"15min":2.5369999999999999},"min":{"1min":0.307,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.9009999999999998,"5min":20.331,"15min":75.191999999999993},"last":1.206},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.661,"15min":2.2509999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.3479999999999999,"5min":8.6069999999999993,"15min":75.120000000000005},"last":1.2290000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":4.1180000000000003,"15min":3.2120000000000002},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.28299999999999997},"max":{"1min":88.763000000000005,"5min":91.174000000000007,"15min":110.358},"last":0.378},{"interface":"front","average":{"1min":6.0300000000000002,"5min":4.2080000000000002,"15min":3.2850000000000001},"min":{"1min":0.33100000000000002,"5min":0.318,"15min":0.311},"max":{"1min":86.831999999999994,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 06:31:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.4359999999999999,"15min":1.2769999999999999},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":2.6200000000000001,"5min":12.666,"15min":17.643999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.071,"15min":1.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":2.9940000000000002,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.39100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:31:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.335,"15min":1.3540000000000001},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":2.4870000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.3380000000000001,"5min":1.5309999999999999,"15min":1.343},"min":{"1min":0.38400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":6.9349999999999996,"5min":16.536999999999999,"15min":24.401},"last":0.42399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":4.1630000000000003,"15min":2.569},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":19.295999999999999,"5min":122.937,"15min":122.937},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.647,"5min":4.444,"15min":2.6850000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.584,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.44800000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402789,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171636,"kb_used_data":1477868,"kb_used_omap":4,"kb_used_meta":693755,"kb_avail":91573516,"statfs":{"total":95995035648,"available":93771280384,"internally_reserved":0,"allocated":1513336832,"data_stored":1511649512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":710406030},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":10.044,"5min":8.0340000000000007,"15min":5.7519999999999998},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.189},"max":{"1min":143.465,"5min":269.79199999999997,"15min":269.79199999999997},"last":6.8410000000000002},{"interface":"front","average":{"1min":10.407,"5min":8.1820000000000004,"15min":6.1429999999999998},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.17899999999999999},"max":{"1min":143.66399999999999,"5min":271.20999999999998,"15min":271.20999999999998},"last":2.0779999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:31:39 2023","interfaces":[{"interface":"back","average":{"1min":9.2430000000000003,"5min":2.9220000000000002,"15min":3.4550000000000001},"min":{"1min":0.23799999999999999,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":138.452,"5min":138.452,"15min":139.75399999999999},"last":6.9139999999999997},{"interface":"front","average":{"1min":9.4960000000000004,"5min":3.746,"15min":4.1379999999999999},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":138.47499999999999,"5min":138.47499999999999,"15min":140.29900000000001},"last":2.044}]},{"osd":3,"last update":"Mon Oct 9 06:31:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.5409999999999999,"15min":3.3050000000000002},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":4.452,"5min":31.062000000000001,"15min":153.18199999999999},"last":7.5490000000000004},{"interface":"front","average":{"1min":0.437,"5min":1.952,"15min":2.4460000000000002},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.95999999999999996,"5min":27.657,"15min":152.52000000000001},"last":6.8879999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.7569999999999999,"15min":3.3610000000000002},"min":{"1min":0.317,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":2.5409999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":7.4969999999999999},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.655,"15min":3.1699999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.605,"5min":11.974,"15min":199.32900000000001},"last":2.0150000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":1.8779999999999999,"15min":2.0049999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.9289999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":6.9400000000000004},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.1779999999999999,"15min":2.4670000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":2.5779999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":2.145}]},{"osd":6,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":2.5680000000000001,"15min":2.6930000000000001},"min":{"1min":0.35599999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.4980000000000002,"5min":52.488,"15min":77.292000000000002},"last":2.1749999999999998},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.8109999999999999,"15min":2.758},"min":{"1min":0.38700000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.8280000000000001,"5min":52.558,"15min":77.378},"last":7.4649999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.681,"15min":2.9660000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":0.79500000000000004,"5min":55.231000000000002,"15min":77.456999999999994},"last":7.532},{"interface":"front","average":{"1min":0.55400000000000005,"5min":3.0539999999999998,"15min":2.4289999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":0.81499999999999995,"5min":55.253999999999998,"15min":77.359999999999999},"last":6.9580000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024222,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2534820,"kb_used_data":1720540,"kb_used_omap":5,"kb_used_meta":814266,"kb_avail":91210332,"statfs":{"total":95995035648,"available":93399379968,"internally_reserved":0,"allocated":1761832960,"data_stored":1759871221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":833809018},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:31:29 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":3.2789999999999999,"15min":3.3290000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":3.2839999999999998,"5min":191.446,"15min":191.446},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.3929999999999998,"15min":3.5579999999999998},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":4.0609999999999999,"5min":191.393,"15min":191.393},"last":4.9630000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.274,"15min":2.226},"min":{"1min":0.28599999999999998,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":4.0970000000000004,"5min":51.886000000000003,"15min":57.162999999999997},"last":4.9359999999999999},{"interface":"front","average":{"1min":1.7130000000000001,"5min":2.899,"15min":2.6339999999999999},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.6429999999999998,"5min":52.302999999999997,"15min":57.585000000000001},"last":6.0140000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:31:07 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":7.8419999999999996,"15min":5.4000000000000004},"min":{"1min":0.26800000000000002,"5min":0.22800000000000001,"15min":0.22500000000000001},"max":{"1min":187.172,"5min":208.11199999999999,"15min":208.11199999999999},"last":5.0999999999999996},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.4950000000000001,"15min":5.391},"min":{"1min":0.31900000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":194.851,"5min":208.25899999999999,"15min":208.25899999999999},"last":4.9909999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.518,"15min":1.429},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":7.21,"5min":9.7159999999999993,"15min":11.162000000000001},"last":1.248},{"interface":"front","average":{"1min":2.1469999999999998,"5min":1.7470000000000001,"15min":1.798},"min":{"1min":0.34100000000000003,"5min":0.32300000000000001,"15min":0.28299999999999997},"max":{"1min":10.903,"5min":18.081,"15min":18.081},"last":4.8970000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.7130000000000001,"15min":1.806},"min":{"1min":0.317,"5min":0.317,"15min":0.27800000000000002},"max":{"1min":7.2409999999999997,"5min":10.865,"15min":62.984999999999999},"last":1.2},{"interface":"front","average":{"1min":2.2109999999999999,"5min":1.9970000000000001,"15min":2.2469999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":10.603,"5min":15.747999999999999,"15min":64.983999999999995},"last":5.0650000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":4.2619999999999996,"15min":2.8980000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":12.522,"5min":120.572,"15min":120.572},"last":5.149},{"interface":"front","average":{"1min":1.758,"5min":3.827,"15min":2.3330000000000002},"min":{"1min":0.371,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":11.093,"5min":120.64400000000001,"15min":120.64400000000001},"last":1.147}]},{"osd":7,"last update":"Mon Oct 9 06:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":3.1840000000000002,"15min":2.4969999999999999},"min":{"1min":0.40000000000000002,"5min":0.36899999999999999,"15min":0.32300000000000001},"max":{"1min":8.8059999999999992,"5min":122.018,"15min":122.018},"last":4.8639999999999999},{"interface":"front","average":{"1min":2.415,"5min":3.6539999999999999,"15min":2.8730000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":10.563000000000001,"5min":122.32299999999999,"15min":122.32299999999999},"last":5.125}]}]},{"osd":2,"up_from":128,"seq":549755814091,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3039224,"kb_used_data":2472048,"kb_used_omap":7,"kb_used_meta":567160,"kb_avail":90705928,"statfs":{"total":95995035648,"available":92882870272,"internally_reserved":0,"allocated":2531377152,"data_stored":2529742565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":580771921},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":4.5640000000000001,"15min":4.7919999999999998},"min":{"1min":0.22,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":29.071999999999999,"5min":162.21100000000001,"15min":163.40100000000001},"last":2.3660000000000001},{"interface":"front","average":{"1min":3.1440000000000001,"5min":4.6989999999999998,"15min":4.4530000000000003},"min":{"1min":0.26100000000000001,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":29.102,"5min":168.49700000000001,"15min":168.49700000000001},"last":1.627}]},{"osd":1,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":3.3849999999999998,"15min":4.1390000000000002},"min":{"1min":0.246,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":10.287000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":2.2629999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":4.3700000000000001,"15min":4.4560000000000004},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.7579999999999991,"5min":91.721999999999994,"15min":222.19399999999999},"last":2.0230000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":3.3370000000000002,"15min":2.8090000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.8499999999999996,"5min":91.644000000000005,"15min":91.644000000000005},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.841,"5min":3.9809999999999999,"15min":3.056},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.9059999999999997,"5min":91.685000000000002,"15min":91.685000000000002},"last":0.59199999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.861,"15min":2.0339999999999998},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.29099999999999998},"max":{"1min":9.6959999999999997,"5min":17.776,"15min":90.938999999999993},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.038,"5min":2.0619999999999998,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":4.0679999999999996,"5min":12.433999999999999,"15min":90.950000000000003},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.0870000000000002,"15min":1.5820000000000001},"min":{"1min":0.39100000000000001,"5min":0.315,"15min":0.309},"max":{"1min":9.6660000000000004,"5min":17.800000000000001,"15min":17.800000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.1280000000000001,"15min":1.8580000000000001},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.6659999999999999,"5min":11.285,"15min":11.285},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":8.5259999999999998,"5min":3.742,"15min":2.9590000000000001},"min":{"1min":0.35099999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":123.47799999999999,"5min":123.47799999999999,"15min":138.291},"last":0.54900000000000004},{"interface":"front","average":{"1min":8.6419999999999995,"5min":3.1789999999999998,"15min":2.645},"min":{"1min":0.47899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":122.517,"5min":122.517,"15min":138.01599999999999},"last":2.1859999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.004,"15min":1.925},"min":{"1min":0.42799999999999999,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":4.3559999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":2.2050000000000001,"15min":1.905},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7279999999999998,"5min":17.734999999999999,"15min":55.731000000000002},"last":0.53800000000000003}]}]},{"osd":3,"up_from":149,"seq":639950127289,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2835652,"kb_used_data":2297532,"kb_used_omap":2,"kb_used_meta":538109,"kb_avail":90909500,"statfs":{"total":95995035648,"available":93091328000,"internally_reserved":0,"allocated":2352672768,"data_stored":2351094185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":551023669},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.806,"5min":3.3180000000000001,"15min":4.1609999999999996},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":47.847000000000001,"5min":47.847000000000001,"15min":187.852},"last":15.48},{"interface":"front","average":{"1min":4.0030000000000001,"5min":3.0419999999999998,"15min":4.4180000000000001},"min":{"1min":0.28199999999999997,"5min":0.20799999999999999,"15min":0.184},"max":{"1min":47.896999999999998,"5min":47.896999999999998,"15min":187.923},"last":0.81799999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.0279999999999996,"5min":2.734,"15min":3.5449999999999999},"min":{"1min":0.29599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":47.960999999999999,"5min":47.960999999999999,"15min":187.89099999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":5.5490000000000004,"5min":4.0780000000000003,"15min":3.0939999999999999},"min":{"1min":0.26100000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":48.087000000000003,"5min":48.087000000000003,"15min":67.028000000000006},"last":4.4299999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.8809999999999998,"15min":3.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":48.055,"5min":48.055,"15min":187.76400000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":5.25,"5min":3.375,"15min":3.1200000000000001},"min":{"1min":0.33100000000000002,"5min":0.27900000000000003,"15min":0.22500000000000001},"max":{"1min":48.128999999999998,"5min":48.128999999999998,"15min":187.79499999999999},"last":3.2029999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":2.657,"15min":1.78},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":48.249000000000002,"5min":48.249000000000002,"15min":48.249000000000002},"last":3.1800000000000002},{"interface":"front","average":{"1min":4.4370000000000003,"5min":2.6930000000000001,"15min":2.4649999999999999},"min":{"1min":0.40899999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":48.031999999999996,"5min":48.031999999999996,"15min":187.67400000000001},"last":3.7669999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":2.4510000000000001,"15min":2.5169999999999999},"min":{"1min":0.40200000000000002,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":48.162999999999997,"5min":48.162999999999997,"15min":187.833},"last":3.8690000000000002},{"interface":"front","average":{"1min":3.9119999999999999,"5min":2.0409999999999999,"15min":2.3079999999999998},"min":{"1min":0.46300000000000002,"5min":0.34599999999999997,"15min":0.32100000000000001},"max":{"1min":48.106000000000002,"5min":48.106000000000002,"15min":187.952},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.7359999999999998,"5min":4.1029999999999998,"15min":3.125},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":48.270000000000003,"5min":130.39599999999999,"15min":187.78200000000001},"last":3.831},{"interface":"front","average":{"1min":3.8370000000000002,"5min":3.6099999999999999,"15min":2.7949999999999999},"min":{"1min":0.40300000000000002,"5min":0.34399999999999997,"15min":0.32900000000000001},"max":{"1min":48.155000000000001,"5min":130.67699999999999,"15min":187.81800000000001},"last":3.1070000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":4.2069999999999999,"5min":4.4710000000000001,"15min":3.109},"min":{"1min":0.41899999999999998,"5min":0.38500000000000001,"15min":0.34100000000000003},"max":{"1min":48.292000000000002,"5min":121.613,"15min":192.20400000000001},"last":3.1509999999999998},{"interface":"front","average":{"1min":5.4050000000000002,"5min":4.7080000000000002,"15min":3.2570000000000001},"min":{"1min":0.44700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":48.200000000000003,"5min":121.559,"15min":187.86500000000001},"last":3.9009999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019325,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1679500,"kb_used_data":784516,"kb_used_omap":0,"kb_used_meta":894976,"kb_avail":92065652,"statfs":{"total":95995035648,"available":94275227648,"internally_reserved":0,"allocated":803344384,"data_stored":802395540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916455424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.696,"15min":2.2210000000000001},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":18.625,"5min":37.485999999999997,"15min":76.944999999999993},"last":0.375},{"interface":"front","average":{"1min":1.962,"5min":1.77,"15min":2.4740000000000002},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.654,"5min":37.502000000000002,"15min":76.988},"last":0.51600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":4.2999999999999998,"15min":2.891},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23100000000000001},"max":{"1min":0.67700000000000005,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":4.3200000000000003,"15min":3.0590000000000002},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":1.0660000000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":0.309}]},{"osd":2,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":2.0950000000000002,"5min":32.613,"15min":69.640000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.7629999999999999,"15min":2.4540000000000002},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.9320000000000004,"5min":34.188000000000002,"15min":69.676000000000002},"last":0.59099999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7669999999999999,"5min":7.024,"15min":4.4420000000000002},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.30299999999999999},"max":{"1min":35.018000000000001,"5min":133.38900000000001,"15min":133.38900000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.1869999999999998,"5min":6.7530000000000001,"15min":4.173},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":35.036999999999999,"5min":132.57599999999999,"15min":132.57599999999999},"last":1.7829999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.93999999999999995,"15min":1.0249999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.7829999999999999,"5min":6.25,"15min":10.673},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.0760000000000001,"15min":1.0309999999999999},"min":{"1min":0.32200000000000001,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":4.9589999999999996,"5min":7.2249999999999996,"15min":12.552},"last":0.35499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:30:58 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":2.4660000000000002,"15min":1.3340000000000001},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.0629999999999997,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.0529999999999999,"5min":2.8029999999999999,"15min":1.7989999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.7210000000000001,"5min":163.381,"15min":163.381},"last":3.004}]},{"osd":7,"last update":"Mon Oct 9 06:31:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.05,"15min":1.476},"min":{"1min":0.32700000000000001,"5min":0.312,"15min":0.308},"max":{"1min":3.6989999999999998,"5min":6.2679999999999998,"15min":62.658000000000001},"last":4.3319999999999999},{"interface":"front","average":{"1min":1.734,"5min":0.995,"15min":1.462},"min":{"1min":0.35299999999999998,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":62.573},"last":0.41699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757690,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1422924,"kb_used_data":525316,"kb_used_omap":0,"kb_used_meta":897600,"kb_avail":92322228,"statfs":{"total":95995035648,"available":94537961472,"internally_reserved":0,"allocated":537923584,"data_stored":536979123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919142400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.0990000000000002,"15min":3.4900000000000002},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":3.819,"5min":131.87200000000001,"15min":131.87200000000001},"last":3.125},{"interface":"front","average":{"1min":0.83999999999999997,"5min":3.8769999999999998,"15min":3.4260000000000002},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.22},"max":{"1min":3.6709999999999998,"5min":131.80199999999999,"15min":131.80199999999999},"last":3.1139999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:31:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":4.9290000000000003,"15min":2.7589999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":3.7109999999999999,"5min":126.71899999999999,"15min":126.71899999999999},"last":1.5249999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":5.2169999999999996,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":7.8099999999999996,"5min":126.931,"15min":126.931},"last":2.6760000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":2.5209999999999999,"15min":2.2000000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9029999999999996,"5min":51.393000000000001,"15min":81.260000000000005},"last":2.4129999999999998},{"interface":"front","average":{"1min":0.497,"5min":2.097,"15min":2.0600000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.29699999999999999},"max":{"1min":1.23,"5min":51.432000000000002,"15min":81.245000000000005},"last":2.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:31:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5049999999999999,"5min":9.5310000000000006,"15min":5.4580000000000002},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.314},"max":{"1min":92.521000000000001,"5min":144.56399999999999,"15min":190.411},"last":2.6179999999999999},{"interface":"front","average":{"1min":5.4720000000000004,"5min":9.4909999999999997,"15min":5.3449999999999998},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.28699999999999998},"max":{"1min":94.492000000000004,"5min":144.62799999999999,"15min":190.70699999999999},"last":3.0529999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.0209999999999999,"15min":1.236},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.6279999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":1.49},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.872,"15min":1.089},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0760000000000001,"5min":6.2610000000000001,"15min":32.82},"last":2.2450000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:31:00 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.3080000000000001,"15min":1.5509999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":3.6859999999999999,"5min":6.2080000000000002,"15min":38.902000000000001},"last":2.4729999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.395,"15min":1.5920000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.6340000000000003,"5min":8.3659999999999997,"15min":35.774000000000001},"last":2.081}]},{"osd":7,"last update":"Mon Oct 9 06:31:15 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":1.363,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.226,"5min":10.461,"15min":30.658999999999999},"last":3.4049999999999998},{"interface":"front","average":{"1min":0.69399999999999995,"5min":1.1970000000000001,"15min":1.448},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":3.1930000000000001,"5min":6.2320000000000002,"15min":30.683},"last":2.048}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1543438336,"data_stored":1543438336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1312817152,"data_stored":1312817152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2563833856,"data_stored":2563833856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2113011712,"data_stored":2113011712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":808583168,"data_stored":808583168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":537985024,"data_stored":537985024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1813774336,"data_stored":1813774336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1876230144,"data_stored":1876230144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:32:01.345 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:32:01.345 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-10-09T06:32:01.367 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:01.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:01.483 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.563 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.565 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: osdmap e376: 8 total, 8 up, 8 in 2023-10-09T06:32:01.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: failed to encode map e376 with expected crc 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:01.667 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:01.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:01.771 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:01.807 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:01.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 67955 67939 84.8982 47.125 0.0248254 0.0117729 2023-10-09T06:32:01.991 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:02.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:02.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:02.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:02.151 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:02.199 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:32:02.239 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:02.251 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:32:02.308 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:32:02.317 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:02] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:32:02.356 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:32:02.403 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:02.409 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:32:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:02.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:02.449 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:32:02.493 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:32:02.512 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:02.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:32:02.578 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: pgmap v1164: 109 pgs: 6 remapped+peering, 103 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 28 KiB/s rd, 75 MiB/s wr, 1.22k op/s; 87 KiB/s, 0 objects/s recovering 2023-10-09T06:32:02.578 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.579 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2068211797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.580 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[212750]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: pgmap v1164: 109 pgs: 6 remapped+peering, 103 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 28 KiB/s rd, 75 MiB/s wr, 1.22k op/s; 87 KiB/s, 0 objects/s recovering 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.581 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2068211797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.583 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:02 smithi107 ceph-mon[220921]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:32:02.593 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:02.687 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:32:02.688 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:32:02.688 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:32:02.688 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:32:02.688 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:32:02.708 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: pgmap v1164: 109 pgs: 6 remapped+peering, 103 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 28 KiB/s rd, 75 MiB/s wr, 1.22k op/s; 87 KiB/s, 0 objects/s recovering 2023-10-09T06:32:02.708 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: osdmap e377: 8 total, 8 up, 8 in 2023-10-09T06:32:02.708 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.708 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: failed to encode map e377 with expected crc 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.709 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:02.710 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:02.710 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2068211797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:02.710 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:02.710 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:02 smithi150 ceph-mon[194178]: osdmap e378: 8 total, 8 up, 8 in 2023-10-09T06:32:02.734 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:02.952 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 68848 68832 84.3279 55.8125 0.016458 0.0118542 2023-10-09T06:32:03.579 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:03.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:03.658 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:32:03.684 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:03.761 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:03.832 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:03.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.839 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.840 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:03 smithi150 ceph-mon[194178]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:32:03.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:03.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[212750]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: failed to encode map e378 with expected crc 2023-10-09T06:32:03.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:03 smithi107 ceph-mon[220921]: osdmap e379: 8 total, 8 up, 8 in 2023-10-09T06:32:03.942 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:03.955 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 69597 69581 83.6028 46.8125 0.0165292 0.0119558 2023-10-09T06:32:04.005 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:04.166 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:32:04.483 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:04.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:04.593 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:04.635 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:04.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: pgmap v1167: 109 pgs: 4 peering, 5 remapped+peering, 100 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 KiB/s rd, 60 MiB/s wr, 997 op/s; 211 KiB/s, 0 objects/s recovering 2023-10-09T06:32:04.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: pgmap v1167: 109 pgs: 4 peering, 5 remapped+peering, 100 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 KiB/s rd, 60 MiB/s wr, 997 op/s; 211 KiB/s, 0 objects/s recovering 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: pgmap v1167: 109 pgs: 4 peering, 5 remapped+peering, 100 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 KiB/s rd, 60 MiB/s wr, 997 op/s; 211 KiB/s, 0 objects/s recovering 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: failed to encode map e379 with expected crc 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:04.894 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:04.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:04.957 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 70628 70612 83.2381 64.4375 0.0129111 0.0120077 2023-10-09T06:32:04.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:05.078 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:05.746 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:05.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:05.858 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:05.886 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:05.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:05.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:05.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:05.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:05.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:05.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:32:05.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:32:05.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:05.904 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:05.957 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 71411 71395 82.6026 48.9375 0.0178357 0.0120996 2023-10-09T06:32:06.033 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:06.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:06.136 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:06.237 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:06.273 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:32:06.273 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67292 11.5 11:ae93ee2a:::benchmark_data_smithi107_418363_object67291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 281.536768575, 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3007185510000001, 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.274 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67292 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733749+0000", 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733812+0000", 2023-10-09T06:32:06.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3002000000000004e-05 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733823+0000", 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0909e-05 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733828+0000", 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6720000000000003e-06 2023-10-09T06:32:06.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733869+0000", 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1508999999999998e-05 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733970+0000", 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100907 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.734066+0000", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5404999999999999e-05 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.735966+0000", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001899916 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.736151+0000", 2023-10-09T06:32:06.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018566700000000001 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872581+0000", 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13643008600000001 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872592+0000", 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0746e-05 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.280 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871235+0000", 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99864300699999997 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871251+0000", 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5563e-05 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034370+0000", 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631196100000001 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034469+0000", 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8395999999999998e-05 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.282 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67301 11.5 11:a1b1512d:::benchmark_data_smithi107_418363_object67300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 281.42226541299999, 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1863352649999999, 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67301 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.283 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848252+0000", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848308+0000", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5899999999999997e-05 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848320+0000", 2023-10-09T06:32:06.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1922e-05 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848326+0000", 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2099999999999998e-06 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848378+0000", 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1793999999999999e-05 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848521+0000", 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142769 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848677+0000", 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000155828 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.287 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.850815+0000", 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021378920000000002 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852117+0000", 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00130197 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872639+0000", 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020522744999999998 2023-10-09T06:32:06.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872646+0000", 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7649999999999996e-06 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871333+0000", 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99868642699999999 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871343+0000", 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0808999999999999e-05 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034505+0000", 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631621480000001 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034589+0000", 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3185999999999999e-05 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67305 11.5 11:aa961c7b:::benchmark_data_smithi107_418363_object67304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 281.41819861599998, 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.182389261, 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.291 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67305 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852319+0000", 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852365+0000", 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6390999999999997e-05 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852381+0000", 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5128e-05 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852388+0000", 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2640000000000002e-06 2023-10-09T06:32:06.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853387+0000", 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00099880900000000002 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853487+0000", 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100614 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853585+0000", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7887999999999995e-05 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.855632+0000", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020469289999999998 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.857531+0000", 2023-10-09T06:32:06.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0018985919999999999 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872679+0000", 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015148790000000001 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872686+0000", 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.922e-06 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871386+0000", 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99869956000000004 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871393+0000", 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2060000000000001e-06 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034627+0000", 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1632337100000001 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034709+0000", 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2608000000000006e-05 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.299 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67306 11.5 11:aea7a0bf:::benchmark_data_smithi107_418363_object67305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 281.40597296800001, 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.17028496, 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67306 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864545+0000", 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864595+0000", 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0657e-05 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864605+0000", 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9380000000000004e-06 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864610+0000", 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0130000000000001e-06 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864660+0000", 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0006000000000002e-05 2023-10-09T06:32:06.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865068+0000", 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00040736199999999998 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865233+0000", 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000165496 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872957+0000", 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0077238020000000001 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872962+0000", 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2599999999999996e-06 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871429+0000", 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99846622200000001 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871438+0000", 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8109999999999997e-06 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034745+0000", 2023-10-09T06:32:06.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1633062599999999 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034831+0000", 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6227e-05 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67309 11.5 11:af8d0774:::benchmark_data_smithi107_418363_object67308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:32:06.307 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 281.39115918099998, 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.174921791, 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67309 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879358+0000", 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879394+0000", 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5299000000000002e-05 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879411+0000", 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7544999999999999e-05 2023-10-09T06:32:06.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879417+0000", 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.784e-06 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879464+0000", 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6631999999999997e-05 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879484+0000", 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0475999999999998e-05 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871466+0000", 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99198237899999997 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871472+0000", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6790000000000002e-06 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034876+0000", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1634033669999999 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035033+0000", 2023-10-09T06:32:06.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000157217 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035259+0000", 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022594499999999999 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041596+0000", 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0063371349999999998 2023-10-09T06:32:06.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041946+0000", 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00035007099999999998 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051163+0000", 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0092172039999999997 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051212+0000", 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8730999999999998e-05 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054281+0000", 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030695850000000001 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:152720 11.f 11:f9ba8b6d:::benchmark_data_smithi107_418363_object62284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:52.388857+0000", 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 133.881661716, 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23051629500000001, 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 152720 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.317 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388857+0000", 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388857+0000", 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388857+0000", 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388860+0000", 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1250000000000001e-06 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.319 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388866+0000", 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1929999999999998e-06 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.388870+0000", 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3100000000000002e-06 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.392648+0000", 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037774829999999999 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.392751+0000", 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010305400000000001 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.392842+0000", 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1303000000000003e-05 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.395907+0000", 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030646919999999999 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.396649+0000", 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00074194400000000002 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.619270+0000", 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22262151199999999 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.619329+0000", 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8881000000000001e-05 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:52.619373+0000", 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4314999999999998e-05 2023-10-09T06:32:06.323 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:166141 11.f 11:fc740d4e:::benchmark_data_smithi107_418363_object75705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:30:14.065830+0000", 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 112.204688699, 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20631164399999999, 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.324 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 166141 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065830+0000", 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065830+0000", 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065829+0000", 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065836+0000", 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7319999999999998e-06 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065853+0000", 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7110000000000001e-05 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065863+0000", 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6120000000000005e-06 2023-10-09T06:32:06.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065914+0000", 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0609999999999998e-05 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.066057+0000", 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014319100000000001 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.066194+0000", 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013691199999999999 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.077962+0000", 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011768664999999999 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.077974+0000", 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1636e-05 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272073+0000", 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.194098732 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272114+0000", 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1118999999999997e-05 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272142+0000", 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7838999999999999e-05 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.331 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:166143 11.f 11:f7782f57:::benchmark_data_smithi107_418363_object75707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:30:14.065879+0000", 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 112.20463978399999, 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.206350648, 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.332 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 166143 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065879+0000", 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065879+0000", 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065878+0000", 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065882+0000", 2023-10-09T06:32:06.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4400000000000001e-06 2023-10-09T06:32:06.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065893+0000", 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1228e-05 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065899+0000", 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2920000000000001e-06 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.069934+0000", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0040342470000000004 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.070040+0000", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105812 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.070146+0000", 2023-10-09T06:32:06.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106018 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.085796+0000", 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015650075999999999 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.085807+0000", 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1396000000000001e-05 2023-10-09T06:32:06.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272180+0000", 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.186372596 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272208+0000", 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8581999999999998e-05 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272230+0000", 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1508000000000001e-05 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:166146 11.f 11:f9fb799b:::benchmark_data_smithi107_418363_object75710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:30:14.065906+0000", 2023-10-09T06:32:06.340 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 112.20461314000001, 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20639036099999999, 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 166146 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065906+0000", 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065906+0000", 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065905+0000", 2023-10-09T06:32:06.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065908+0000", 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7980000000000001e-06 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065914+0000", 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1260000000000003e-06 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.065918+0000", 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0289999999999999e-06 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.073876+0000", 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0079576490000000007 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.344 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.074019+0000", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142621 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.074157+0000", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013849500000000001 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.085812+0000", 2023-10-09T06:32:06.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011655241 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.085852+0000", 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0163999999999997e-05 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272252+0000", 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.18639972899999999 2023-10-09T06:32:06.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272277+0000", 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5012999999999999e-05 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:14.272296+0000", 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8865e-05 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.347 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:180294 11.f 11:ff04e402:::benchmark_data_smithi107_418363_object89858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:30:36.826357+0000", 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 89.444161637999997, 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19857266900000001, 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:32:06.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180294 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826357+0000", 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826357+0000", 2023-10-09T06:32:06.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826356+0000", 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826363+0000", 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9070000000000007e-06 2023-10-09T06:32:06.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826377+0000", 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3828e-05 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826384+0000", 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6549999999999998e-06 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826431+0000", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6684999999999998e-05 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826546+0000", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115567 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.826659+0000", 2023-10-09T06:32:06.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011296800000000001 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.830328+0000", 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036686380000000001 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:36.831106+0000", 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077859399999999997 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:37.024867+0000", 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19376030999999999 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:37.024901+0000", 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4270999999999998e-05 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:30:37.024930+0000", 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8866000000000001e-05 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6275 12.d 12:b8b44611:::benchmark_data_smithi107_479504_object6274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 52.245126589000002, 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267782297, 2023-10-09T06:32:06.355 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6275 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025391+0000", 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025438+0000", 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7531999999999998e-05 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025454+0000", 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5452e-05 2023-10-09T06:32:06.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025461+0000", 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1380000000000001e-06 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025514+0000", 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2778999999999997e-05 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025621+0000", 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010712 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025721+0000", 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100147 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027384+0000", 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0016627790000000001 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027404+0000", 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0047999999999998e-05 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293102+0000", 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265697933 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293139+0000", 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7493000000000002e-05 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293175+0000", 2023-10-09T06:32:06.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5145999999999999e-05 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6278 12.d 12:be6c2ec3:::benchmark_data_smithi107_479504_object6277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 52.245024198999999, 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267768063, 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.363 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6278 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025494+0000", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025557+0000", 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2922000000000005e-05 2023-10-09T06:32:06.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025575+0000", 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7739000000000001e-05 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025579+0000", 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0400000000000003e-06 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026133+0000", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055439499999999995 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026261+0000", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127372 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026428+0000", 2023-10-09T06:32:06.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016770500000000001 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027443+0000", 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001015099 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027452+0000", 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6920000000000008e-06 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293213+0000", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265760789 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293242+0000", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9085e-05 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293263+0000", 2023-10-09T06:32:06.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0744999999999999e-05 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6279 12.d 12:bf6b6632:::benchmark_data_smithi107_479504_object6278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 52.244932519999999, 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26777375199999998, 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6279 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025640+0000", 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4079999999999997e-05 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025645+0000", 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0640000000000001e-06 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025648+0000", 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0460000000000001e-06 2023-10-09T06:32:06.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026860+0000", 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001212252 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026986+0000", 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012579999999999999 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027075+0000", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8715999999999996e-05 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027532+0000", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045738199999999999 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027923+0000", 2023-10-09T06:32:06.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00039128200000000001 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293289+0000", 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26536579300000002 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293328+0000", 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8287000000000001e-05 2023-10-09T06:32:06.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293360+0000", 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2518000000000003e-05 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:37240 12.d 12:bb21ad69:::benchmark_data_smithi107_479504_object37239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:32:06.377 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 34.422474508999997, 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25148841100000002, 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 37240 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848043+0000", 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848103+0000", 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9973999999999999e-05 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848112+0000", 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7090000000000008e-06 2023-10-09T06:32:06.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848117+0000", 2023-10-09T06:32:06.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3640000000000001e-06 2023-10-09T06:32:06.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848144+0000", 2023-10-09T06:32:06.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7644000000000002e-05 2023-10-09T06:32:06.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.558 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848210+0000", 2023-10-09T06:32:06.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5368000000000002e-05 2023-10-09T06:32:06.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.558 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.559 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848283+0000", 2023-10-09T06:32:06.559 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3858999999999999e-05 2023-10-09T06:32:06.559 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849143+0000", 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000859705 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097694+0000", 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24855127599999999 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099429+0000", 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001734759 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099492+0000", 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2785000000000003e-05 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099533+0000", 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0816999999999999e-05 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46340.0:37240 12.d e367/361 ondisk, result = 0)", 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 34.420750984000001, 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24792936400000001, 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849767+0000", 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849771+0000", 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.523e-06 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849788+0000", 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6719e-05 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849792+0000", 2023-10-09T06:32:06.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9400000000000004e-06 2023-10-09T06:32:06.566 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097677+0000", 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24788569099999999 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097687+0000", 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7059999999999999e-06 2023-10-09T06:32:06.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097697+0000", 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0288e-05 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46340.0:37240 12.d e367/361 ondisk, result = 0)", 2023-10-09T06:32:06.604 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.888190+0000", 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 34.382328645000001, 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.211349804, 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.888190+0000", 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.888190+0000", 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.888189+0000", 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.888204+0000", 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4790000000000001e-05 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.098820+0000", 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.21061607099999999 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.098830+0000", 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0891e-05 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099413+0000", 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00058260199999999999 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099422+0000", 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3530000000000001e-06 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099540+0000", 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117557 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:58468 12.d 12:bcd0b019:::benchmark_data_smithi107_479504_object58467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:51.328040+0000", 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.942479221999999, 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.18307844300000001, 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.610 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 58468 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328040+0000", 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328040+0000", 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.611 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328038+0000", 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328106+0000", 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8318000000000006e-05 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.612 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328123+0000", 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6472000000000001e-05 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328130+0000", 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.669e-06 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328176+0000", 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.621e-05 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328302+0000", 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012636900000000001 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.328439+0000", 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000136806 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.330544+0000", 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002105043 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.339369+0000", 2023-10-09T06:32:06.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0088245259999999992 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511047+0000", 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.171678945 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511092+0000", 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4966000000000003e-05 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.616 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511118+0000", 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5579999999999999e-05 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:58477 12.d 12:bba1e8b1:::benchmark_data_smithi107_479504_object58476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e371)", 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:51.337167+0000", 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 14.933352067, 2023-10-09T06:32:06.617 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.17404676999999999, 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 58477 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.618 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337167+0000", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337167+0000", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337166+0000", 2023-10-09T06:32:06.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337233+0000", 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7201999999999998e-05 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337248+0000", 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5031999999999999e-05 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337253+0000", 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2349999999999999e-06 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337293+0000", 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9779e-05 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.621 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337406+0000", 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000112891 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.337502+0000", 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5651999999999998e-05 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.622 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.339549+0000", 2023-10-09T06:32:06.623 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020476700000000001 2023-10-09T06:32:06.623 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.623 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.623 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.623 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.343179+0000", 2023-10-09T06:32:06.627 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003629845 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511152+0000", 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16797295900000001 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511188+0000", 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6093999999999997e-05 2023-10-09T06:32:06.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:51.511214+0000", 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5384999999999998e-05 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:69005 12.d 12:b17151a3:::benchmark_data_smithi107_479504_object69004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-10-09T06:32:06.629 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:03.090797+0000", 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 3.1797222120000002, 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15573831899999999, 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.630 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 69005 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090797+0000", 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090797+0000", 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090773+0000", 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999762 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090824+0000", 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1371e-05 2023-10-09T06:32:06.632 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090835+0000", 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1257000000000001e-05 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090840+0000", 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2039999999999996e-06 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.090879+0000", 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8486999999999999e-05 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.242135+0000", 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.151255794 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.242307+0000", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017233 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.245074+0000", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027667059999999999 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.246277+0000", 2023-10-09T06:32:06.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001202938 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.246459+0000", 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000182165 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.246498+0000", 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8856999999999997e-05 2023-10-09T06:32:06.636 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:03.246535+0000", 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7221999999999997e-05 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:70794 12.d 12:b0ee36f1:::benchmark_data_smithi107_479504_object70793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-10-09T06:32:06.637 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:05.285683+0000", 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.98483558599999999, 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.142270233, 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 70794 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.638 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285683+0000", 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285683+0000", 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:32:06.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285683+0000", 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285712+0000", 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9578e-05 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285721+0000", 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.183e-06 2023-10-09T06:32:06.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.285726+0000", 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.13e-06 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.290581+0000", 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004854545 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.290698+0000", 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117367 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.290844+0000", 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000145327 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.293566+0000", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027220780000000002 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.293571+0000", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.524e-06 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.427873+0000", 2023-10-09T06:32:06.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13430192599999999 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.427920+0000", 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6876000000000003e-05 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:05.427953+0000", 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3327999999999997e-05 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.644 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.645 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.645 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:32:06.645 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:32:06.645 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:32:06.645 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:32:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:06.647 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:06.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:06.647 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:06 smithi107 ceph-mon[212750]: pgmap v1169: 109 pgs: 2 activating, 4 peering, 2 remapped+peering, 101 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 KiB/s rd, 53 MiB/s wr, 881 op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T06:32:06.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:06 smithi150 ceph-mon[194178]: pgmap v1169: 109 pgs: 2 activating, 4 peering, 2 remapped+peering, 101 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 KiB/s rd, 53 MiB/s wr, 881 op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T06:32:06.748 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:06.776 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:06.819 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:06.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:06 smithi107 ceph-mon[220921]: pgmap v1169: 109 pgs: 2 activating, 4 peering, 2 remapped+peering, 101 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 KiB/s rd, 53 MiB/s wr, 881 op/s; 170 KiB/s, 0 objects/s recovering 2023-10-09T06:32:06.922 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:06.958 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 72231 72215 82.0307 51.25 0.015138 0.0121861 2023-10-09T06:32:06.984 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:07.120 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:07.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:07.230 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:32:07.230 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:07.266 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:07.328 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:07.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:07.334 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:07.433 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.529 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/116020527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2119636087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[212750]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:07.543 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:07.563 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 1 pg_num to 31 2023-10-09T06:32:07.597 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:32:07.782 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/116020527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2119636087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:07 smithi107 ceph-mon[220921]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:07.824 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:07.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:07.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/116020527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2119636087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-10-09T06:32:07.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:07 smithi150 ceph-mon[194178]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:32:07.931 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:07.962 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 73167 73151 81.6106 58.5 0.0209736 0.0122479 2023-10-09T06:32:08.002 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: pgmap v1170: 109 pgs: 2 activating, 4 peering, 103 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4.7 KiB/s rd, 43 MiB/s wr, 696 op/s; 50 KiB/s, 0 objects/s recovering 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: 1.11 scrub starts 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[220921]: 1.11 scrub ok 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: pgmap v1170: 109 pgs: 2 activating, 4 peering, 103 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4.7 KiB/s rd, 43 MiB/s wr, 696 op/s; 50 KiB/s, 0 objects/s recovering 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:08.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:32:08.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:08 smithi107 ceph-mon[212750]: 1.11 scrub ok 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: pgmap v1170: 109 pgs: 2 activating, 4 peering, 103 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 4.7 KiB/s rd, 43 MiB/s wr, 696 op/s; 50 KiB/s, 0 objects/s recovering 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: osdmap e380: 8 total, 8 up, 8 in 2023-10-09T06:32:08.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: failed to encode map e380 with expected crc 2023-10-09T06:32:08.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: 1.11 scrub starts 2023-10-09T06:32:08.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:08 smithi150 ceph-mon[194178]: 1.11 scrub ok 2023-10-09T06:32:08.961 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 73896 73880 80.9752 45.5625 0.022908 0.0123438 2023-10-09T06:32:09.793 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:09.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:09.801 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:32:09.801 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T06:32:09.801 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:09.802 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[220921]: 1.5 scrub starts 2023-10-09T06:32:09.802 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[220921]: 1.5 scrub ok 2023-10-09T06:32:09.802 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:09.901 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:09 smithi150 ceph-mon[194178]: 1.5 scrub starts 2023-10-09T06:32:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:09 smithi150 ceph-mon[194178]: 1.5 scrub ok 2023-10-09T06:32:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:09 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:09.948 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:09.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 74989 74973 80.7544 68.3125 0.0108636 0.0123785 2023-10-09T06:32:10.018 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:10.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:10.127 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:10.192 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:10.518 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:10.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:10.626 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:10.657 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:10.658 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:10.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:10.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:10.819 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: pgmap v1172: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 4.0 KiB/s rd, 55 MiB/s wr, 879 op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:32:10.859 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:10 smithi150 ceph-mon[194178]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.861 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:10.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: pgmap v1172: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 4.0 KiB/s rd, 55 MiB/s wr, 879 op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[212750]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: pgmap v1172: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 4.0 KiB/s rd, 55 MiB/s wr, 879 op/s; 43 KiB/s, 0 objects/s recovering 2023-10-09T06:32:10.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T06:32:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-10-09T06:32:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: osdmap e381: 8 total, 8 up, 8 in 2023-10-09T06:32:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:10 smithi107 ceph-mon[220921]: failed to encode map e381 with expected crc 2023-10-09T06:32:10.926 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:10.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:10.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 76023 76007 80.481 64.625 0.01952 0.0124208 2023-10-09T06:32:11.032 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:11.118 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: 1.2 scrub starts 2023-10-09T06:32:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: 1.2 scrub ok 2023-10-09T06:32:11.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[220921]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: 1.2 scrub starts 2023-10-09T06:32:11.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: 1.2 scrub ok 2023-10-09T06:32:11.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:32:11.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:11 smithi107 ceph-mon[212750]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.689 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:11.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:11.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:11.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:32:11.829 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:11.870 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:11.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: 1.2 scrub starts 2023-10-09T06:32:11.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: 1.2 scrub ok 2023-10-09T06:32:11.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: osdmap e382: 8 total, 8 up, 8 in 2023-10-09T06:32:11.873 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.874 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:11 smithi150 ceph-mon[194178]: failed to encode map e382 with expected crc 2023-10-09T06:32:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:11.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:11.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:32:11.963855+0000 min lat: 0.00111887 max lat: 0.351321 avg lat: 0.0124671 2023-10-09T06:32:11.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:32:11.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 76854 76838 80.0054 51.9375 0.218778 0.0124671 2023-10-09T06:32:11.976 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:12.009 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:12.062 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:12.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:12.171 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:12.247 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:12.332 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:12] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:32:12.416 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:12.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:12.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:12.520 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:12.603 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:12.695 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:32:12.738 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:12.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:12.753 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:32:12.763 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:12.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:32:12.839 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:32:12.854 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:12.878 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:32:12.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.0 scrub starts 2023-10-09T06:32:12.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.0 scrub ok 2023-10-09T06:32:12.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.4 scrub starts 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.4 scrub ok 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: pgmap v1175: 119 pgs: 8 unknown, 111 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.0 scrub starts 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.0 scrub ok 2023-10-09T06:32:12.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.4 scrub starts 2023-10-09T06:32:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.4 scrub ok 2023-10-09T06:32:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: pgmap v1175: 119 pgs: 8 unknown, 111 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:32:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.1 scrub starts 2023-10-09T06:32:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: 1.1 scrub ok 2023-10-09T06:32:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:12 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.0 scrub starts 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.0 scrub ok 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.4 scrub starts 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.4 scrub ok 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: pgmap v1175: 119 pgs: 8 unknown, 111 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.1 scrub starts 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: 1.1 scrub ok 2023-10-09T06:32:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:12 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:12.920 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:32:12.956 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:12.960 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:32:12.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 77717 77701 79.5781 53.9375 0.00562868 0.0125589 2023-10-09T06:32:13.007 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.009 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:32:13.232 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:32:13.232 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:32:13.771 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:13.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:13.880 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[212750]: 1.c scrub starts 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[212750]: 1.c scrub ok 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2207921688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[212750]: 1.14 scrub ok 2023-10-09T06:32:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[220921]: 1.c scrub starts 2023-10-09T06:32:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[220921]: 1.c scrub ok 2023-10-09T06:32:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2207921688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[220921]: 1.14 scrub starts 2023-10-09T06:32:13.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:13 smithi107 ceph-mon[220921]: 1.14 scrub ok 2023-10-09T06:32:13.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:13 smithi150 ceph-mon[194178]: 1.c scrub starts 2023-10-09T06:32:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:13 smithi150 ceph-mon[194178]: 1.c scrub ok 2023-10-09T06:32:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2207921688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:13 smithi150 ceph-mon[194178]: 1.14 scrub starts 2023-10-09T06:32:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:13 smithi150 ceph-mon[194178]: 1.14 scrub ok 2023-10-09T06:32:13.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:13.949 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:13.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:13.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:13.963 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 78649 78633 79.234 58.25 0.100924 0.012606 2023-10-09T06:32:14.055 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:14.093 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:14.167 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:14.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:14.276 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:14.311 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:14.381 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:14.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:14.489 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:14.541 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:32:14.541 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:32:14.542 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2023-10-09T06:32:14.542 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:32:14.542 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:14.616 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:14.724 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.1c scrub starts 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.1c scrub ok 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: pgmap v1176: 119 pgs: 4 unknown, 115 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.f scrub starts 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.f scrub ok 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.12 scrub starts 2023-10-09T06:32:14.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[212750]: 1.12 scrub ok 2023-10-09T06:32:14.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.1c scrub starts 2023-10-09T06:32:14.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.1c scrub ok 2023-10-09T06:32:14.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: pgmap v1176: 119 pgs: 4 unknown, 115 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:32:14.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.f scrub starts 2023-10-09T06:32:14.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.f scrub ok 2023-10-09T06:32:14.730 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.12 scrub starts 2023-10-09T06:32:14.730 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:14 smithi107 ceph-mon[220921]: 1.12 scrub ok 2023-10-09T06:32:14.753 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:14.821 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:14.824 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.1c scrub starts 2023-10-09T06:32:14.824 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.1c scrub ok 2023-10-09T06:32:14.824 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: pgmap v1176: 119 pgs: 4 unknown, 115 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-09T06:32:14.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.f scrub starts 2023-10-09T06:32:14.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.f scrub ok 2023-10-09T06:32:14.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.12 scrub starts 2023-10-09T06:32:14.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:14 smithi150 ceph-mon[194178]: 1.12 scrub ok 2023-10-09T06:32:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:14.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:14.929 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:14.965 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 79685 79669 79.0041 64.75 0.00834032 0.0126522 2023-10-09T06:32:15.016 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:15.529 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:32:15.688 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:15.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:15.693 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:15 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:15.693 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:15 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:15.794 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:15 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-10-09T06:32:15.914 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:15.967 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 80454 80438 78.5159 48.0625 0.0247621 0.0127299 2023-10-09T06:32:15.975 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:16.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:16.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:16.117 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:16.202 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:16.430 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:16.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:16.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:16.538 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:16.648 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:16.753 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:32:16.756 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.3 scrub starts 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.3 scrub ok 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: pgmap v1177: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.9 scrub starts 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.9 scrub ok 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.10 scrub ok 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: 1.e scrub starts 2023-10-09T06:32:16.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.3 scrub starts 2023-10-09T06:32:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.3 scrub ok 2023-10-09T06:32:16.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: pgmap v1177: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.9 scrub starts 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.9 scrub ok 2023-10-09T06:32:16.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.7 scrub starts 2023-10-09T06:32:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.7 scrub ok 2023-10-09T06:32:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:32:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.10 scrub ok 2023-10-09T06:32:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: 1.e scrub starts 2023-10-09T06:32:16.909 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.3 scrub starts 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.3 scrub ok 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: pgmap v1177: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: osdmap e383: 8 total, 8 up, 8 in 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.910 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: failed to encode map e383 with expected crc 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.9 scrub starts 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.9 scrub ok 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.7 scrub starts 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.7 scrub ok 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.10 scrub ok 2023-10-09T06:32:16.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: 1.e scrub starts 2023-10-09T06:32:16.912 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:16.937 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:16.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:16.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 81240 81224 78.0627 49.125 0.0160441 0.0128035 2023-10-09T06:32:17.048 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:17.073 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:17.582 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:17.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:17.694 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:17.706 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:32:17.737 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:17.799 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:17.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:17.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: 1.e scrub ok 2023-10-09T06:32:17.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:32:17.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: 1.d scrub starts 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: 1.d scrub ok 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: 1.6 scrub starts 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: 1.6 scrub ok 2023-10-09T06:32:17.859 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: 1.e scrub ok 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: 1.d scrub starts 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: 1.d scrub ok 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T06:32:17.861 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:17.909 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: 1.e scrub ok 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: osdmap e384: 8 total, 8 up, 8 in 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.948 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: failed to encode map e384 with expected crc 2023-10-09T06:32:17.949 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: 1.d scrub starts 2023-10-09T06:32:17.949 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: 1.d scrub ok 2023-10-09T06:32:17.949 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: 1.6 scrub starts 2023-10-09T06:32:17.949 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: 1.6 scrub ok 2023-10-09T06:32:17.949 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:17.951 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:17.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 81911 81895 77.5155 41.9375 0.00891183 0.0128958 2023-10-09T06:32:18.011 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:18.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:18.118 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:18.154 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:18.228 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:18.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:18.337 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:18.414 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:18.629 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:18.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:18.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: 1.5 scrub starts 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: 1.5 scrub ok 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: pgmap v1180: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.684 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.685 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[212750]: failed to encode map e385 with expected crc 2023-10-09T06:32:18.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:18.807 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:18.968 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 82852 82836 77.2364 58.8125 0.0628659 0.0129359 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: 1.5 scrub starts 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: 1.5 scrub ok 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: pgmap v1180: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:19.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:32:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:18 smithi107 ceph-mon[220921]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: 1.5 scrub starts 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: 1.5 scrub ok 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: pgmap v1180: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/38807898' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: osdmap e385: 8 total, 8 up, 8 in 2023-10-09T06:32:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:18 smithi150 ceph-mon[194178]: failed to encode map e385 with expected crc 2023-10-09T06:32:19.494 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:19.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:19.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:19.514 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:19.601 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:19.683 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:19.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[212750]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[212750]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[212750]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[212750]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[220921]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[220921]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[220921]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.843 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:19 smithi107 ceph-mon[220921]: failed to encode map e386 with expected crc 2023-10-09T06:32:19.931 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:19.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:19.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 83607 83591 76.7929 47.1875 0.0110228 0.0130172 2023-10-09T06:32:20.038 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:20.077 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:19 smithi150 ceph-mon[194178]: failed to encode map e386 with expected crc 2023-10-09T06:32:20.077 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:19 smithi150 ceph-mon[194178]: osdmap e386: 8 total, 8 up, 8 in 2023-10-09T06:32:20.077 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:19 smithi150 ceph-mon[194178]: failed to encode map e386 with expected crc 2023-10-09T06:32:20.077 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:19 smithi150 ceph-mon[194178]: failed to encode map e386 with expected crc 2023-10-09T06:32:20.080 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:20.140 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:20.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:20.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:20.249 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:20.285 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:20.331 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:32:20.342 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:32:20.350 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:20.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:20.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:20.445 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:32:20.446 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:32:20.458 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:20.520 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:20.575 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:20.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:20.678 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:20.721 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:20.798 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:20.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:20.907 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:20.934 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:20.970 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 84407 84391 76.4046 50 0.0123728 0.0130668 2023-10-09T06:32:20.975 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:20.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: 1.13 deep-scrub starts 2023-10-09T06:32:20.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: 1.13 deep-scrub ok 2023-10-09T06:32:20.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: pgmap v1183: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:32:20.981 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.982 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:20 smithi150 ceph-mon[194178]: failed to encode map e387 with expected crc 2023-10-09T06:32:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:20.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:21.084 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: 1.13 deep-scrub starts 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: 1.13 deep-scrub ok 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: pgmap v1183: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[212750]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: 1.13 deep-scrub starts 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: 1.13 deep-scrub ok 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: pgmap v1183: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: osdmap e387: 8 total, 8 up, 8 in 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:20 smithi107 ceph-mon[220921]: failed to encode map e387 with expected crc 2023-10-09T06:32:21.172 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:21.251 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:32:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[212750]: 1.b deep-scrub starts 2023-10-09T06:32:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[212750]: 1.b deep-scrub ok 2023-10-09T06:32:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[212750]: 1.1e scrub starts 2023-10-09T06:32:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[212750]: 1.1e scrub ok 2023-10-09T06:32:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[212750]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[220921]: 1.b deep-scrub starts 2023-10-09T06:32:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[220921]: 1.b deep-scrub ok 2023-10-09T06:32:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[220921]: 1.1e scrub starts 2023-10-09T06:32:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[220921]: 1.1e scrub ok 2023-10-09T06:32:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:21 smithi107 ceph-mon[220921]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:21.973 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 85213 85197 76.0303 50.375 0.0207565 0.0131478 2023-10-09T06:32:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:21 smithi150 ceph-mon[194178]: 1.b deep-scrub starts 2023-10-09T06:32:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:21 smithi150 ceph-mon[194178]: 1.b deep-scrub ok 2023-10-09T06:32:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:21 smithi150 ceph-mon[194178]: 1.1e scrub starts 2023-10-09T06:32:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:21 smithi150 ceph-mon[194178]: 1.1e scrub ok 2023-10-09T06:32:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:21 smithi150 ceph-mon[194178]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:32:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:22] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T06:32:22.523 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:22.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:22.566 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:32:22.649 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:22.672 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1184,"stamp":"2023-10-09T06:32:19.719470+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5319041376,"num_objects":81177,"num_object_clones":0,"num_object_copies":243531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81177,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":81219,"num_write_kb":5193812,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":821106,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86198,"ondisk_log_size":86198,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22386944,"kb_used_data":16077568,"kb_used_omap":20,"kb_used_meta":6309291,"kb_avail":727574272,"statfs":{"total":767960285184,"available":745036054528,"internally_reserved":0,"allocated":16463429632,"data_stored":16452563901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6460714638},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,2,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":286495006,"num_objects":4372,"num_object_clones":0,"num_object_copies":13116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4372,"num_whiteouts":0,"num_read":-78,"num_read_kb":-146,"num_write":4335,"num_write_kb":279487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-206924,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.073212"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":721,"reported_epoch":385,"state":"active+undersized+remapped","last_fresh":"2023-10-09T06:32:17.713818+0000","last_change":"2023-10-09T06:32:17.714010+0000","last_active":"2023-10-09T06:32:17.713818+0000","last_peered":"2023-10-09T06:32:17.713818+0000","last_clean":"2023-10-09T06:32:10.615869+0000","last_became_active":"2023-10-09T06:32:17.708382+0000","last_became_peered":"2023-10-09T06:32:17.708382+0000","last_unstale":"2023-10-09T06:32:17.713818+0000","last_undegraded":"2023-10-09T06:32:17.713818+0000","last_fullsized":"2023-10-09T06:32:17.691670+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","2","3","7"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":364,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671312+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:32:16.671312+0000","last_peered":"2023-10-09T06:32:16.671312+0000","last_clean":"2023-10-09T06:32:16.671312+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:32:16.671312+0000","last_undegraded":"2023-10-09T06:32:16.671312+0000","last_fullsized":"2023-10-09T06:32:16.671312+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.686543+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:32:16.686543+0000","last_peered":"2023-10-09T06:32:16.686543+0000","last_clean":"2023-10-09T06:32:16.686543+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:32:16.686543+0000","last_undegraded":"2023-10-09T06:32:16.686543+0000","last_fullsized":"2023-10-09T06:32:16.686543+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":687,"reported_epoch":385,"state":"active+undersized+remapped","last_fresh":"2023-10-09T06:32:17.710021+0000","last_change":"2023-10-09T06:32:17.709947+0000","last_active":"2023-10-09T06:32:17.710021+0000","last_peered":"2023-10-09T06:32:17.710021+0000","last_clean":"2023-10-09T06:31:57.498736+0000","last_became_active":"2023-10-09T06:32:17.709947+0000","last_became_peered":"2023-10-09T06:32:17.709947+0000","last_unstale":"2023-10-09T06:32:17.710021+0000","last_undegraded":"2023-10-09T06:32:17.710021+0000","last_fullsized":"2023-10-09T06:32:17.696896+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,0],"avail_no_missing":["5","0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":376,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.647948+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:32:15.647948+0000","last_peered":"2023-10-09T06:32:15.647948+0000","last_clean":"2023-10-09T06:32:15.647948+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:32:15.647948+0000","last_undegraded":"2023-10-09T06:32:15.647948+0000","last_fullsized":"2023-10-09T06:32:15.647948+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":380,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.658501+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:32:15.658501+0000","last_peered":"2023-10-09T06:32:15.658501+0000","last_clean":"2023-10-09T06:32:15.658501+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:32:15.658501+0000","last_undegraded":"2023-10-09T06:32:15.658501+0000","last_fullsized":"2023-10-09T06:32:15.658501+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":681,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.609904+0000","last_change":"2023-10-09T06:32:10.609904+0000","last_active":"2023-10-09T06:32:10.609904+0000","last_peered":"2023-10-09T06:32:10.609904+0000","last_clean":"2023-10-09T06:32:10.609904+0000","last_became_active":"2023-10-09T06:32:10.609574+0000","last_became_peered":"2023-10-09T06:32:10.609574+0000","last_unstale":"2023-10-09T06:32:10.609904+0000","last_undegraded":"2023-10-09T06:32:10.609904+0000","last_fullsized":"2023-10-09T06:32:10.609904+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688645+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:32:16.688645+0000","last_peered":"2023-10-09T06:32:16.688645+0000","last_clean":"2023-10-09T06:32:16.688645+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:32:16.688645+0000","last_undegraded":"2023-10-09T06:32:16.688645+0000","last_fullsized":"2023-10-09T06:32:16.688645+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":372,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.599831+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:32:10.599831+0000","last_peered":"2023-10-09T06:32:10.599831+0000","last_clean":"2023-10-09T06:32:10.599831+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:32:10.599831+0000","last_undegraded":"2023-10-09T06:32:10.599831+0000","last_fullsized":"2023-10-09T06:32:10.599831+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":694,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.610585+0000","last_change":"2023-10-09T06:32:10.610585+0000","last_active":"2023-10-09T06:32:10.610585+0000","last_peered":"2023-10-09T06:32:10.610585+0000","last_clean":"2023-10-09T06:32:10.610585+0000","last_became_active":"2023-10-09T06:32:10.610277+0000","last_became_peered":"2023-10-09T06:32:10.610277+0000","last_unstale":"2023-10-09T06:32:10.610585+0000","last_undegraded":"2023-10-09T06:32:10.610585+0000","last_fullsized":"2023-10-09T06:32:10.610585+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.376885+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.376885+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017910280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":55,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583904+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:32:10.583904+0000","last_peered":"2023-10-09T06:32:10.583904+0000","last_clean":"2023-10-09T06:32:10.583904+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:32:10.583904+0000","last_undegraded":"2023-10-09T06:32:10.583904+0000","last_fullsized":"2023-10-09T06:32:10.583904+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":84,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.696552+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:32:17.696552+0000","last_peered":"2023-10-09T06:32:17.696552+0000","last_clean":"2023-10-09T06:32:17.696552+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:32:17.696552+0000","last_undegraded":"2023-10-09T06:32:17.696552+0000","last_fullsized":"2023-10-09T06:32:17.696552+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":530,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677003+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:32:16.677003+0000","last_peered":"2023-10-09T06:32:16.677003+0000","last_clean":"2023-10-09T06:32:16.677003+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:32:16.677003+0000","last_undegraded":"2023-10-09T06:32:16.677003+0000","last_fullsized":"2023-10-09T06:32:16.677003+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678471+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:32:16.678471+0000","last_peered":"2023-10-09T06:32:16.678471+0000","last_clean":"2023-10-09T06:32:16.678471+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:32:16.678471+0000","last_undegraded":"2023-10-09T06:32:16.678471+0000","last_fullsized":"2023-10-09T06:32:16.678471+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.697893+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:32:17.697893+0000","last_peered":"2023-10-09T06:32:17.697893+0000","last_clean":"2023-10-09T06:32:17.697893+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:32:17.697893+0000","last_undegraded":"2023-10-09T06:32:17.697893+0000","last_fullsized":"2023-10-09T06:32:17.697893+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":571,"reported_epoch":385,"state":"active+undersized+remapped","last_fresh":"2023-10-09T06:32:17.714884+0000","last_change":"2023-10-09T06:32:17.714603+0000","last_active":"2023-10-09T06:32:17.714884+0000","last_peered":"2023-10-09T06:32:17.714884+0000","last_clean":"2023-10-09T06:27:33.205763+0000","last_became_active":"2023-10-09T06:32:17.714603+0000","last_became_peered":"2023-10-09T06:32:17.714603+0000","last_unstale":"2023-10-09T06:32:17.714884+0000","last_undegraded":"2023-10-09T06:32:17.714884+0000","last_fullsized":"2023-10-09T06:32:17.696030+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[3,6],"avail_no_missing":["3","0","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":266,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677914+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:32:16.677914+0000","last_peered":"2023-10-09T06:32:16.677914+0000","last_clean":"2023-10-09T06:32:16.677914+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:32:16.677914+0000","last_undegraded":"2023-10-09T06:32:16.677914+0000","last_fullsized":"2023-10-09T06:32:16.677914+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":318,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648193+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:32:15.648193+0000","last_peered":"2023-10-09T06:32:15.648193+0000","last_clean":"2023-10-09T06:32:15.648193+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:32:15.648193+0000","last_undegraded":"2023-10-09T06:32:15.648193+0000","last_fullsized":"2023-10-09T06:32:15.648193+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":693,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.604393+0000","last_change":"2023-10-09T06:32:10.604393+0000","last_active":"2023-10-09T06:32:10.604393+0000","last_peered":"2023-10-09T06:32:10.604393+0000","last_clean":"2023-10-09T06:32:10.604393+0000","last_became_active":"2023-10-09T06:32:10.603921+0000","last_became_peered":"2023-10-09T06:32:10.603921+0000","last_unstale":"2023-10-09T06:32:10.604393+0000","last_undegraded":"2023-10-09T06:32:10.604393+0000","last_fullsized":"2023-10-09T06:32:10.604393+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":391,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671552+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:32:16.671552+0000","last_peered":"2023-10-09T06:32:16.671552+0000","last_clean":"2023-10-09T06:32:16.671552+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:32:16.671552+0000","last_undegraded":"2023-10-09T06:32:16.671552+0000","last_fullsized":"2023-10-09T06:32:16.671552+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":355,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671488+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:32:16.671488+0000","last_peered":"2023-10-09T06:32:16.671488+0000","last_clean":"2023-10-09T06:32:16.671488+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:32:16.671488+0000","last_undegraded":"2023-10-09T06:32:16.671488+0000","last_fullsized":"2023-10-09T06:32:16.671488+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":653,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583012+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:32:10.583012+0000","last_peered":"2023-10-09T06:32:10.583012+0000","last_clean":"2023-10-09T06:32:10.583012+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:32:10.583012+0000","last_undegraded":"2023-10-09T06:32:10.583012+0000","last_fullsized":"2023-10-09T06:32:10.583012+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":234,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676350+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:32:16.676350+0000","last_peered":"2023-10-09T06:32:16.676350+0000","last_clean":"2023-10-09T06:32:16.676350+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:32:16.676350+0000","last_undegraded":"2023-10-09T06:32:16.676350+0000","last_fullsized":"2023-10-09T06:32:16.676350+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688545+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:32:16.688545+0000","last_peered":"2023-10-09T06:32:16.688545+0000","last_clean":"2023-10-09T06:32:16.688545+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:32:16.688545+0000","last_undegraded":"2023-10-09T06:32:16.688545+0000","last_fullsized":"2023-10-09T06:32:16.688545+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":499,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678118+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:32:16.678118+0000","last_peered":"2023-10-09T06:32:16.678118+0000","last_clean":"2023-10-09T06:32:16.678118+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:32:16.678118+0000","last_undegraded":"2023-10-09T06:32:16.678118+0000","last_fullsized":"2023-10-09T06:32:16.678118+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":378,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583492+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:32:10.583492+0000","last_peered":"2023-10-09T06:32:10.583492+0000","last_clean":"2023-10-09T06:32:10.583492+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:32:10.583492+0000","last_undegraded":"2023-10-09T06:32:10.583492+0000","last_fullsized":"2023-10-09T06:32:10.583492+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677611+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:32:16.677611+0000","last_peered":"2023-10-09T06:32:16.677611+0000","last_clean":"2023-10-09T06:32:16.677611+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:32:16.677611+0000","last_undegraded":"2023-10-09T06:32:16.677611+0000","last_fullsized":"2023-10-09T06:32:16.677611+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":696,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648063+0000","last_change":"2023-10-09T06:32:03.479311+0000","last_active":"2023-10-09T06:32:15.648063+0000","last_peered":"2023-10-09T06:32:15.648063+0000","last_clean":"2023-10-09T06:32:15.648063+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:32:15.648063+0000","last_undegraded":"2023-10-09T06:32:15.648063+0000","last_fullsized":"2023-10-09T06:32:15.648063+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":228,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676364+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:32:16.676364+0000","last_peered":"2023-10-09T06:32:16.676364+0000","last_clean":"2023-10-09T06:32:16.676364+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:32:16.676364+0000","last_undegraded":"2023-10-09T06:32:16.676364+0000","last_fullsized":"2023-10-09T06:32:16.676364+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":55,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.584058+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:32:10.584058+0000","last_peered":"2023-10-09T06:32:10.584058+0000","last_clean":"2023-10-09T06:32:10.584058+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:32:10.584058+0000","last_undegraded":"2023-10-09T06:32:10.584058+0000","last_fullsized":"2023-10-09T06:32:10.584058+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":499,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678122+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:32:16.678122+0000","last_peered":"2023-10-09T06:32:16.678122+0000","last_clean":"2023-10-09T06:32:16.678122+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:32:16.678122+0000","last_undegraded":"2023-10-09T06:32:16.678122+0000","last_fullsized":"2023-10-09T06:32:16.678122+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":461,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.599187+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:32:10.599187+0000","last_peered":"2023-10-09T06:32:10.599187+0000","last_clean":"2023-10-09T06:32:10.599187+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:32:10.599187+0000","last_undegraded":"2023-10-09T06:32:10.599187+0000","last_fullsized":"2023-10-09T06:32:10.599187+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.694168+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:32:16.694168+0000","last_peered":"2023-10-09T06:32:16.694168+0000","last_clean":"2023-10-09T06:32:16.694168+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:32:16.694168+0000","last_undegraded":"2023-10-09T06:32:16.694168+0000","last_fullsized":"2023-10-09T06:32:16.694168+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":512,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.690466+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:32:16.690466+0000","last_peered":"2023-10-09T06:32:16.690466+0000","last_clean":"2023-10-09T06:32:16.690466+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:32:16.690466+0000","last_undegraded":"2023-10-09T06:32:16.690466+0000","last_fullsized":"2023-10-09T06:32:16.690466+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"385'5126","reported_seq":5158,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:18.224528+0000","last_change":"2023-10-09T06:31:03.215492+0000","last_active":"2023-10-09T06:32:18.224528+0000","last_peered":"2023-10-09T06:32:18.224528+0000","last_clean":"2023-10-09T06:32:18.224528+0000","last_became_active":"2023-10-09T06:31:03.215135+0000","last_became_peered":"2023-10-09T06:31:03.215135+0000","last_unstale":"2023-10-09T06:32:18.224528+0000","last_undegraded":"2023-10-09T06:32:18.224528+0000","last_fullsized":"2023-10-09T06:32:18.224528+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:15.490526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335937536,"num_objects":5126,"num_object_clones":0,"num_object_copies":15378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5126,"num_write_kb":328064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":270,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678066+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:32:16.678066+0000","last_peered":"2023-10-09T06:32:16.678066+0000","last_clean":"2023-10-09T06:32:16.678066+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:32:16.678066+0000","last_undegraded":"2023-10-09T06:32:16.678066+0000","last_fullsized":"2023-10-09T06:32:16.678066+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678304+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:32:16.678304+0000","last_peered":"2023-10-09T06:32:16.678304+0000","last_clean":"2023-10-09T06:32:16.678304+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:32:16.678304+0000","last_undegraded":"2023-10-09T06:32:16.678304+0000","last_fullsized":"2023-10-09T06:32:16.678304+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":768,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688085+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:32:16.688085+0000","last_peered":"2023-10-09T06:32:16.688085+0000","last_clean":"2023-10-09T06:32:16.688085+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:32:16.688085+0000","last_undegraded":"2023-10-09T06:32:16.688085+0000","last_fullsized":"2023-10-09T06:32:16.688085+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"383'5064","reported_seq":5094,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.303844+0000","last_change":"2023-10-09T06:31:02.763465+0000","last_active":"2023-10-09T06:32:16.303844+0000","last_peered":"2023-10-09T06:32:16.303844+0000","last_clean":"2023-10-09T06:32:16.303844+0000","last_became_active":"2023-10-09T06:31:02.763153+0000","last_became_peered":"2023-10-09T06:31:02.763153+0000","last_unstale":"2023-10-09T06:32:16.303844+0000","last_undegraded":"2023-10-09T06:32:16.303844+0000","last_fullsized":"2023-10-09T06:32:16.303844+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:01.874473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331874304,"num_objects":5064,"num_object_clones":0,"num_object_copies":15192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5064,"num_write_kb":324096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":232,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.679880+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:32:16.679880+0000","last_peered":"2023-10-09T06:32:16.679880+0000","last_clean":"2023-10-09T06:32:16.679880+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:32:16.679880+0000","last_undegraded":"2023-10-09T06:32:16.679880+0000","last_fullsized":"2023-10-09T06:32:16.679880+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":382,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671211+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:32:16.671211+0000","last_peered":"2023-10-09T06:32:16.671211+0000","last_clean":"2023-10-09T06:32:16.671211+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:32:16.671211+0000","last_undegraded":"2023-10-09T06:32:16.671211+0000","last_fullsized":"2023-10-09T06:32:16.671211+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":684,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:13.928782+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:32:13.928782+0000","last_peered":"2023-10-09T06:32:13.928782+0000","last_clean":"2023-10-09T06:32:13.928782+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:32:13.928782+0000","last_undegraded":"2023-10-09T06:32:13.928782+0000","last_fullsized":"2023-10-09T06:32:13.928782+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"385'5178","reported_seq":5210,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:18.202998+0000","last_change":"2023-10-09T06:31:02.762020+0000","last_active":"2023-10-09T06:32:18.202998+0000","last_peered":"2023-10-09T06:32:18.202998+0000","last_clean":"2023-10-09T06:32:18.202998+0000","last_became_active":"2023-10-09T06:31:02.761650+0000","last_became_peered":"2023-10-09T06:31:02.761650+0000","last_unstale":"2023-10-09T06:32:18.202998+0000","last_undegraded":"2023-10-09T06:32:18.202998+0000","last_fullsized":"2023-10-09T06:32:18.202998+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:06.600297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339345408,"num_objects":5178,"num_object_clones":0,"num_object_copies":15534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5178,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":288,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677731+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:32:16.677731+0000","last_peered":"2023-10-09T06:32:16.677731+0000","last_clean":"2023-10-09T06:32:16.677731+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:32:16.677731+0000","last_undegraded":"2023-10-09T06:32:16.677731+0000","last_fullsized":"2023-10-09T06:32:16.677731+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":359,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.598362+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:32:10.598362+0000","last_peered":"2023-10-09T06:32:10.598362+0000","last_clean":"2023-10-09T06:32:10.598362+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:32:10.598362+0000","last_undegraded":"2023-10-09T06:32:10.598362+0000","last_fullsized":"2023-10-09T06:32:10.598362+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":509,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.672170+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:32:16.672170+0000","last_peered":"2023-10-09T06:32:16.672170+0000","last_clean":"2023-10-09T06:32:16.672170+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:32:16.672170+0000","last_undegraded":"2023-10-09T06:32:16.672170+0000","last_fullsized":"2023-10-09T06:32:16.672170+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"383'4997","reported_seq":5028,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.696354+0000","last_change":"2023-10-09T06:31:02.761953+0000","last_active":"2023-10-09T06:32:15.696354+0000","last_peered":"2023-10-09T06:32:15.696354+0000","last_clean":"2023-10-09T06:32:15.696354+0000","last_became_active":"2023-10-09T06:31:02.761625+0000","last_became_peered":"2023-10-09T06:31:02.761625+0000","last_unstale":"2023-10-09T06:32:15.696354+0000","last_undegraded":"2023-10-09T06:32:15.696354+0000","last_fullsized":"2023-10-09T06:32:15.696354+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4997,"log_dups_size":0,"ondisk_log_size":4997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:05:31.403447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327483392,"num_objects":4997,"num_object_clones":0,"num_object_copies":14991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4997,"num_write_kb":319808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677824+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:32:16.677824+0000","last_peered":"2023-10-09T06:32:16.677824+0000","last_clean":"2023-10-09T06:32:16.677824+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:32:16.677824+0000","last_undegraded":"2023-10-09T06:32:16.677824+0000","last_fullsized":"2023-10-09T06:32:16.677824+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.584550+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:32:10.584550+0000","last_peered":"2023-10-09T06:32:10.584550+0000","last_clean":"2023-10-09T06:32:10.584550+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:32:10.584550+0000","last_undegraded":"2023-10-09T06:32:10.584550+0000","last_fullsized":"2023-10-09T06:32:10.584550+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"383'5015","reported_seq":5045,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.302531+0000","last_change":"2023-10-09T06:31:02.762571+0000","last_active":"2023-10-09T06:32:16.302531+0000","last_peered":"2023-10-09T06:32:16.302531+0000","last_clean":"2023-10-09T06:32:16.302531+0000","last_became_active":"2023-10-09T06:31:02.762287+0000","last_became_peered":"2023-10-09T06:31:02.762287+0000","last_unstale":"2023-10-09T06:32:16.302531+0000","last_undegraded":"2023-10-09T06:32:16.302531+0000","last_fullsized":"2023-10-09T06:32:16.302531+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5015,"log_dups_size":0,"ondisk_log_size":5015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:15.365778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328663040,"num_objects":5015,"num_object_clones":0,"num_object_copies":15045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5015,"num_write_kb":320960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":737,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676216+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:32:16.676216+0000","last_peered":"2023-10-09T06:32:16.676216+0000","last_clean":"2023-10-09T06:32:16.676216+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:32:16.676216+0000","last_undegraded":"2023-10-09T06:32:16.676216+0000","last_fullsized":"2023-10-09T06:32:16.676216+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.690639+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:32:16.690639+0000","last_peered":"2023-10-09T06:32:16.690639+0000","last_clean":"2023-10-09T06:32:16.690639+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:32:16.690639+0000","last_undegraded":"2023-10-09T06:32:16.690639+0000","last_fullsized":"2023-10-09T06:32:16.690639+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":365,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671383+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:32:16.671383+0000","last_peered":"2023-10-09T06:32:16.671383+0000","last_clean":"2023-10-09T06:32:16.671383+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:32:16.671383+0000","last_undegraded":"2023-10-09T06:32:16.671383+0000","last_fullsized":"2023-10-09T06:32:16.671383+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"383'5057","reported_seq":5087,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.291497+0000","last_change":"2023-10-09T06:31:03.217042+0000","last_active":"2023-10-09T06:32:16.291497+0000","last_peered":"2023-10-09T06:32:16.291497+0000","last_clean":"2023-10-09T06:32:16.291497+0000","last_became_active":"2023-10-09T06:31:03.216729+0000","last_became_peered":"2023-10-09T06:31:03.216729+0000","last_unstale":"2023-10-09T06:32:16.291497+0000","last_undegraded":"2023-10-09T06:32:16.291497+0000","last_fullsized":"2023-10-09T06:32:16.291497+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:37.232295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331415552,"num_objects":5057,"num_object_clones":0,"num_object_copies":15171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5057,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":707,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678204+0000","last_change":"2023-10-09T06:32:10.599948+0000","last_active":"2023-10-09T06:32:16.678204+0000","last_peered":"2023-10-09T06:32:16.678204+0000","last_clean":"2023-10-09T06:32:16.678204+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:32:16.678204+0000","last_undegraded":"2023-10-09T06:32:16.678204+0000","last_fullsized":"2023-10-09T06:32:16.678204+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":340,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.598224+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:32:10.598224+0000","last_peered":"2023-10-09T06:32:10.598224+0000","last_clean":"2023-10-09T06:32:10.598224+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:32:10.598224+0000","last_undegraded":"2023-10-09T06:32:10.598224+0000","last_fullsized":"2023-10-09T06:32:10.598224+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":362,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.672150+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:32:16.672150+0000","last_peered":"2023-10-09T06:32:16.672150+0000","last_clean":"2023-10-09T06:32:16.672150+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:32:16.672150+0000","last_undegraded":"2023-10-09T06:32:16.672150+0000","last_fullsized":"2023-10-09T06:32:16.672150+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":705,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648181+0000","last_change":"2023-10-09T06:32:10.593067+0000","last_active":"2023-10-09T06:32:15.648181+0000","last_peered":"2023-10-09T06:32:15.648181+0000","last_clean":"2023-10-09T06:32:15.648181+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:32:15.648181+0000","last_undegraded":"2023-10-09T06:32:15.648181+0000","last_fullsized":"2023-10-09T06:32:15.648181+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"384'5096","reported_seq":5128,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.388257+0000","last_change":"2023-10-09T06:31:02.762312+0000","last_active":"2023-10-09T06:32:17.388257+0000","last_peered":"2023-10-09T06:32:17.388257+0000","last_clean":"2023-10-09T06:32:17.388257+0000","last_became_active":"2023-10-09T06:31:02.762007+0000","last_became_peered":"2023-10-09T06:31:02.762007+0000","last_unstale":"2023-10-09T06:32:17.388257+0000","last_undegraded":"2023-10-09T06:32:17.388257+0000","last_fullsized":"2023-10-09T06:32:17.388257+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:40.453840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5096,"num_write_kb":326144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":366,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671110+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:32:16.671110+0000","last_peered":"2023-10-09T06:32:16.671110+0000","last_clean":"2023-10-09T06:32:16.671110+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:32:16.671110+0000","last_undegraded":"2023-10-09T06:32:16.671110+0000","last_fullsized":"2023-10-09T06:32:16.671110+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":345,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.672281+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:32:16.672281+0000","last_peered":"2023-10-09T06:32:16.672281+0000","last_clean":"2023-10-09T06:32:16.672281+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:32:16.672281+0000","last_undegraded":"2023-10-09T06:32:16.672281+0000","last_fullsized":"2023-10-09T06:32:16.672281+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"384'5007","reported_seq":5039,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.378951+0000","last_change":"2023-10-09T06:31:03.215765+0000","last_active":"2023-10-09T06:32:17.378951+0000","last_peered":"2023-10-09T06:32:17.378951+0000","last_clean":"2023-10-09T06:32:17.378951+0000","last_became_active":"2023-10-09T06:31:03.215425+0000","last_became_peered":"2023-10-09T06:31:03.215425+0000","last_unstale":"2023-10-09T06:32:17.378951+0000","last_undegraded":"2023-10-09T06:32:17.378951+0000","last_fullsized":"2023-10-09T06:32:17.378951+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5007,"log_dups_size":0,"ondisk_log_size":5007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:46:27.586391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328138752,"num_objects":5007,"num_object_clones":0,"num_object_copies":15021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5007,"num_write_kb":320448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":685,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678682+0000","last_change":"2023-10-09T06:32:10.606850+0000","last_active":"2023-10-09T06:32:16.678682+0000","last_peered":"2023-10-09T06:32:16.678682+0000","last_clean":"2023-10-09T06:32:16.678682+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:32:16.678682+0000","last_undegraded":"2023-10-09T06:32:16.678682+0000","last_fullsized":"2023-10-09T06:32:16.678682+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:06.743797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:06.743797+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0021137209999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688895+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:32:16.688895+0000","last_peered":"2023-10-09T06:32:16.688895+0000","last_clean":"2023-10-09T06:32:16.688895+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:32:16.688895+0000","last_undegraded":"2023-10-09T06:32:16.688895+0000","last_fullsized":"2023-10-09T06:32:16.688895+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":336,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.655903+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:32:15.655903+0000","last_peered":"2023-10-09T06:32:15.655903+0000","last_clean":"2023-10-09T06:32:15.655903+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:32:15.655903+0000","last_undegraded":"2023-10-09T06:32:15.655903+0000","last_fullsized":"2023-10-09T06:32:15.655903+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"383'4991","reported_seq":5022,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.683355+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:32:15.683355+0000","last_peered":"2023-10-09T06:32:15.683355+0000","last_clean":"2023-10-09T06:32:15.683355+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:32:15.683355+0000","last_undegraded":"2023-10-09T06:32:15.683355+0000","last_fullsized":"2023-10-09T06:32:15.683355+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":4991,"log_dups_size":0,"ondisk_log_size":4991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327090176,"num_objects":4991,"num_object_clones":0,"num_object_copies":14973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4991,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":675,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.917428+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:32:15.917428+0000","last_peered":"2023-10-09T06:32:15.917428+0000","last_clean":"2023-10-09T06:32:15.917428+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:32:15.917428+0000","last_undegraded":"2023-10-09T06:32:15.917428+0000","last_fullsized":"2023-10-09T06:32:15.917428+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":388,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648327+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:32:15.648327+0000","last_peered":"2023-10-09T06:32:15.648327+0000","last_clean":"2023-10-09T06:32:15.648327+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:32:15.648327+0000","last_undegraded":"2023-10-09T06:32:15.648327+0000","last_fullsized":"2023-10-09T06:32:15.648327+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677532+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:32:16.677532+0000","last_peered":"2023-10-09T06:32:16.677532+0000","last_clean":"2023-10-09T06:32:16.677532+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:32:16.677532+0000","last_undegraded":"2023-10-09T06:32:16.677532+0000","last_fullsized":"2023-10-09T06:32:16.677532+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":322,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648040+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:32:15.648040+0000","last_peered":"2023-10-09T06:32:15.648040+0000","last_clean":"2023-10-09T06:32:15.648040+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:32:15.648040+0000","last_undegraded":"2023-10-09T06:32:15.648040+0000","last_fullsized":"2023-10-09T06:32:15.648040+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"382'5032","reported_seq":5063,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.659366+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:32:15.659366+0000","last_peered":"2023-10-09T06:32:15.659366+0000","last_clean":"2023-10-09T06:32:15.659366+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:32:15.659366+0000","last_undegraded":"2023-10-09T06:32:15.659366+0000","last_fullsized":"2023-10-09T06:32:15.659366+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5032,"num_write_kb":322048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":731,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677175+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:32:16.677175+0000","last_peered":"2023-10-09T06:32:16.677175+0000","last_clean":"2023-10-09T06:32:16.677175+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:32:16.677175+0000","last_undegraded":"2023-10-09T06:32:16.677175+0000","last_fullsized":"2023-10-09T06:32:16.677175+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678760+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:32:16.678760+0000","last_peered":"2023-10-09T06:32:16.678760+0000","last_clean":"2023-10-09T06:32:16.678760+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:32:16.678760+0000","last_undegraded":"2023-10-09T06:32:16.678760+0000","last_fullsized":"2023-10-09T06:32:16.678760+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":366,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671403+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:32:16.671403+0000","last_peered":"2023-10-09T06:32:16.671403+0000","last_clean":"2023-10-09T06:32:16.671403+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:32:16.671403+0000","last_undegraded":"2023-10-09T06:32:16.671403+0000","last_fullsized":"2023-10-09T06:32:16.671403+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":352,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648285+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:32:15.648285+0000","last_peered":"2023-10-09T06:32:15.648285+0000","last_clean":"2023-10-09T06:32:15.648285+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:32:15.648285+0000","last_undegraded":"2023-10-09T06:32:15.648285+0000","last_fullsized":"2023-10-09T06:32:15.648285+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"383'5086","reported_seq":5116,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.326369+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:32:16.326369+0000","last_peered":"2023-10-09T06:32:16.326369+0000","last_clean":"2023-10-09T06:32:16.326369+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:32:16.326369+0000","last_undegraded":"2023-10-09T06:32:16.326369+0000","last_fullsized":"2023-10-09T06:32:16.326369+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5086,"num_write_kb":325504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":727,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676325+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:32:16.676325+0000","last_peered":"2023-10-09T06:32:16.676325+0000","last_clean":"2023-10-09T06:32:16.676325+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:32:16.676325+0000","last_undegraded":"2023-10-09T06:32:16.676325+0000","last_fullsized":"2023-10-09T06:32:16.676325+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":374,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.689452+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:32:16.689452+0000","last_peered":"2023-10-09T06:32:16.689452+0000","last_clean":"2023-10-09T06:32:16.689452+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:32:16.689452+0000","last_undegraded":"2023-10-09T06:32:16.689452+0000","last_fullsized":"2023-10-09T06:32:16.689452+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":360,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.647964+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:32:15.647964+0000","last_peered":"2023-10-09T06:32:15.647964+0000","last_clean":"2023-10-09T06:32:15.647964+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:32:15.647964+0000","last_undegraded":"2023-10-09T06:32:15.647964+0000","last_fullsized":"2023-10-09T06:32:15.647964+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":382,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.655977+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:32:15.655977+0000","last_peered":"2023-10-09T06:32:15.655977+0000","last_clean":"2023-10-09T06:32:15.655977+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:32:15.655977+0000","last_undegraded":"2023-10-09T06:32:15.655977+0000","last_fullsized":"2023-10-09T06:32:15.655977+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"385'5284","reported_seq":5316,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:18.445541+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:32:18.445541+0000","last_peered":"2023-10-09T06:32:18.445541+0000","last_clean":"2023-10-09T06:32:18.445541+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:32:18.445541+0000","last_undegraded":"2023-10-09T06:32:18.445541+0000","last_fullsized":"2023-10-09T06:32:18.445541+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5284,"log_dups_size":0,"ondisk_log_size":5284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346292224,"num_objects":5284,"num_object_clones":0,"num_object_copies":15852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5284,"num_write_kb":338176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":72,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.678000+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:32:16.678000+0000","last_peered":"2023-10-09T06:32:16.678000+0000","last_clean":"2023-10-09T06:32:16.678000+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:32:16.678000+0000","last_undegraded":"2023-10-09T06:32:16.678000+0000","last_fullsized":"2023-10-09T06:32:16.678000+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":708,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676151+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:32:16.676151+0000","last_peered":"2023-10-09T06:32:16.676151+0000","last_clean":"2023-10-09T06:32:16.676151+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:32:16.676151+0000","last_undegraded":"2023-10-09T06:32:16.676151+0000","last_fullsized":"2023-10-09T06:32:16.676151+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":93,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688795+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:32:16.688795+0000","last_peered":"2023-10-09T06:32:16.688795+0000","last_clean":"2023-10-09T06:32:16.688795+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:32:16.688795+0000","last_undegraded":"2023-10-09T06:32:16.688795+0000","last_fullsized":"2023-10-09T06:32:16.688795+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":345,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.599957+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:32:10.599957+0000","last_peered":"2023-10-09T06:32:10.599957+0000","last_clean":"2023-10-09T06:32:10.599957+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:32:10.599957+0000","last_undegraded":"2023-10-09T06:32:10.599957+0000","last_fullsized":"2023-10-09T06:32:10.599957+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"383'5066","reported_seq":5097,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.683480+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:32:15.683480+0000","last_peered":"2023-10-09T06:32:15.683480+0000","last_clean":"2023-10-09T06:32:15.683480+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:32:15.683480+0000","last_undegraded":"2023-10-09T06:32:15.683480+0000","last_fullsized":"2023-10-09T06:32:15.683480+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5066,"num_write_kb":324224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":691,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648306+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:32:15.648306+0000","last_peered":"2023-10-09T06:32:15.648306+0000","last_clean":"2023-10-09T06:32:15.648306+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:32:15.648306+0000","last_undegraded":"2023-10-09T06:32:15.648306+0000","last_fullsized":"2023-10-09T06:32:15.648306+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":248,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.676051+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:32:16.676051+0000","last_peered":"2023-10-09T06:32:16.676051+0000","last_clean":"2023-10-09T06:32:16.676051+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:32:16.676051+0000","last_undegraded":"2023-10-09T06:32:16.676051+0000","last_fullsized":"2023-10-09T06:32:16.676051+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":55,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583162+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:32:10.583162+0000","last_peered":"2023-10-09T06:32:10.583162+0000","last_clean":"2023-10-09T06:32:10.583162+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:32:10.583162+0000","last_undegraded":"2023-10-09T06:32:10.583162+0000","last_fullsized":"2023-10-09T06:32:10.583162+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":366,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.584464+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:32:10.584464+0000","last_peered":"2023-10-09T06:32:10.584464+0000","last_clean":"2023-10-09T06:32:10.584464+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:32:10.584464+0000","last_undegraded":"2023-10-09T06:32:10.584464+0000","last_fullsized":"2023-10-09T06:32:10.584464+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"385'5093","reported_seq":5126,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.953758+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:32:17.953758+0000","last_peered":"2023-10-09T06:32:17.953758+0000","last_clean":"2023-10-09T06:32:17.953758+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:32:17.953758+0000","last_undegraded":"2023-10-09T06:32:17.953758+0000","last_fullsized":"2023-10-09T06:32:17.953758+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5093,"log_dups_size":0,"ondisk_log_size":5093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333774848,"num_objects":5093,"num_object_clones":0,"num_object_copies":15279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5093,"num_write_kb":325952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":695,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.605731+0000","last_change":"2023-10-09T06:32:10.604209+0000","last_active":"2023-10-09T06:32:10.605731+0000","last_peered":"2023-10-09T06:32:10.605731+0000","last_clean":"2023-10-09T06:32:10.605731+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:32:10.605731+0000","last_undegraded":"2023-10-09T06:32:10.605731+0000","last_fullsized":"2023-10-09T06:32:10.605731+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":248,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677869+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:32:16.677869+0000","last_peered":"2023-10-09T06:32:16.677869+0000","last_clean":"2023-10-09T06:32:16.677869+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:32:16.677869+0000","last_undegraded":"2023-10-09T06:32:16.677869+0000","last_fullsized":"2023-10-09T06:32:16.677869+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":378,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648227+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:32:15.648227+0000","last_peered":"2023-10-09T06:32:15.648227+0000","last_clean":"2023-10-09T06:32:15.648227+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:32:15.648227+0000","last_undegraded":"2023-10-09T06:32:15.648227+0000","last_fullsized":"2023-10-09T06:32:15.648227+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":364,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583829+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:32:10.583829+0000","last_peered":"2023-10-09T06:32:10.583829+0000","last_clean":"2023-10-09T06:32:10.583829+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:32:10.583829+0000","last_undegraded":"2023-10-09T06:32:10.583829+0000","last_fullsized":"2023-10-09T06:32:10.583829+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"385'5045","reported_seq":5078,"reported_epoch":385,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.928953+0000","last_change":"2023-10-09T06:31:03.217038+0000","last_active":"2023-10-09T06:32:17.928953+0000","last_peered":"2023-10-09T06:32:17.928953+0000","last_clean":"2023-10-09T06:32:17.928953+0000","last_became_active":"2023-10-09T06:31:03.216747+0000","last_became_peered":"2023-10-09T06:31:03.216747+0000","last_unstale":"2023-10-09T06:32:17.928953+0000","last_undegraded":"2023-10-09T06:32:17.928953+0000","last_fullsized":"2023-10-09T06:32:17.928953+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5045,"log_dups_size":0,"ondisk_log_size":5045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:22.026356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330629120,"num_objects":5045,"num_object_clones":0,"num_object_copies":15135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5045,"num_write_kb":322880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":708,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:17.264241+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:32:17.264241+0000","last_peered":"2023-10-09T06:32:17.264241+0000","last_clean":"2023-10-09T06:32:17.264241+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:32:17.264241+0000","last_undegraded":"2023-10-09T06:32:17.264241+0000","last_fullsized":"2023-10-09T06:32:17.264241+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":114,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.694264+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:32:16.694264+0000","last_peered":"2023-10-09T06:32:16.694264+0000","last_clean":"2023-10-09T06:32:16.694264+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:32:16.694264+0000","last_undegraded":"2023-10-09T06:32:16.694264+0000","last_fullsized":"2023-10-09T06:32:16.694264+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":345,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.656477+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:32:15.656477+0000","last_peered":"2023-10-09T06:32:15.656477+0000","last_clean":"2023-10-09T06:32:15.656477+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:32:15.656477+0000","last_undegraded":"2023-10-09T06:32:15.656477+0000","last_fullsized":"2023-10-09T06:32:15.656477+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":102,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688818+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:32:16.688818+0000","last_peered":"2023-10-09T06:32:16.688818+0000","last_clean":"2023-10-09T06:32:16.688818+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:32:16.688818+0000","last_undegraded":"2023-10-09T06:32:16.688818+0000","last_fullsized":"2023-10-09T06:32:16.688818+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"383'5015","reported_seq":5045,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.296573+0000","last_change":"2023-10-09T06:31:03.215173+0000","last_active":"2023-10-09T06:32:16.296573+0000","last_peered":"2023-10-09T06:32:16.296573+0000","last_clean":"2023-10-09T06:32:16.296573+0000","last_became_active":"2023-10-09T06:31:03.214809+0000","last_became_peered":"2023-10-09T06:31:03.214809+0000","last_unstale":"2023-10-09T06:32:16.296573+0000","last_undegraded":"2023-10-09T06:32:16.296573+0000","last_fullsized":"2023-10-09T06:32:16.296573+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5015,"log_dups_size":0,"ondisk_log_size":5015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:04:12.323433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328663040,"num_objects":5015,"num_object_clones":0,"num_object_copies":15045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5015,"num_write_kb":320960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":736,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688702+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:32:16.688702+0000","last_peered":"2023-10-09T06:32:16.688702+0000","last_clean":"2023-10-09T06:32:16.688702+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:32:16.688702+0000","last_undegraded":"2023-10-09T06:32:16.688702+0000","last_fullsized":"2023-10-09T06:32:16.688702+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":367,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.599936+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:32:10.599936+0000","last_peered":"2023-10-09T06:32:10.599936+0000","last_clean":"2023-10-09T06:32:10.599936+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:32:10.599936+0000","last_undegraded":"2023-10-09T06:32:10.599936+0000","last_fullsized":"2023-10-09T06:32:10.599936+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":347,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.584217+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:32:10.584217+0000","last_peered":"2023-10-09T06:32:10.584217+0000","last_clean":"2023-10-09T06:32:10.584217+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:32:10.584217+0000","last_undegraded":"2023-10-09T06:32:10.584217+0000","last_fullsized":"2023-10-09T06:32:10.584217+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":348,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671169+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:32:16.671169+0000","last_peered":"2023-10-09T06:32:16.671169+0000","last_clean":"2023-10-09T06:32:16.671169+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:32:16.671169+0000","last_undegraded":"2023-10-09T06:32:16.671169+0000","last_fullsized":"2023-10-09T06:32:16.671169+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":642,"reported_epoch":385,"state":"active+undersized+remapped","last_fresh":"2023-10-09T06:32:17.729270+0000","last_change":"2023-10-09T06:32:17.729000+0000","last_active":"2023-10-09T06:32:17.729270+0000","last_peered":"2023-10-09T06:32:17.729270+0000","last_clean":"2023-10-09T06:27:32.622178+0000","last_became_active":"2023-10-09T06:32:17.729000+0000","last_became_peered":"2023-10-09T06:32:17.729000+0000","last_unstale":"2023-10-09T06:32:17.729270+0000","last_undegraded":"2023-10-09T06:32:17.729270+0000","last_fullsized":"2023-10-09T06:32:17.695972+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:33.751368+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:33.751368+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.016450896999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","2","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":348,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583740+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:32:10.583740+0000","last_peered":"2023-10-09T06:32:10.583740+0000","last_clean":"2023-10-09T06:32:10.583740+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:32:10.583740+0000","last_undegraded":"2023-10-09T06:32:10.583740+0000","last_fullsized":"2023-10-09T06:32:10.583740+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":371,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-10-09T06:32:15.648202+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:32:15.648202+0000","last_peered":"2023-10-09T06:32:15.648202+0000","last_clean":"2023-10-09T06:32:15.648202+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:32:15.648202+0000","last_undegraded":"2023-10-09T06:32:15.648202+0000","last_fullsized":"2023-10-09T06:32:15.648202+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":572,"reported_epoch":385,"state":"active+undersized+remapped","last_fresh":"2023-10-09T06:32:17.708994+0000","last_change":"2023-10-09T06:32:17.708732+0000","last_active":"2023-10-09T06:32:17.708994+0000","last_peered":"2023-10-09T06:32:17.708994+0000","last_clean":"2023-10-09T06:27:33.205388+0000","last_became_active":"2023-10-09T06:32:17.708732+0000","last_became_peered":"2023-10-09T06:32:17.708732+0000","last_unstale":"2023-10-09T06:32:17.708994+0000","last_undegraded":"2023-10-09T06:32:17.708994+0000","last_fullsized":"2023-10-09T06:32:17.695281+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.865112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.865112+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,7],"avail_no_missing":["3","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":76,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677911+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:32:16.677911+0000","last_peered":"2023-10-09T06:32:16.677911+0000","last_clean":"2023-10-09T06:32:16.677911+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:32:16.677911+0000","last_undegraded":"2023-10-09T06:32:16.677911+0000","last_fullsized":"2023-10-09T06:32:16.677911+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":392,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671376+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:32:16.671376+0000","last_peered":"2023-10-09T06:32:16.671376+0000","last_clean":"2023-10-09T06:32:16.671376+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:32:16.671376+0000","last_undegraded":"2023-10-09T06:32:16.671376+0000","last_fullsized":"2023-10-09T06:32:16.671376+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":76,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.677446+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:32:16.677446+0000","last_peered":"2023-10-09T06:32:16.677446+0000","last_clean":"2023-10-09T06:32:16.677446+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:32:16.677446+0000","last_undegraded":"2023-10-09T06:32:16.677446+0000","last_fullsized":"2023-10-09T06:32:16.677446+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":402,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671403+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:32:16.671403+0000","last_peered":"2023-10-09T06:32:16.671403+0000","last_clean":"2023-10-09T06:32:16.671403+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:32:16.671403+0000","last_undegraded":"2023-10-09T06:32:16.671403+0000","last_fullsized":"2023-10-09T06:32:16.671403+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":752,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.610742+0000","last_change":"2023-10-09T06:32:10.609576+0000","last_active":"2023-10-09T06:32:10.610742+0000","last_peered":"2023-10-09T06:32:10.610742+0000","last_clean":"2023-10-09T06:32:10.610742+0000","last_became_active":"2023-10-09T06:32:10.609210+0000","last_became_peered":"2023-10-09T06:32:10.609210+0000","last_unstale":"2023-10-09T06:32:10.610742+0000","last_undegraded":"2023-10-09T06:32:10.610742+0000","last_fullsized":"2023-10-09T06:32:10.610742+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:28.717694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:28.717694+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0069898099999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":218454,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":84,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.688310+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:32:16.688310+0000","last_peered":"2023-10-09T06:32:16.688310+0000","last_clean":"2023-10-09T06:32:16.688310+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:32:16.688310+0000","last_undegraded":"2023-10-09T06:32:16.688310+0000","last_fullsized":"2023-10-09T06:32:16.688310+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.689699+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:32:16.689699+0000","last_peered":"2023-10-09T06:32:16.689699+0000","last_clean":"2023-10-09T06:32:16.689699+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:32:16.689699+0000","last_undegraded":"2023-10-09T06:32:16.689699+0000","last_fullsized":"2023-10-09T06:32:16.689699+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":477,"reported_epoch":384,"state":"remapped+peering","last_fresh":"2023-10-09T06:32:16.678799+0000","last_change":"2023-10-09T06:32:16.678799+0000","last_active":"2023-10-09T06:21:14.782791+0000","last_peered":"2023-10-09T06:20:09.888953+0000","last_clean":"2023-10-09T06:19:04.606807+0000","last_became_active":"2023-10-09T06:19:07.672145+0000","last_became_peered":"2023-10-09T06:19:07.672145+0000","last_unstale":"2023-10-09T06:32:16.678799+0000","last_undegraded":"2023-10-09T06:32:16.678799+0000","last_fullsized":"2023-10-09T06:32:16.678799+0000","mapping_epoch":384,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:39.977928+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:39.977928+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":67088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":377,"reported_epoch":382,"state":"active+clean","last_fresh":"2023-10-09T06:32:10.583823+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:32:10.583823+0000","last_peered":"2023-10-09T06:32:10.583823+0000","last_clean":"2023-10-09T06:32:10.583823+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:32:10.583823+0000","last_undegraded":"2023-10-09T06:32:10.583823+0000","last_fullsized":"2023-10-09T06:32:10.583823+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":345,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-10-09T06:32:16.671049+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:32:16.671049+0000","last_peered":"2023-10-09T06:32:16.671049+0000","last_clean":"2023-10-09T06:32:16.671049+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:32:16.671049+0000","last_undegraded":"2023-10-09T06:32:16.671049+0000","last_fullsized":"2023-10-09T06:32:16.671049+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5318377472,"num_objects":81152,"num_object_clones":0,"num_object_copies":243456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81152,"num_write_kb":5193728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15976628224,"data_stored":15976628224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81152,"ondisk_log_size":81152,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":658400,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":810918,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":87,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234420,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3206984,"kb_used_data":2298560,"kb_used_omap":0,"kb_used_meta":908416,"kb_avail":90538168,"statfs":{"total":95995035648,"available":92711084032,"internally_reserved":0,"allocated":2353725440,"data_stored":2352748725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":930217984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:06 2023","interfaces":[{"interface":"back","average":{"1min":10.897,"5min":6.9299999999999997,"15min":5.5739999999999998},"min":{"1min":0.33500000000000002,"5min":0.218,"15min":0.20899999999999999},"max":{"1min":196.28899999999999,"5min":215.97800000000001,"15min":219.41900000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":10.824999999999999,"5min":6.9269999999999996,"15min":5.6799999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":196.333,"5min":215.577,"15min":219.399},"last":1.71}]},{"osd":1,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":5.9080000000000004,"5min":6.9850000000000003,"15min":4.4669999999999996},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":107.87,"5min":215.61699999999999,"15min":215.61699999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":6.0869999999999997,"5min":6.1299999999999999,"15min":4.0709999999999997},"min":{"1min":0.38600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.904,"5min":216.22200000000001,"15min":216.22200000000001},"last":0.55500000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:31:38 2023","interfaces":[{"interface":"back","average":{"1min":10.584,"5min":6.4859999999999998,"15min":6.665},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":198.94200000000001,"5min":216.34299999999999,"15min":216.34299999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":10.708,"5min":6.5149999999999997,"15min":6.8470000000000004},"min":{"1min":0.30299999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":198.92099999999999,"5min":216.25200000000001,"15min":216.25200000000001},"last":4.3620000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":7.4969999999999999,"15min":3.931},"min":{"1min":0.37,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":3.4460000000000002,"5min":215.922,"15min":215.922},"last":1.6240000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":7.6669999999999998,"15min":4.0170000000000003},"min":{"1min":0.35599999999999998,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":3.3210000000000002,"5min":215.94900000000001,"15min":215.94900000000001},"last":2.7349999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":4.0579999999999998,"15min":2.5419999999999998},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.23300000000000001},"max":{"1min":4.4589999999999996,"5min":218.446,"15min":218.446},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.755,"5min":3.9969999999999999,"15min":2.395},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.23699999999999999},"max":{"1min":3.2210000000000001,"5min":216.172,"15min":216.172},"last":1.5369999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":4.2270000000000003,"15min":2.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":3.7349999999999999,"5min":216.37200000000001,"15min":216.37200000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.1839999999999999,"5min":4.1559999999999997,"15min":2.4369999999999998},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":4.5670000000000002,"5min":218.15899999999999,"15min":218.15899999999999},"last":0.60099999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:31:21 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":4.2880000000000003,"15min":2.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":4.548,"5min":218.19999999999999,"15min":218.19999999999999},"last":1.5189999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":3.944,"15min":2.149},"min":{"1min":0.40799999999999997,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":3.2650000000000001,"5min":218.05500000000001,"15min":218.05500000000001},"last":1.5800000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496055,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3157404,"kb_used_data":2254292,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":90587748,"statfs":{"total":95995035648,"available":92761853952,"internally_reserved":0,"allocated":2308395008,"data_stored":2307445773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.9669999999999996,"5min":2.6960000000000002,"15min":3.3069999999999999},"min":{"1min":0.24199999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":75.061999999999998,"5min":75.061999999999998,"15min":110.926},"last":0.28199999999999997},{"interface":"front","average":{"1min":5.944,"5min":3.1829999999999998,"15min":3.5720000000000001},"min":{"1min":0.28299999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":75.021000000000001,"5min":75.021000000000001,"15min":110.911},"last":0.48299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":6.9020000000000001,"15min":4.8869999999999996},"min":{"1min":0.249,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":2.375,"5min":292.92200000000003,"15min":292.92200000000003},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.84799999999999998,"5min":6.9800000000000004,"15min":4.7400000000000002},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.253},"max":{"1min":4.4359999999999999,"5min":292.904,"15min":292.904},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.9930000000000001,"15min":2.5369999999999999},"min":{"1min":0.307,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.9009999999999998,"5min":20.331,"15min":75.191999999999993},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.661,"15min":2.2509999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.3479999999999999,"5min":8.6069999999999993,"15min":75.120000000000005},"last":0.46500000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":4.2960000000000003,"15min":3.2770000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28299999999999997},"max":{"1min":5.069,"5min":91.174000000000007,"15min":110.358},"last":0.313},{"interface":"front","average":{"1min":1.4710000000000001,"5min":4.375,"15min":3.3450000000000002},"min":{"1min":0.375,"5min":0.318,"15min":0.311},"max":{"1min":5.0339999999999998,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.50900000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:31:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.4359999999999999,"15min":1.2769999999999999},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":2.6200000000000001,"5min":12.666,"15min":17.643999999999998},"last":0.435},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.071,"15min":1.1160000000000001},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":2.9940000000000002,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:31:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.335,"15min":1.3540000000000001},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":2.4870000000000001,"5min":12.747999999999999,"15min":22.379999999999999},"last":3.653},{"interface":"front","average":{"1min":1.3380000000000001,"5min":1.5309999999999999,"15min":1.343},"min":{"1min":0.38400000000000001,"5min":0.32400000000000001,"15min":0.29999999999999999},"max":{"1min":6.9349999999999996,"5min":16.536999999999999,"15min":24.401},"last":0.64200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5840000000000001,"5min":4.1630000000000003,"15min":2.569},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":19.295999999999999,"5min":122.937,"15min":122.937},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.647,"5min":4.444,"15min":2.6850000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":19.584,"5min":122.92100000000001,"15min":122.92100000000001},"last":0.66900000000000004}]}]},{"osd":1,"up_from":110,"seq":472446402793,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2492416,"kb_used_data":1798648,"kb_used_omap":4,"kb_used_meta":693755,"kb_avail":91252736,"statfs":{"total":95995035648,"available":93442801664,"internally_reserved":0,"allocated":1841815552,"data_stored":1840080999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":710406030},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":10.044,"5min":8.0340000000000007,"15min":5.7519999999999998},"min":{"1min":0.20699999999999999,"5min":0.19,"15min":0.189},"max":{"1min":143.465,"5min":269.79199999999997,"15min":269.79199999999997},"last":1.714},{"interface":"front","average":{"1min":10.407,"5min":8.1820000000000004,"15min":6.1429999999999998},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.17899999999999999},"max":{"1min":143.66399999999999,"5min":271.20999999999998,"15min":271.20999999999998},"last":1.5760000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:31:39 2023","interfaces":[{"interface":"back","average":{"1min":9.2430000000000003,"5min":2.9220000000000002,"15min":3.4550000000000001},"min":{"1min":0.23799999999999999,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":138.452,"5min":138.452,"15min":139.75399999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":9.4960000000000004,"5min":3.746,"15min":4.1379999999999999},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":138.47499999999999,"5min":138.47499999999999,"15min":140.29900000000001},"last":0.221}]},{"osd":3,"last update":"Mon Oct 9 06:31:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":2.5409999999999999,"15min":3.3050000000000002},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":4.452,"5min":31.062000000000001,"15min":153.18199999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.437,"5min":1.952,"15min":2.4460000000000002},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.95999999999999996,"5min":27.657,"15min":152.52000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.7569999999999999,"15min":3.3610000000000002},"min":{"1min":0.317,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":2.5409999999999999,"5min":13.087999999999999,"15min":199.43799999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.655,"15min":3.1699999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.605,"5min":11.974,"15min":199.32900000000001},"last":0.28499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":1.8779999999999999,"15min":2.0049999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.9289999999999998,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.438},{"interface":"front","average":{"1min":0.76500000000000001,"5min":2.1779999999999999,"15min":2.4670000000000001},"min":{"1min":0.38900000000000001,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":2.5779999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.371}]},{"osd":6,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":2.5680000000000001,"15min":2.6930000000000001},"min":{"1min":0.35599999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.4980000000000002,"5min":52.488,"15min":77.292000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.69399999999999995,"5min":2.8109999999999999,"15min":2.758},"min":{"1min":0.38700000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.8280000000000001,"5min":52.558,"15min":77.378},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":3.681,"15min":2.9660000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":0.79500000000000004,"5min":55.231000000000002,"15min":77.456999999999994},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":3.0539999999999998,"15min":2.4289999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34399999999999997},"max":{"1min":0.81499999999999995,"5min":55.253999999999998,"15min":77.359999999999999},"last":0.46800000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024226,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2937940,"kb_used_data":2101516,"kb_used_omap":5,"kb_used_meta":836410,"kb_avail":90807212,"statfs":{"total":95995035648,"available":92986585088,"internally_reserved":0,"allocated":2151952384,"data_stored":2149954132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":856484474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,2,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:31:29 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":3.2789999999999999,"15min":3.3290000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":3.2839999999999998,"5min":191.446,"15min":191.446},"last":1.8009999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.3929999999999998,"15min":3.5579999999999998},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":4.0609999999999999,"5min":191.393,"15min":191.393},"last":1.8149999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.274,"15min":2.226},"min":{"1min":0.28599999999999998,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":4.0970000000000004,"5min":51.886000000000003,"15min":57.162999999999997},"last":1.542},{"interface":"front","average":{"1min":1.7130000000000001,"5min":2.899,"15min":2.6339999999999999},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.6429999999999998,"5min":52.302999999999997,"15min":57.585000000000001},"last":2.2149999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:32:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2450000000000001,"5min":8.2870000000000008,"15min":5.5910000000000002},"min":{"1min":0.40699999999999997,"5min":0.26200000000000001,"15min":0.22500000000000001},"max":{"1min":11.125,"5min":208.11199999999999,"15min":208.11199999999999},"last":1.9650000000000001},{"interface":"front","average":{"1min":2.7280000000000002,"5min":7.8470000000000004,"15min":5.5460000000000003},"min":{"1min":0.36099999999999999,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":11.101000000000001,"5min":208.25899999999999,"15min":208.25899999999999},"last":1.9790000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.6220000000000001,"15min":1.4359999999999999},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":4.9260000000000002,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.698,"5min":1.919,"15min":1.702},"min":{"1min":0.34899999999999998,"5min":0.33200000000000002,"15min":0.28299999999999997},"max":{"1min":4.8970000000000002,"5min":18.081,"15min":18.081},"last":1.6240000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.8009999999999999,"15min":1.73},"min":{"1min":0.315,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":4.9569999999999999,"5min":10.865,"15min":62.984999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.9550000000000001,"5min":2.4140000000000001,"15min":2.194},"min":{"1min":0.47599999999999998,"5min":0.34899999999999998,"15min":0.30299999999999999},"max":{"1min":13.968999999999999,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.92900000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":4.6120000000000001,"15min":2.8010000000000002},"min":{"1min":0.439,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":8.3190000000000008,"5min":120.572,"15min":120.572},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.593,"5min":3.915,"15min":2.2959999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.30299999999999999},"max":{"1min":5.1790000000000003,"5min":120.64400000000001,"15min":120.64400000000001},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":3.3239999999999998,"15min":2.4119999999999999},"min":{"1min":0.376,"5min":0.375,"15min":0.32300000000000001},"max":{"1min":4.8639999999999999,"5min":122.018,"15min":122.018},"last":1.591},{"interface":"front","average":{"1min":2.702,"5min":4.0069999999999997,"15min":2.7709999999999999},"min":{"1min":0.53500000000000003,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":8.3480000000000008,"5min":122.32299999999999,"15min":122.32299999999999},"last":0.96199999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814095,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3766564,"kb_used_data":3185436,"kb_used_omap":7,"kb_used_meta":581112,"kb_avail":89978588,"statfs":{"total":95995035648,"available":92138074112,"internally_reserved":0,"allocated":3261886464,"data_stored":3260219044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":595058769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":4.5640000000000001,"15min":4.7919999999999998},"min":{"1min":0.22,"5min":0.19400000000000001,"15min":0.157},"max":{"1min":29.071999999999999,"5min":162.21100000000001,"15min":163.40100000000001},"last":4.7590000000000003},{"interface":"front","average":{"1min":3.1440000000000001,"5min":4.6989999999999998,"15min":4.4530000000000003},"min":{"1min":0.26100000000000001,"5min":0.20300000000000001,"15min":0.182},"max":{"1min":29.102,"5min":168.49700000000001,"15min":168.49700000000001},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":3.3849999999999998,"15min":4.1390000000000002},"min":{"1min":0.246,"5min":0.20999999999999999,"15min":0.188},"max":{"1min":10.287000000000001,"5min":92.114000000000004,"15min":222.50299999999999},"last":4.6619999999999999},{"interface":"front","average":{"1min":1.5409999999999999,"5min":4.3700000000000001,"15min":4.4560000000000004},"min":{"1min":0.32200000000000001,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.7579999999999991,"5min":91.721999999999994,"15min":222.19399999999999},"last":4.2240000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:32:11 2023","interfaces":[{"interface":"back","average":{"1min":5.4690000000000003,"5min":4.2690000000000001,"15min":3.1499999999999999},"min":{"1min":0.216,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":77.692999999999998,"5min":91.644000000000005,"15min":91.644000000000005},"last":1.242},{"interface":"front","average":{"1min":5.8920000000000003,"5min":4.96,"15min":3.4209999999999998},"min":{"1min":0.29199999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":78.575999999999993,"5min":91.685000000000002,"15min":91.685000000000002},"last":4.1820000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.861,"15min":2.0339999999999998},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.29099999999999998},"max":{"1min":9.6959999999999997,"5min":17.776,"15min":90.938999999999993},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.038,"5min":2.0619999999999998,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":4.0679999999999996,"5min":12.433999999999999,"15min":90.950000000000003},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.0870000000000002,"15min":1.5820000000000001},"min":{"1min":0.39100000000000001,"5min":0.315,"15min":0.309},"max":{"1min":9.6660000000000004,"5min":17.800000000000001,"15min":17.800000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.1280000000000001,"15min":1.8580000000000001},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.6659999999999999,"5min":11.285,"15min":11.285},"last":0.61399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":8.5259999999999998,"5min":3.742,"15min":2.9590000000000001},"min":{"1min":0.35099999999999998,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":123.47799999999999,"5min":123.47799999999999,"15min":138.291},"last":0.56100000000000005},{"interface":"front","average":{"1min":8.6419999999999995,"5min":3.1789999999999998,"15min":2.645},"min":{"1min":0.47899999999999998,"5min":0.34899999999999998,"15min":0.33000000000000002},"max":{"1min":122.517,"5min":122.517,"15min":138.01599999999999},"last":4.4340000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.004,"15min":1.925},"min":{"1min":0.42799999999999999,"5min":0.38100000000000001,"15min":0.36499999999999999},"max":{"1min":4.3559999999999999,"5min":12.673999999999999,"15min":55.851999999999997},"last":4.4889999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":2.2050000000000001,"15min":1.905},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7279999999999998,"5min":17.734999999999999,"15min":55.731000000000002},"last":0.60199999999999998}]}]},{"osd":3,"up_from":149,"seq":639950127293,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3393728,"kb_used_data":2791608,"kb_used_omap":2,"kb_used_meta":602109,"kb_avail":90351424,"statfs":{"total":95995035648,"available":92519858176,"internally_reserved":0,"allocated":2858606592,"data_stored":2856988293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":616559669},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":21.966000000000001,"5min":7.4710000000000001,"15min":5.5839999999999996},"min":{"1min":0.249,"5min":0.19,"15min":0.19},"max":{"1min":161.821,"5min":161.821,"15min":187.852},"last":158.173},{"interface":"front","average":{"1min":20.521000000000001,"5min":6.8310000000000004,"15min":5.7229999999999999},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":163.52000000000001,"5min":163.52000000000001,"15min":187.923},"last":158.11699999999999}]},{"osd":1,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":10.779999999999999,"5min":4.6929999999999996,"15min":4.2009999999999996},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.214},"max":{"1min":155.23099999999999,"5min":155.23099999999999,"15min":187.89099999999999},"last":155.23099999999999},{"interface":"front","average":{"1min":11.704000000000001,"5min":6.1909999999999998,"15min":3.8340000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":155.24799999999999,"5min":155.24799999999999,"15min":155.24799999999999},"last":155.24799999999999}]},{"osd":2,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":11.336,"5min":4.8789999999999996,"15min":3.9870000000000001},"min":{"1min":0.35599999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":167.88499999999999,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.37},{"interface":"front","average":{"1min":12.445,"5min":5.6749999999999998,"15min":3.9209999999999998},"min":{"1min":0.307,"5min":0.27900000000000003,"15min":0.22500000000000001},"max":{"1min":168.16900000000001,"5min":168.16900000000001,"15min":187.79499999999999},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":3.012,"15min":1.915},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.0839999999999996,"5min":48.249000000000002,"15min":48.249000000000002},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.355,"5min":2.7050000000000001,"15min":2.4750000000000001},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.29899999999999999},"max":{"1min":6.4249999999999998,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.63800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":2.5219999999999998,"15min":2.552},"min":{"1min":0.35399999999999998,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":9.5359999999999996,"5min":48.162999999999997,"15min":187.833},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.47,"5min":2.137,"15min":2.3239999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":8.0800000000000001,"5min":48.106000000000002,"15min":187.952},"last":0.78900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":4.1699999999999999,"15min":3.1579999999999999},"min":{"1min":0.41099999999999998,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":6.3920000000000003,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.367,"5min":3.8919999999999999,"15min":2.8730000000000002},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.32900000000000001},"max":{"1min":8.0470000000000006,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.61599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3759999999999999,"5min":4.7089999999999996,"15min":3.2250000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":5.4939999999999998,"5min":121.613,"15min":192.20400000000001},"last":0.627},{"interface":"front","average":{"1min":1.6240000000000001,"5min":4.7409999999999997,"15min":3.2850000000000001},"min":{"1min":0.39800000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.4539999999999997,"5min":121.559,"15min":187.86500000000001},"last":0.82299999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019329,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1874480,"kb_used_data":983592,"kb_used_omap":0,"kb_used_meta":890880,"kb_avail":91870672,"statfs":{"total":95995035648,"available":94075568128,"internally_reserved":0,"allocated":1007198208,"data_stored":1006237291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912261120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:02 2023","interfaces":[{"interface":"back","average":{"1min":25.273,"5min":6.2830000000000004,"15min":3.7549999999999999},"min":{"1min":0.23699999999999999,"5min":0.22800000000000001,"15min":0.20499999999999999},"max":{"1min":287.58800000000002,"5min":287.58800000000002,"15min":287.58800000000002},"last":36.512},{"interface":"front","average":{"1min":25.100999999999999,"5min":6.282,"15min":3.988},"min":{"1min":0.36899999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":287.57100000000003,"5min":287.57100000000003,"15min":287.57100000000003},"last":36.494999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":4.2999999999999998,"15min":2.891},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23100000000000001},"max":{"1min":0.67700000000000005,"5min":128.24299999999999,"15min":128.24299999999999},"last":34.311999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":4.3200000000000003,"15min":3.0590000000000002},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.245},"max":{"1min":1.0660000000000001,"5min":128.21600000000001,"15min":128.21600000000001},"last":34.210999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.26600000000000001},"max":{"1min":2.0950000000000002,"5min":32.613,"15min":69.640000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.2,"5min":1.7629999999999999,"15min":2.4540000000000002},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.9320000000000004,"5min":34.188000000000002,"15min":69.676000000000002},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":10.574,"5min":9,"15min":5.109},"min":{"1min":0.38300000000000001,"5min":0.35399999999999998,"15min":0.30299999999999999},"max":{"1min":161.19300000000001,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":10.404,"5min":8.6720000000000006,"15min":4.8250000000000002},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":161.934,"5min":161.934,"15min":161.934},"last":0.495}]},{"osd":5,"last update":"Mon Oct 9 06:31:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.93999999999999995,"15min":1.0249999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26100000000000001},"max":{"1min":2.7829999999999999,"5min":6.25,"15min":10.673},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.0760000000000001,"15min":1.0309999999999999},"min":{"1min":0.32200000000000001,"5min":0.26600000000000001,"15min":0.26100000000000001},"max":{"1min":4.9589999999999996,"5min":7.2249999999999996,"15min":12.552},"last":0.52900000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:32:02 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":2.4870000000000001,"15min":1.2969999999999999},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.1099999999999999,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.77000000000000002,"5min":2.8420000000000001,"15min":1.655},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":3.004,"5min":163.381,"15min":163.381},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:32:17 2023","interfaces":[{"interface":"back","average":{"1min":4.875,"5min":1.8879999999999999,"15min":1.71},"min":{"1min":0.36199999999999999,"5min":0.312,"15min":0.308},"max":{"1min":70.247,"5min":70.247,"15min":70.247},"last":0.58499999999999996},{"interface":"front","average":{"1min":4.625,"5min":1.784,"15min":1.6240000000000001},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":70.159000000000006,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.65900000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757694,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557428,"kb_used_data":663916,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":92187724,"statfs":{"total":95995035648,"available":94400229376,"internally_reserved":0,"allocated":679849984,"data_stored":678889644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":22.475000000000001,"5min":8.4580000000000002,"15min":4.798},"min":{"1min":0.33600000000000002,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":294.89699999999999,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.25},{"interface":"front","average":{"1min":22.164999999999999,"5min":8.2110000000000003,"15min":4.7679999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22},"max":{"1min":294.30399999999997,"5min":294.30399999999997,"15min":294.30399999999997},"last":0.32100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:31:44 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":4.9290000000000003,"15min":2.7589999999999999},"min":{"1min":0.27700000000000002,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":3.7109999999999999,"5min":126.71899999999999,"15min":126.71899999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.94799999999999995,"5min":5.2169999999999996,"15min":2.9540000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":7.8099999999999996,"5min":126.931,"15min":126.931},"last":3.1709999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":2.5209999999999999,"15min":2.2000000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.9029999999999996,"5min":51.393000000000001,"15min":81.260000000000005},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.497,"5min":2.097,"15min":2.0600000000000001},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.29699999999999999},"max":{"1min":1.23,"5min":51.432000000000002,"15min":81.245000000000005},"last":2.9180000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:32:06 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":9.0830000000000002,"15min":5.5049999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.9580000000000002,"5min":144.56399999999999,"15min":190.411},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.6339999999999999,"5min":9.0790000000000006,"15min":5.4119999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":5.1470000000000002,"5min":144.62799999999999,"15min":190.70699999999999},"last":1.1699999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:31:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.0209999999999999,"15min":1.236},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.6279999999999999,"5min":10.529999999999999,"15min":32.850999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.872,"15min":1.089},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0760000000000001,"5min":6.2610000000000001,"15min":32.82},"last":0.26800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:32:00 2023","interfaces":[{"interface":"back","average":{"1min":3.903,"5min":1.891,"15min":1.6379999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":57.505000000000003,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":4.2889999999999997,"5min":2.093,"15min":1.754},"min":{"1min":0.33500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":55.040999999999997,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.30499999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:32:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.4239999999999999,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.4049999999999998,"5min":10.461,"15min":30.658999999999999},"last":0.37},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.1819999999999999,"15min":1.403},"min":{"1min":0.35799999999999998,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":2.048,"5min":6.2320000000000002,"15min":30.683},"last":0.35799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1969160192,"data_stored":1969160192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1669136384,"data_stored":1669136384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3283812352,"data_stored":3283812352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2675834880,"data_stored":2675834880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1014890496,"data_stored":1014890496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":678494208,"data_stored":678494208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2313027584,"data_stored":2313027584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2372272128,"data_stored":2372272128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:32:22.681 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:32:22.682 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_7 pg_num 26 2023-10-09T06:32:22.745 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:22.850 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[212750]: 1.1d deep-scrub starts 2023-10-09T06:32:22.850 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[212750]: 1.1d deep-scrub ok 2023-10-09T06:32:22.850 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[212750]: pgmap v1185: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s 2023-10-09T06:32:22.850 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:22.850 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[220921]: 1.1d deep-scrub starts 2023-10-09T06:32:22.851 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[220921]: 1.1d deep-scrub ok 2023-10-09T06:32:22.851 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[220921]: pgmap v1185: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s 2023-10-09T06:32:22.851 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:22.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:22.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:22.965 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:22.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 85920 85904 75.5784 44.1875 0.0225915 0.0132256 2023-10-09T06:32:22.995 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:23.049 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:23.053 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:22 smithi150 ceph-mon[194178]: 1.1d deep-scrub starts 2023-10-09T06:32:23.053 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:22 smithi150 ceph-mon[194178]: 1.1d deep-scrub ok 2023-10-09T06:32:23.053 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:22 smithi150 ceph-mon[194178]: pgmap v1185: 119 pgs: 5 active+undersized+remapped, 1 remapped+peering, 113 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 38 MiB/s wr, 608 op/s 2023-10-09T06:32:23.053 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:23.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:23.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:23.155 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:23.203 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:23.235 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:32:23.246 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:23.278 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:32:23.319 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:32:23.366 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:23.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:23.397 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:32:23.417 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:32:23.459 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:32:23.479 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:23.503 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:32:23.510 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:23.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:32:23.563 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.564 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.565 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.565 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.565 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:32:23.571 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:23.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:23.679 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:23.717 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:23.787 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:23.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:23.894 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[212750]: 1.1c scrub starts 2023-10-09T06:32:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[212750]: 1.1c scrub ok 2023-10-09T06:32:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/253497884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[220921]: 1.1c scrub starts 2023-10-09T06:32:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[220921]: 1.1c scrub ok 2023-10-09T06:32:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/253497884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:23.919 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:23.961 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:23.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:23.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 86657 86641 75.1685 46.0625 0.00580815 0.0132839 2023-10-09T06:32:24.067 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:24.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:23 smithi150 ceph-mon[194178]: 1.1c scrub starts 2023-10-09T06:32:24.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:23 smithi150 ceph-mon[194178]: 1.1c scrub ok 2023-10-09T06:32:24.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/253497884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:24.092 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:24.134 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:24.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:24.241 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.280 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:24.342 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:24.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:24.449 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:24.484 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:24.492 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:24.551 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:24.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:24.657 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:24.755 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:24.976 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 87434 87418 74.8042 48.5625 0.0141709 0.0133635 2023-10-09T06:32:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[220921]: pgmap v1186: 119 pgs: 3 active+undersized+remapped, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:32:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:32:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:32:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[212750]: pgmap v1186: 119 pgs: 3 active+undersized+remapped, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:32:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:32:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:24 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:32:25.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:24 smithi150 ceph-mon[194178]: pgmap v1186: 119 pgs: 3 active+undersized+remapped, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-09T06:32:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:24 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T06:32:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:24 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:32:25.250 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:25.281 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:25.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:25.393 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:25.493 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:25.961 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:25.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:25.977 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 88302 88286 74.5265 54.25 0.0161923 0.0134121 2023-10-09T06:32:26.062 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.15 scrub starts 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.15 scrub ok 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.19 scrub starts 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.19 scrub ok 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:32:26.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.15 scrub starts 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.15 scrub ok 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.19 scrub starts 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.19 scrub ok 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:32:26.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:25 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:32:26.153 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.15 scrub starts 2023-10-09T06:32:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.15 scrub ok 2023-10-09T06:32:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.19 scrub starts 2023-10-09T06:32:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.19 scrub ok 2023-10-09T06:32:26.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:32:26.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:25 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:32:26.506 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:26.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:26.617 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:26.644 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:26.684 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:26.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:26.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:26.790 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:26.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: pgmap v1187: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 683 op/s 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: 1.16 scrub starts 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: 1.16 scrub ok 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:26.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: pgmap v1187: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 683 op/s 2023-10-09T06:32:26.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: 1.16 scrub starts 2023-10-09T06:32:26.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: 1.16 scrub ok 2023-10-09T06:32:26.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: 1.8 scrub starts 2023-10-09T06:32:26.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: 1.8 scrub ok 2023-10-09T06:32:26.796 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:26.836 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:32:26.840 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: pgmap v1187: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 683 op/s 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: 1.16 scrub starts 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: 1.16 scrub ok 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: 1.8 scrub starts 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: 1.8 scrub ok 2023-10-09T06:32:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:32:26.909 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:26.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:26.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:26.977 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 88928 88912 74.0546 39.125 0.00552415 0.013487 2023-10-09T06:32:27.014 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:27.049 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:27.091 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:27.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:27.197 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:27.233 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:27.237 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:32:27.298 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:27.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:27.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:27.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.438 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:27.507 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:27.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:27.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:27.614 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.646 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:27.699 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:27.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:27.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:27.807 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:27.851 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:27.921 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:27.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:27.982 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 89743 89727 73.745 50.9375 0.039483 0.0135552 2023-10-09T06:32:28.031 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:28.078 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: 1.1a deep-scrub starts 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: 1.1a deep-scrub ok 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[220921]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: 1.1a deep-scrub starts 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: 1.1a deep-scrub ok 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:27 smithi107 ceph-mon[212750]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.088 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: 1.1a deep-scrub starts 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: 1.1a deep-scrub ok 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4030534099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: osdmap e388: 8 total, 8 up, 8 in 2023-10-09T06:32:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:27 smithi150 ceph-mon[194178]: failed to encode map e388 with expected crc 2023-10-09T06:32:28.307 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:28.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:28.420 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:28.458 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:28.532 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:28.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:28.541 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:28.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:28.675 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:28.731 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:28.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:28.839 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:28.877 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:28.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:28 smithi107 ceph-mon[220921]: pgmap v1189: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.6 KiB/s rd, 48 MiB/s wr, 768 op/s 2023-10-09T06:32:28.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:28 smithi107 ceph-mon[212750]: pgmap v1189: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.6 KiB/s rd, 48 MiB/s wr, 768 op/s 2023-10-09T06:32:28.925 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:28.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:28.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:28.982 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 90498 90482 73.4002 47.1875 0.0211557 0.0136182 2023-10-09T06:32:29.035 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:29.039 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:32:29.039 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:32:29.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:32:29.039 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82043 10.8 10:13dd737b:::benchmark_data_smithi107_363200_object82042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.21775011199998, 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99744534200000001, 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.040 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82043 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817960+0000", 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817959+0000", 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817982+0000", 2023-10-09T06:32:29.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3319000000000002e-05 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817989+0000", 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8510000000000001e-06 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.817995+0000", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3319999999999999e-06 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.041469+0000", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22347494400000001 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.041480+0000", 2023-10-09T06:32:29.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0916e-05 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.815307+0000", 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.77382648399999998 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.815405+0000", 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8045000000000001e-05 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.046 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82057 10.14 10:2aa4dde9:::benchmark_data_smithi107_363200_object82056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.21770498900003, 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.027758696, 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.047 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82057 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818005+0000", 2023-10-09T06:32:29.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818004+0000", 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818027+0000", 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2691e-05 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818033+0000", 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0950000000000001e-06 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818038+0000", 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9620000000000001e-06 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112804+0000", 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29476629399999998 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112813+0000", 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5290000000000008e-06 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834246+0000", 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72143284799999996 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834408+0000", 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000162688 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.834565+0000", 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000156637 2023-10-09T06:32:29.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838810+0000", 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004244772 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842036+0000", 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032265340000000001 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845692+0000", 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036560910000000002 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845731+0000", 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9054000000000003e-05 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845763+0000", 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1931999999999999e-05 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82062 10.6 10:6651e52c:::benchmark_data_smithi107_363200_object82061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.21766747900006, 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.02105132, 2023-10-09T06:32:29.055 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82062 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818042+0000", 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818064+0000", 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2092000000000001e-05 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818070+0000", 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0480000000000004e-06 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818074+0000", 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.476e-06 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.068355+0000", 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25028024399999999 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.068366+0000", 2023-10-09T06:32:29.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1059e-05 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.820857+0000", 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.75249149800000004 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.820938+0000", 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0890999999999995e-05 2023-10-09T06:32:29.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.821038+0000", 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9990999999999999e-05 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830794+0000", 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0097563709999999998 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837309+0000", 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065145050000000003 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838985+0000", 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0016758540000000001 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839051+0000", 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5963000000000005e-05 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839094+0000", 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2735000000000003e-05 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82063 10.14 10:28d0e219:::benchmark_data_smithi107_363200_object82062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.21763074299997, 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0277804100000001, 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.064 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82063 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:32:29.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818079+0000", 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818103+0000", 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.457e-05 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818109+0000", 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.874e-06 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.818114+0000", 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5750000000000002e-06 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112833+0000", 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29471960000000003 2023-10-09T06:32:29.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112839+0000", 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0900000000000001e-06 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835720+0000", 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72288063899999999 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835814+0000", 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4205999999999999e-05 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.835915+0000", 2023-10-09T06:32:29.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010126100000000001 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.841460+0000", 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055449770000000004 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842068+0000", 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00060749199999999995 2023-10-09T06:32:29.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845793+0000", 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0037246670000000001 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845828+0000", 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5938e-05 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845859+0000", 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0892000000000002e-05 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82065 10.14 10:2b997f2c:::benchmark_data_smithi107_363200_object82064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.17124399299996, 2023-10-09T06:32:29.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98147874499999999, 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82065 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:32:29.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864466+0000", 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864463+0000", 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999971 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864505+0000", 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2292000000000001e-05 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864533+0000", 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7350000000000001e-05 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.864546+0000", 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3811e-05 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112859+0000", 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24831273300000001 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112865+0000", 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6740000000000002e-06 2023-10-09T06:32:29.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.836997+0000", 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72413222300000002 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837124+0000", 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012672599999999999 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.837297+0000", 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017315000000000001 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.841468+0000", 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.004171152 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.842344+0000", 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00087613600000000002 2023-10-09T06:32:29.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845891+0000", 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035471579999999999 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845926+0000", 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4802000000000003e-05 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845944+0000", 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8235000000000001e-05 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82067 10.4 10:24dff4b3:::benchmark_data_smithi107_363200_object82066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 543.08432070499998, 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.87997381100000005, 2023-10-09T06:32:29.083 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82067 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951389+0000", 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951387+0000", 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951421+0000", 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4085000000000003e-05 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951440+0000", 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9405000000000001e-05 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.951453+0000", 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2582000000000001e-05 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112887+0000", 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.161433932 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.112894+0000", 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5320000000000001e-06 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.802843+0000", 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.68994917 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.802992+0000", 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014873499999999999 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.803142+0000", 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015000899999999999 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.812320+0000", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091776930000000007 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.819893+0000", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075735519999999999 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831287+0000", 2023-10-09T06:32:29.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011393133999999999 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831328+0000", 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1437000000000003e-05 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831363+0000", 2023-10-09T06:32:29.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4864999999999999e-05 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82072 10.6 10:6072d45f:::benchmark_data_smithi107_363200_object82071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 542.905213463, 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70870548099999997, 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.092 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82072 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130496+0000", 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130494+0000", 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:32:29.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130525+0000", 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1429e-05 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130553+0000", 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7976000000000001e-05 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130570+0000", 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6478e-05 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130627+0000", 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.766e-05 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130641+0000", 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3349e-05 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826121+0000", 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.69548077900000005 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826306+0000", 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018486500000000001 2023-10-09T06:32:29.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.826495+0000", 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018890399999999999 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830806+0000", 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043105299999999999 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.838342+0000", 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075360100000000001 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839133+0000", 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00079077200000000002 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839170+0000", 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7970999999999999e-05 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839202+0000", 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1214999999999998e-05 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.100 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82075 10.6 10:61ef0778:::benchmark_data_smithi107_363200_object82074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 542.903162071, 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.71286330200000003, 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82075 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132548+0000", 2023-10-09T06:32:29.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132545+0000", 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132581+0000", 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5306999999999999e-05 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132596+0000", 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5758999999999999e-05 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132607+0000", 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0243e-05 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132661+0000", 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4249000000000002e-05 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.132673+0000", 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1867e-05 2023-10-09T06:32:29.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.830875+0000", 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.69820269099999999 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831031+0000", 2023-10-09T06:32:29.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000155606 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4", 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.831178+0000", 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014734399999999999 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.836336+0000", 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051574699999999999 2023-10-09T06:32:29.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.839233+0000", 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002896872 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845317+0000", 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0060840479999999999 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845364+0000", 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7309999999999999e-05 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845411+0000", 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6808000000000003e-05 2023-10-09T06:32:29.109 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:49053 11.6 11:6d704939:::benchmark_data_smithi107_418363_object49052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 322.76301184300002, 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31953208300000002, 2023-10-09T06:32:29.110 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 49053 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272698+0000", 2023-10-09T06:32:29.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272696+0000", 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272716+0000", 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9911e-05 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272731+0000", 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5e-05 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272740+0000", 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.816e-06 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272790+0000", 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9577000000000003e-05 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.272963+0000", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173609 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.273120+0000", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015633199999999999 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.275458+0000", 2023-10-09T06:32:29.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023386370000000002 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.279066+0000", 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036078880000000001 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592130+0000", 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31306408099999999 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592188+0000", 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7485999999999998e-05 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592230+0000", 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2120999999999999e-05 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:49069 11.6 11:6dcb3553:::benchmark_data_smithi107_418363_object49068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 322.74706276799998, 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30429343399999997, 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 49069 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.119 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288647+0000", 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288645+0000", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288669+0000", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3502999999999998e-05 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288686+0000", 2023-10-09T06:32:29.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7229000000000001e-05 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288695+0000", 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2599999999999994e-06 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288734+0000", 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8872000000000003e-05 2023-10-09T06:32:29.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288844+0000", 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011034700000000001 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.288944+0000", 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9584000000000001e-05 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.290400+0000", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001455643 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.291170+0000", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000770099 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592831+0000", 2023-10-09T06:32:29.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30166106100000001 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592902+0000", 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0794e-05 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:06.592940+0000", 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8803000000000001e-05 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.125 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96901 11.7 11:e6f829c8:::benchmark_data_smithi107_418363_object6465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 252.32390365000001, 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33668907599999998, 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.126 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96901 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711806+0000", 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711804+0000", 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711813+0000", 2023-10-09T06:32:29.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711826+0000", 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3749e-05 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.711836+0000", 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2690000000000005e-06 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727061+0000", 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015225851 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for rw locks", 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727140+0000", 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8693e-05 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751492+0000", 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024352087000000001 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751556+0000", 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3713999999999993e-05 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:32:29.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.751643+0000", 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6861999999999996e-05 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035341+0000", 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28369838200000003 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035383+0000", 2023-10-09T06:32:29.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1430999999999997e-05 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045288+0000", 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099051139999999996 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045323+0000", 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4962000000000002e-05 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.045372+0000", 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9527000000000002e-05 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.048495+0000", 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031229059999999999 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.134 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96909 11.7 11:e4f104ee:::benchmark_data_smithi107_418363_object6473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 252.31608345800001, 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31569427700000002, 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.135 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96909 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719626+0000", 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719624+0000", 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719635+0000", 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0190000000000001e-05 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719648+0000", 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3749e-05 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.719656+0000", 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6860000000000006e-06 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727160+0000", 2023-10-09T06:32:29.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007503582 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727297+0000", 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013722100000000001 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727445+0000", 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000147851 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.735152+0000", 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077073970000000004 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.737003+0000", 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001850955 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.738237+0000", 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012342570000000001 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.032138+0000", 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29390069400000002 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.032193+0000", 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5451000000000002e-05 2023-10-09T06:32:29.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035321+0000", 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031271390000000001 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:96914 11.7 11:ec492cfc:::benchmark_data_smithi107_418363_object6478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:32:29.144 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 252.308120411, 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.31101062400000001, 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 96914 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727589+0000", 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727588+0000", 2023-10-09T06:32:29.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727595+0000", 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.793e-06 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727606+0000", 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1551e-05 2023-10-09T06:32:29.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.727615+0000", 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.619e-06 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730424+0000", 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028088430000000001 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730560+0000", 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013612499999999999 2023-10-09T06:32:29.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,4,5", 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.730671+0000", 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011068 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741875+0000", 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011204529 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741912+0000", 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6882999999999999e-05 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:16.741920+0000", 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7700000000000001e-06 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035421+0000", 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.293500814 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.035478+0000", 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7027999999999997e-05 2023-10-09T06:32:29.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:28:17.038600+0000", 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031222509999999999 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:40050 12.3 12:c400e986:::benchmark_data_smithi107_479504_object40049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T06:32:29.153 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:31:34.075446+0000", 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 54.960264049999999, 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34026955599999997, 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 40050 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075446+0000", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075446+0000", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075444+0000", 2023-10-09T06:32:29.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075488+0000", 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3683000000000002e-05 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075500+0000", 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2266000000000001e-05 2023-10-09T06:32:29.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075507+0000", 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0999999999999998e-06 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075550+0000", 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2549999999999997e-05 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075655+0000", 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010542700000000001 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075751+0000", 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5853999999999994e-05 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.408623+0000", 2023-10-09T06:32:29.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33287190100000003 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.408658+0000", 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5098999999999997e-05 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.415622+0000", 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069639039999999999 2023-10-09T06:32:29.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.415676+0000", 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3619999999999998e-05 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.415715+0000", 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9650999999999997e-05 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.160 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:40055 12.e 12:790fd0b3:::benchmark_data_smithi107_479504_object40054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:31:34.075519+0000", 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 54.960190496000003, 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33389081399999998, 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.161 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 40055 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075519+0000", 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075519+0000", 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075519+0000", 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075538+0000", 2023-10-09T06:32:29.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9556999999999999e-05 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075545+0000", 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6830000000000002e-06 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075551+0000", 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9050000000000002e-06 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075595+0000", 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3951e-05 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.075730+0000", 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013534800000000001 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:32:29.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.407065+0000", 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.331335147 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.409290+0000", 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022242859999999998 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.409297+0000", 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8730000000000007e-06 2023-10-09T06:32:29.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.409329+0000", 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1424999999999998e-05 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.409373+0000", 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3956999999999999e-05 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.409410+0000", 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7194000000000003e-05 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:40051 12.6 e368/361 12:62084569:::benchmark_data_smithi107_479504_object40050:head v 368'2447, mlcod=368'2445)", 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:31:34.076564+0000", 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 54.959145960000001, 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33720618099999999, 2023-10-09T06:32:29.168 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.076564+0000", 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.076564+0000", 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.076563+0000", 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.076590+0000", 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6840000000000001e-05 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.170 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.406975+0000", 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33038483600000002 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.406990+0000", 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5092000000000001e-05 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.171 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.407871+0000", 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00088086000000000002 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.407892+0000", 2023-10-09T06:32:29.172 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1857000000000001e-05 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.413735+0000", 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058421979999999998 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.413770+0000", 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5274000000000001e-05 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.173 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.46340.0:40050 12.3 e368/361 ondisk, result = 0)", 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:31:34.077636+0000", 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 54.958073247999998, 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33098936899999998, 2023-10-09T06:32:29.174 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.077636+0000", 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.077636+0000", 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.175 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.077636+0000", 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.176 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.077642+0000", 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8760000000000002e-06 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.406980+0000", 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32933851200000003 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.406991+0000", 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0481e-05 2023-10-09T06:32:29.177 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.408609+0000", 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001618427 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.178 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.408617+0000", 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7479999999999993e-06 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:31:34.408626+0000", 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1139999999999995e-06 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.179 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:86632 12.3 12:c7c62f22:::benchmark_data_smithi107_479504_object86631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:23.876533+0000", 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 5.1591770669999999, 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28630518900000002, 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:32:29.180 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 86632 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876533+0000", 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.181 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.182 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:28 smithi150 ceph-mon[194178]: pgmap v1189: 119 pgs: 119 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 2.6 KiB/s rd, 48 MiB/s wr, 768 op/s 2023-10-09T06:32:29.183 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876533+0000", 2023-10-09T06:32:29.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.183 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876532+0000", 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876553+0000", 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0562000000000001e-05 2023-10-09T06:32:29.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876559+0000", 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.876565+0000", 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6620000000000002e-06 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.879733+0000", 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031683240000000001 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.879869+0000", 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000135418 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:32:29.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.879988+0000", 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011963099999999999 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883243+0000", 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032548020000000002 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883266+0000", 2023-10-09T06:32:29.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2722e-05 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162742+0000", 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27947633300000002 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162796+0000", 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3504999999999997e-05 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162838+0000", 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2261000000000001e-05 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:86635 12.3 12:ce3bdbd2:::benchmark_data_smithi107_479504_object86634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:23.883263+0000", 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 5.1524468990000001, 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28590502099999998, 2023-10-09T06:32:29.189 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 86635 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883263+0000", 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883263+0000", 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883262+0000", 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883322+0000", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0509e-05 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883331+0000", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2059999999999996e-06 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883341+0000", 2023-10-09T06:32:29.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0142e-05 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883382+0000", 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0234000000000001e-05 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883529+0000", 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014736600000000001 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.883670+0000", 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000141328 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.165324+0000", 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.281653551 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.165860+0000", 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00053630400000000003 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.169087+0000", 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032265689999999999 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.169134+0000", 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7163000000000002e-05 2023-10-09T06:32:29.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.169168+0000", 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3918999999999998e-05 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.46340.0:86632 12.3 e387/361 ondisk, result = 0)", 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:23.884504+0000", 2023-10-09T06:32:29.196 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 5.1512052199999996, 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.278341322, 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.884504+0000", 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:29.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.884504+0000", 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.884504+0000", 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:23.884509+0000", 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2310000000000004e-06 2023-10-09T06:32:29.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.158737+0000", 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27422768800000002 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.158752+0000", 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5517000000000001e-05 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162727+0000", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0039750189999999998 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162736+0000", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3899999999999993e-06 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:24.162846+0000", 2023-10-09T06:32:29.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110213 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:29.201 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:32:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:29.202 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:29.202 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:29.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:29.303 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:29.372 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:29.842 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 12 pg_num to 26 2023-10-09T06:32:29.986 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 91053 91037 72.9005 34.6875 0.012549 0.0137122 2023-10-09T06:32:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:32:30.157 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:32:30.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:32:30.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:32:30.465 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:32:30.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[212750]: pgmap v1190: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T06:32:30.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:32:30.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:32:30.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[212750]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:32:30.842 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[212750]: failed to encode map e389 with expected crc 2023-10-09T06:32:30.990 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 91670 91654 72.4636 38.5625 0.1271 0.0137836 2023-10-09T06:32:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[220921]: pgmap v1190: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T06:32:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[220921]: failed to encode map e389 with expected crc 2023-10-09T06:32:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:32:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[220921]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:32:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:30 smithi107 ceph-mon[220921]: failed to encode map e389 with expected crc 2023-10-09T06:32:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:30 smithi150 ceph-mon[194178]: pgmap v1190: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s 2023-10-09T06:32:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:30 smithi150 ceph-mon[194178]: failed to encode map e389 with expected crc 2023-10-09T06:32:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/272105540' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:32:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:30 smithi150 ceph-mon[194178]: osdmap e389: 8 total, 8 up, 8 in 2023-10-09T06:32:31.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:30 smithi150 ceph-mon[194178]: failed to encode map e389 with expected crc 2023-10-09T06:32:31.340 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:31.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:31.450 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:31.484 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:31.540 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:31.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:31.647 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.679 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:31.741 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:31.742 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:31.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:31.849 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:31.893 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:31.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:32:31.996256+0000 min lat: 0.00111887 max lat: 0.351321 avg lat: 0.0138453 2023-10-09T06:32:31.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:32:31.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 92382 92366 72.1086 44.5 0.0315346 0.0138453 2023-10-09T06:32:32.097 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:32:32.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:32.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:32.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:32:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:32.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:32:32.160 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:32.161 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:32] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T06:32:32.207 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:32.238 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:32.306 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:32.403 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:32:32.427 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:32:32.446 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:32:32.465 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:32:32.484 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:32:32.504 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:32:32.523 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:32:32.542 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:32:32.544 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.544 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.544 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.544 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.545 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.545 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.545 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.545 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:32:32.586 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:32.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:32.690 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:32.719 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:32.764 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:32.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:32.873 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:32.902 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:32.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 92391 92375 71.2259 0.5625 1.08826 0.0138789 2023-10-09T06:32:33.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: pgmap v1192: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s 2023-10-09T06:32:33.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:32:33.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:32:33.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:33.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2043691645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: pgmap v1192: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:33.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2043691645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:33.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: pgmap v1192: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 620 op/s 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: osdmap e390: 8 total, 8 up, 8 in 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: failed to encode map e390 with expected crc 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:33.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2043691645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:32:33.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[212750]: failed to encode map e391 with expected crc 2023-10-09T06:32:33.995 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 93269 93253 71.0265 54.875 0.00453384 0.0140744 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:33 smithi150 ceph-mon[194178]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: osdmap e391: 8 total, 8 up, 8 in 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.157 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:33 smithi107 ceph-mon[220921]: failed to encode map e391 with expected crc 2023-10-09T06:32:34.172 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:34.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:34.283 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:34.388 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:34.753 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:34.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:34.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:34.862 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:34.888 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:34.933 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:34.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:34.996 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 94313 94297 70.9568 65.25 0.00984761 0.0140887 2023-10-09T06:32:35.042 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:35.107 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:35.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:34 smithi107 ceph-mon[212750]: pgmap v1195: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T06:32:35.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:34 smithi107 ceph-mon[220921]: pgmap v1195: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T06:32:35.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:34 smithi150 ceph-mon[194178]: pgmap v1195: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s 2023-10-09T06:32:35.411 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:35.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:35.519 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:35.564 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:35.621 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:35.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:35.680 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:32:35.680 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:32:35.726 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:35.771 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:35.837 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:35.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:35.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:35.945 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:35.997 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 95500 95484 70.9939 74.1875 0.0133444 0.0140814 2023-10-09T06:32:36.053 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:36.385 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:32:36.438 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:32:36.439 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:32:36.788 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:36.795 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:32:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:36.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:36.901 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:36.951 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:32:36.998 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 96298 96282 70.7445 49.875 0.00622821 0.0141303 2023-10-09T06:32:37.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:36 smithi107 ceph-mon[212750]: pgmap v1196: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2023-10-09T06:32:37.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:37.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:37.027 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:36 smithi107 ceph-mon[220921]: pgmap v1196: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2023-10-09T06:32:37.027 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:37.155 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:37.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:37.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:37.268 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:37.294 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:36 smithi150 ceph-mon[194178]: pgmap v1196: 129 pgs: 1 activating, 3 peering, 7 unknown, 118 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 514 op/s 2023-10-09T06:32:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:32:37.722 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:37.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:37.828 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:37.846 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:37.851 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:37.859 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:37.912 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:37.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:37.960 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:37.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 97141 97125 70.5346 52.6875 0.0121263 0.014173 2023-10-09T06:32:38.018 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:38.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[220921]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:37 smithi107 ceph-mon[212750]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.099 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/544117635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: osdmap e392: 8 total, 8 up, 8 in 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:37 smithi150 ceph-mon[194178]: failed to encode map e392 with expected crc 2023-10-09T06:32:38.269 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:38.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:38.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:38.380 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.425 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:38.486 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:38.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:38.592 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.624 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:38.702 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:38.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:38.811 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:38.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:38.890 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:38.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:38.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 98235 98219 70.5097 68.375 0.012323 0.0141776 2023-10-09T06:32:39.000 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:39.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:38 smithi150 ceph-mon[194178]: pgmap v1198: 129 pgs: 1 activating, 3 peering, 2 unknown, 123 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:39.046 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:39.088 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:39.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:39.195 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:39.226 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:39.268 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:39.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:39.376 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:39.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:38 smithi107 ceph-mon[212750]: pgmap v1198: 129 pgs: 1 activating, 3 peering, 2 unknown, 123 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:39.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:38 smithi107 ceph-mon[220921]: pgmap v1198: 129 pgs: 1 activating, 3 peering, 2 unknown, 123 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 724 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:39.421 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:39.484 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:39.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:39.592 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:39.631 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:39.840 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:32:39.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 99034 99018 70.276 49.9375 0.0167581 0.0142243 2023-10-09T06:32:40.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:32:40.103 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:32:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:32:40.411 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:40.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:40.521 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:40.551 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:40.598 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:40.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:40.703 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:40.733 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:41.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 99608 99592 69.8897 35.875 0.00229151 0.0141931 2023-10-09T06:32:41.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:41.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: pgmap v1199: 129 pgs: 129 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 26 KiB/s, 0 objects/s recovering 2023-10-09T06:32:41.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:32:41.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:32:41.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[212750]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:41.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:41.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: pgmap v1199: 129 pgs: 129 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 26 KiB/s, 0 objects/s recovering 2023-10-09T06:32:41.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:32:41.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:32:41.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:40 smithi107 ceph-mon[220921]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.130 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:41.135 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:41.159 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:41.254 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:41.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:41.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:41.359 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:41.388 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: pgmap v1199: 129 pgs: 129 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 26 KiB/s, 0 objects/s recovering 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: osdmap e393: 8 total, 8 up, 8 in 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:40 smithi150 ceph-mon[194178]: failed to encode map e393 with expected crc 2023-10-09T06:32:41.510 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:32:41.533 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:32:41.537 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:41.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:41.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:41.555 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:32:41.577 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:32:41.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:32:41.614 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:32:41.634 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:32:41.642 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:41.654 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.656 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:32:41.669 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:41.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:41.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:41.818 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:41.845 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:41.885 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:32:41.885 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:32:41.885 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:32:41.886 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:32:41.886 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:32:41.887 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:41.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:41.974 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:32:41.992 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:41.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 90 16 99608 99592 69.1135 0 - 0.0141931 2023-10-09T06:32:42.018 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:42.060 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:42.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:42.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:42] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2023-10-09T06:32:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4092488374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2268671248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4092488374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2268671248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.165 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:42.192 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:42.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:42.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: osdmap e394: 8 total, 8 up, 8 in 2023-10-09T06:32:42.238 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: failed to encode map e394 with expected crc 2023-10-09T06:32:42.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4092488374' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:32:42.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2268671248' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.239 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:32:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:42.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:42.341 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:42.376 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:42.999 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.7322 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 99608 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 68.614 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 33.3442 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 192.938 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1097 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 533.507 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3087 2023-10-09T06:32:43.000 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:32:43.001 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0145673 2023-10-09T06:32:43.001 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0390268 2023-10-09T06:32:43.001 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.54561 2023-10-09T06:32:43.001 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00111887 2023-10-09T06:32:43.004 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:32:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: pgmap v1202: 129 pgs: 1 unknown, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 365 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:32:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[212750]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: pgmap v1202: 129 pgs: 1 unknown, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 365 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:32:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:32:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:42 smithi107 ceph-mon[220921]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: pgmap v1202: 129 pgs: 1 unknown, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 365 op/s; 32 KiB/s, 0 objects/s recovering 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: osdmap e395: 8 total, 8 up, 8 in 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: failed to encode map e395 with expected crc 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:32:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:42 smithi150 ceph-mon[194178]: failed to encode map e395 with expected crc 2023-10-09T06:32:44.042 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:44.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:44.154 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:44.231 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:44.658 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:32:44.743 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:44.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:44.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:44.880 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:44.938 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:44.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:44.969 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:44 smithi107 ceph-mon[220921]: pgmap v1204: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 23 MiB/s wr, 364 op/s 2023-10-09T06:32:45.045 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:45.084 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:45.136 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:45.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:44 smithi150 ceph-mon[194178]: pgmap v1204: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 23 MiB/s wr, 364 op/s 2023-10-09T06:32:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:45.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:45.242 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:45.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:44 smithi107 ceph-mon[212750]: pgmap v1204: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 23 MiB/s wr, 364 op/s 2023-10-09T06:32:45.269 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:45.480 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:32:45.588 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:45.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:45.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:45.701 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:45.850 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:45.917 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:45.993 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:46.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:46.102 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:46.201 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:46.478 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:46.481 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:32:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:46.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:46.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:46.678 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:46.821 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:46.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:46.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:46.962 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:47.016 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:32:47.024 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:47.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:47.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[220921]: pgmap v1205: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 3.0 MiB/s wr, 54 op/s 2023-10-09T06:32:47.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:47.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/516122622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[212750]: pgmap v1205: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 3.0 MiB/s wr, 54 op/s 2023-10-09T06:32:47.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:47.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/516122622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.135 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:47.167 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:47.227 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:47.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:46 smithi150 ceph-mon[194178]: pgmap v1205: 129 pgs: 1 unknown, 128 active+clean; 5.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 3.0 MiB/s wr, 54 op/s 2023-10-09T06:32:47.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:32:47.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/516122622' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:32:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:47.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:47.333 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:47.367 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:47.432 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:47.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:47.543 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:32:47.572 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:48.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:48.308 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[212750]: 12.8 scrub starts 2023-10-09T06:32:48.309 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:32:48.310 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:48 smithi107 ceph-mon[220921]: 12.8 scrub starts 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: osdmap e396: 8 total, 8 up, 8 in 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: failed to encode map e396 with expected crc 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:32:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:32:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:48 smithi150 ceph-mon[194178]: 12.8 scrub starts 2023-10-09T06:32:49.098 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:32:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:49.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:32:49.186 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:49.202 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:49.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[212750]: pgmap v1207: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 160 B/s rd, 52 MiB/s wr, 1.08k op/s; 8867/294234 objects misplaced (3.014%); 901 KiB/s, 14 objects/s recovering 2023-10-09T06:32:49.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:32:49.215 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[212750]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[212750]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[220921]: pgmap v1207: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 160 B/s rd, 52 MiB/s wr, 1.08k op/s; 8867/294234 objects misplaced (3.014%); 901 KiB/s, 14 objects/s recovering 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[220921]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[220921]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:32:49.216 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:49 smithi107 ceph-mon[220921]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.235 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:49.283 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:49.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:49.389 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:49 smithi150 ceph-mon[194178]: pgmap v1207: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 160 B/s rd, 52 MiB/s wr, 1.08k op/s; 8867/294234 objects misplaced (3.014%); 901 KiB/s, 14 objects/s recovering 2023-10-09T06:32:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:49 smithi150 ceph-mon[194178]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:49 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:32:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:49 smithi150 ceph-mon[194178]: osdmap e397: 8 total, 8 up, 8 in 2023-10-09T06:32:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:49 smithi150 ceph-mon[194178]: failed to encode map e397 with expected crc 2023-10-09T06:32:49.414 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:49.455 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:49.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:49.560 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:49.603 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:49.606 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:49.665 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:49.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:49.772 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:49.798 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:49.840 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:49.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:49.947 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:32:49.973 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:50.015 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:32:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:50.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:32:50.041 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:32:50.121 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:50.152 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: 12.1 scrub starts 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1909191221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:50 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:50.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: 12.1 scrub starts 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1909191221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:50.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: 12.1 scrub starts 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: osdmap e398: 8 total, 8 up, 8 in 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: failed to encode map e398 with expected crc 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1909191221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:50.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:50 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:32:51.215 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:32:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 34 MiB/s wr, 1.07k op/s; 8867/289293 objects misplaced (3.065%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:32:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:32:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:32:51.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:32:51.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[212750]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 34 MiB/s wr, 1.07k op/s; 8867/289293 objects misplaced (3.065%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:32:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:32:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:32:51.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:32:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:51 smithi107 ceph-mon[220921]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: pgmap v1210: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 170 B/s rd, 34 MiB/s wr, 1.07k op/s; 8867/289293 objects misplaced (3.065%); 959 KiB/s, 14 objects/s recovering 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: osdmap e399: 8 total, 8 up, 8 in 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: failed to encode map e399 with expected crc 2023-10-09T06:32:51.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:51 smithi150 ceph-mon[194178]: failed to encode map e399 with expected crc 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:32:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:32:52] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:32:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:32:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: osdmap e400: 8 total, 8 up, 8 in 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: failed to encode map e400 with expected crc 2023-10-09T06:32:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:32:52.592 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:52.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:52.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:52.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:52.766 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 8318/286863 objects misplaced (2.900%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: 12.9 scrub starts 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[212750]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 8318/286863 objects misplaced (2.900%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: 12.9 scrub starts 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:53 smithi107 ceph-mon[220921]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:32:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: pgmap v1213: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 8318/286863 objects misplaced (2.900%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T06:32:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: 12.9 scrub starts 2023-10-09T06:32:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:32:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: osdmap e401: 8 total, 8 up, 8 in 2023-10-09T06:32:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: failed to encode map e401 with expected crc 2023-10-09T06:32:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:53 smithi150 ceph-mon[194178]: osdmap e402: 8 total, 8 up, 8 in 2023-10-09T06:32:54.312 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:54.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:32:54.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:54 smithi107 ceph-mon[220921]: failed to encode map e403 with expected crc 2023-10-09T06:32:54.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: failed to encode map e402 with expected crc 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: osdmap e403: 8 total, 8 up, 8 in 2023-10-09T06:32:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:54 smithi150 ceph-mon[194178]: failed to encode map e403 with expected crc 2023-10-09T06:32:54.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:32:54.429 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:54.479 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:54.530 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:54.635 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:32:54.663 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:54.714 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:32:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:54.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:32:54.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:54.928 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[220921]: pgmap v1216: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 6212/282921 objects degraded (2.196%); 13950/282921 objects misplaced (4.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:32:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[220921]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[220921]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 6212/282921 objects degraded (2.196%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:32:55.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[212750]: pgmap v1216: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 6212/282921 objects degraded (2.196%); 13950/282921 objects misplaced (4.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:32:55.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[212750]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:55 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 6212/282921 objects degraded (2.196%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82045 10.12 10:48c729cb:::benchmark_data_smithi107_363200_object82044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.809945+0000", 2023-10-09T06:32:55.355 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 569.41136085000005, 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0562037929999999, 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82045 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.809945+0000", 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.809945+0000", 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.809942+0000", 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999971 2023-10-09T06:32:55.357 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.809978+0000", 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5964000000000003e-05 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810019+0000", 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0889000000000001e-05 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.358 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810034+0000", 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4911e-05 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.990548+0000", 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18051365599999999 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.359 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.990554+0000", 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6189999999999996e-06 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.866036+0000", 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.87548160900000005 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.866149+0000", 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000112958 2023-10-09T06:32:55.360 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82058 10.2 10:47b9b366:::benchmark_data_smithi107_363200_object82057:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:25.810051+0000", 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 569.41125528099997, 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0441413479999999, 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.361 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82058 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810051+0000", 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810051+0000", 2023-10-09T06:32:55.362 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810050+0000", 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810073+0000", 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2815000000000001e-05 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.363 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810080+0000", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2910000000000002e-06 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810086+0000", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7379999999999999e-06 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810156+0000", 2023-10-09T06:32:55.364 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9931000000000005e-05 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.810164+0000", 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6790000000000005e-06 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.990402+0000", 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18023814299999999 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.365 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:25.990414+0000", 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1963999999999999e-05 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.845892+0000", 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.85547827700000001 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:32:55.366 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.846703+0000", 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00081055199999999997 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854089+0000", 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073866130000000002 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854192+0000", 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010281999999999999 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.367 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:82074 10.2 10:426dbcf6:::benchmark_data_smithi107_363200_object82073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:23:26.130554+0000", 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 569.09075199699998, 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72387427999999998, 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.368 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 82074 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130554+0000", 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130554+0000", 2023-10-09T06:32:55.369 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130552+0000", 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130579+0000", 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7404999999999999e-05 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.370 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130603+0000", 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3275e-05 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130613+0000", 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0796e-05 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.371 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130669+0000", 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5998000000000002e-05 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.130680+0000", 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0750999999999999e-05 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.846783+0000", 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.71610316799999996 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.372 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.846936+0000", 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015267199999999999 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.847105+0000", 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016921899999999999 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.373 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.853629+0000", 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065238630000000004 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854273+0000", 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00064380100000000005 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854304+0000", 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0811e-05 2023-10-09T06:32:55.374 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854343+0000", 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9585999999999997e-05 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:23:26.854428+0000", 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5043999999999994e-05 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.375 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150210 10.13 10:ce565ba0:::benchmark_data_smithi107_363200_object46118:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.766630+0000", 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.45467596100002, 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.007640125, 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.376 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150210 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.766630+0000", 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.766630+0000", 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.377 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.766628+0000", 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.766639+0000", 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0581000000000001e-05 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.378 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.786068+0000", 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019428675999999999 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.786077+0000", 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2059999999999996e-06 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.791939+0000", 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058620740000000001 2023-10-09T06:32:55.379 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.791949+0000", 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9369999999999999e-06 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.774165+0000", 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98221620600000004 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.380 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.774270+0000", 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105159 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150217 10.13 10:c89f2f3e:::benchmark_data_smithi107_363200_object46125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.787376+0000", 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.43392974099999, 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98740550199999999, 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.381 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150217 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787376+0000", 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.382 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787376+0000", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787375+0000", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787381+0000", 2023-10-09T06:32:55.383 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2260000000000002e-06 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787394+0000", 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3038e-05 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787403+0000", 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8009999999999993e-06 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.384 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.791969+0000", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045659189999999999 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.791976+0000", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5899999999999996e-06 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.774715+0000", 2023-10-09T06:32:55.385 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98273929299999996 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.774782+0000", 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6843999999999999e-05 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.386 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150223 10.13 10:cbd36455:::benchmark_data_smithi107_363200_object46131:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.787409+0000", 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.43389713300002, 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99706505400000001, 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150223 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.387 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787409+0000", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787409+0000", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787408+0000", 2023-10-09T06:32:55.388 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787411+0000", 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9129999999999999e-06 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787418+0000", 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7190000000000004e-06 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.389 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.787428+0000", 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9320000000000008e-06 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.791995+0000", 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045668640000000003 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.390 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.792001+0000", 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4400000000000002e-06 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.774891+0000", 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98289014600000002 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.775001+0000", 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000110009 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.391 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.775370+0000", 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00036863299999999999 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.779847+0000", 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044766670000000001 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.392 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784376+0000", 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045297150000000001 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784388+0000", 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1770000000000001e-05 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784434+0000", 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5658999999999998e-05 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.393 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784474+0000", 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0012000000000002e-05 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150274 10.13 10:c9066bfd:::benchmark_data_smithi107_363200_object46182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.836478+0000", 2023-10-09T06:32:55.394 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.384828352, 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94814356399999999, 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150274 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836478+0000", 2023-10-09T06:32:55.395 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836478+0000", 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836476+0000", 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.396 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836484+0000", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1170000000000004e-06 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836497+0000", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2879e-05 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.836507+0000", 2023-10-09T06:32:55.397 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8579999999999995e-06 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.837452+0000", 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00094520200000000002 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.837463+0000", 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1021e-05 2023-10-09T06:32:55.398 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.777489+0000", 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94002582499999998 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.777609+0000", 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000120238 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.399 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.777944+0000", 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00033495900000000002 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784485+0000", 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065405760000000002 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784525+0000", 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0380000000000003e-05 2023-10-09T06:32:55.400 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784554+0000", 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8326999999999998e-05 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784590+0000", 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6538000000000002e-05 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.401 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784621+0000", 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1114000000000001e-05 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150323 10.13 10:ca1dabe2:::benchmark_data_smithi107_363200_object46231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.888426+0000", 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.33288040299999, 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.90103385400000002, 2023-10-09T06:32:55.402 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150323 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888426+0000", 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.403 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888426+0000", 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888424+0000", 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.404 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888432+0000", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.588e-06 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888445+0000", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3061000000000001e-05 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888452+0000", 2023-10-09T06:32:55.405 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8960000000000003e-06 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888509+0000", 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6045999999999998e-05 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.888519+0000", 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0412e-05 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.406 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.779928+0000", 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.89140888900000004 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.780016+0000", 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7789999999999998e-05 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.407 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.780307+0000", 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000291019 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784657+0000", 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043502469999999998 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784710+0000", 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2834999999999998e-05 2023-10-09T06:32:55.408 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789373+0000", 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0046629990000000001 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789419+0000", 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6128000000000003e-05 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.409 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789459+0000", 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0608000000000002e-05 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150369 10.13 10:cbea6d26:::benchmark_data_smithi107_363200_object46277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.948436+0000", 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.27286999, 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.843642373, 2023-10-09T06:32:55.410 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150369 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948436+0000", 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.411 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948436+0000", 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948434+0000", 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.412 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948444+0000", 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.046e-05 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948469+0000", 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4544000000000001e-05 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.948480+0000", 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1365e-05 2023-10-09T06:32:55.413 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.951193+0000", 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027125119999999998 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.951204+0000", 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1912e-05 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.414 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.781910+0000", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83070561300000001 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.782023+0000", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000113416 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.782339+0000", 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00031543400000000002 2023-10-09T06:32:55.415 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789467+0000", 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0071279869999999997 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.791967+0000", 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00249964 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.416 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.791999+0000", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2738999999999999e-05 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792039+0000", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9969000000000003e-05 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792078+0000", 2023-10-09T06:32:55.417 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9127e-05 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150378 10.13 10:caf1ca97:::benchmark_data_smithi107_363200_object46286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:51.962476+0000", 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.25883022199997, 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.82978466500000003, 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.418 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150378 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962476+0000", 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.419 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962476+0000", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962475+0000", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962481+0000", 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2049999999999999e-06 2023-10-09T06:32:55.420 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962495+0000", 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4012999999999999e-05 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962503+0000", 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1310000000000006e-06 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.421 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962576+0000", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2426000000000001e-05 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:51.962590+0000", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4498000000000001e-05 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784750+0000", 2023-10-09T06:32:55.422 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.82215961599999998 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.784880+0000", 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000129956 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.785184+0000", 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030445500000000003 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.423 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.791868+0000", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0066841310000000003 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792119+0000", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00025052100000000002 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792182+0000", 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3458000000000001e-05 2023-10-09T06:32:55.424 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792213+0000", 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0508999999999999e-05 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792260+0000", 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7703000000000001e-05 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.425 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150407 10.13 10:cb7a2cb7:::benchmark_data_smithi107_363200_object46315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:52.007106+0000", 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.21420025700002, 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.79521011100000005, 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.426 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150407 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007106+0000", 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007106+0000", 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007104+0000", 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007112+0000", 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6860000000000006e-06 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.428 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007131+0000", 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9244000000000001e-05 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007141+0000", 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7280000000000007e-06 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007208+0000", 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7558000000000006e-05 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.007223+0000", 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4803e-05 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.787107+0000", 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.779884401 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.787208+0000", 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100493 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.787522+0000", 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00031411500000000001 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.792150+0000", 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0046281619999999999 2023-10-09T06:32:55.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.793368+0000", 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0012182219999999999 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802218+0000", 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088496019999999998 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802272+0000", 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4070999999999998e-05 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802316+0000", 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3806e-05 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.433 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150409 10.13 10:c8e00228:::benchmark_data_smithi107_363200_object46317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:52.016564+0000", 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.20474163, 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.78582707299999999, 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150409 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.434 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016564+0000", 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016564+0000", 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016562+0000", 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016572+0000", 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4190000000000008e-06 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016587+0000", 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5276000000000001e-05 2023-10-09T06:32:55.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016598+0000", 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0342e-05 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016665+0000", 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7522999999999997e-05 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.016677+0000", 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1953999999999999e-05 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789492+0000", 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.77281506600000005 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789599+0000", 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106709 2023-10-09T06:32:55.438 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.789935+0000", 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00033597299999999999 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.793397+0000", 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034618610000000001 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.439 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802209+0000", 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088122400000000007 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802326+0000", 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000117229 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802362+0000", 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5747000000000002e-05 2023-10-09T06:32:55.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.802391+0000", 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9606000000000001e-05 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150468 10.13 10:cf268f23:::benchmark_data_smithi107_363200_object46376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:52.101221+0000", 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.12008507100001, 2023-10-09T06:32:55.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.71028359799999996, 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150468 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101221+0000", 2023-10-09T06:32:55.442 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101221+0000", 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101220+0000", 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.443 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101226+0000", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6329999999999999e-06 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101252+0000", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5517e-05 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101259+0000", 2023-10-09T06:32:55.444 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4250000000000001e-06 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101320+0000", 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0788000000000001e-05 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.101331+0000", 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1001e-05 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.445 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.793417+0000", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.69208583000000001 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.793526+0000", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108903 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.793910+0000", 2023-10-09T06:32:55.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00038440600000000001 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.803601+0000", 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0096911519999999998 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811337+0000", 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077360950000000001 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811417+0000", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9548000000000005e-05 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811467+0000", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9774000000000002e-05 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811505+0000", 2023-10-09T06:32:55.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7802999999999997e-05 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45650.0:150520 10.13 10:ca5082dc:::benchmark_data_smithi107_363200_object46428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:24:52.186773+0000", 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 483.03453299199998, 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.62762002800000005, 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.449 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45650", 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150520 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186773+0000", 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.450 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186773+0000", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186772+0000", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186779+0000", 2023-10-09T06:32:55.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9e-06 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186792+0000", 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3570000000000001e-05 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186799+0000", 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3100000000000003e-06 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.452 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186849+0000", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9802000000000002e-05 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.186858+0000", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6149999999999997e-06 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.795850+0000", 2023-10-09T06:32:55.453 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.60899235500000004 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.795988+0000", 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000138104 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.796311+0000", 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032311800000000001 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.454 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811406+0000", 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015094273 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.811553+0000", 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014756 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.455 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.814310+0000", 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002757013 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.814354+0000", 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3937999999999998e-05 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:24:52.814393+0000", 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8840000000000001e-05 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.456 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67277 11.0 11:0d53defb:::benchmark_data_smithi107_418363_object67276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.722782+0000", 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 330.498524041, 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3541648340000001, 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.457 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67277 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722782+0000", 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722782+0000", 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.458 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722781+0000", 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722801+0000", 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0103e-05 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.459 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722809+0000", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1140000000000006e-06 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.722815+0000", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0059999999999998e-06 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.843008+0000", 2023-10-09T06:32:55.460 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.120193091 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.843158+0000", 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000150135 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.843323+0000", 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016483200000000001 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.461 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.847781+0000", 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044583319999999997 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.848019+0000", 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000237569 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.462 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875946+0000", 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027927395000000001 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875956+0000", 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3659999999999998e-06 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.858756+0000", 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98280071000000002 2023-10-09T06:32:55.463 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.858772+0000", 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.521e-05 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.076832+0000", 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2180602140000001 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.464 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.076947+0000", 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011487 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67282 11.0 11:0cd89695:::benchmark_data_smithi107_418363_object67281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.723759+0000", 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 330.49754665, 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3570101139999999, 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.465 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67282 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.723759+0000", 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.466 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.723759+0000", 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.723758+0000", 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.467 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.723808+0000", 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.969e-05 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.842947+0000", 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.119139331 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.842961+0000", 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4229999999999999e-05 2023-10-09T06:32:55.468 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.845308+0000", 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002346688 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.845434+0000", 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012607699999999999 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:32:55.469 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.845572+0000", 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013727699999999999 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.848525+0000", 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029538989999999998 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.849953+0000", 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001427577 2023-10-09T06:32:55.470 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875974+0000", 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.026020623 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875978+0000", 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.809e-06 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.471 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.859869+0000", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98389038399999995 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.859890+0000", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1501999999999998e-05 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.080675+0000", 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2207851110000001 2023-10-09T06:32:55.472 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.080769+0000", 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4016000000000002e-05 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67293 11.b 11:d780638d:::benchmark_data_smithi107_418363_object67292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.875592+0000", 2023-10-09T06:32:55.473 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 330.34571380199998, 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4123048329999999, 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67293 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.474 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875592+0000", 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875592+0000", 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875591+0000", 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.475 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875614+0000", 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3611000000000001e-05 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875623+0000", 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0750000000000004e-06 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.476 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875633+0000", 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4569999999999994e-06 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876502+0000", 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00086930600000000003 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876508+0000", 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.823e-06 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.477 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.860423+0000", 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98391472899999999 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.860429+0000", 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3130000000000004e-06 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.478 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.268986+0000", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4085571100000001 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.269741+0000", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075427000000000003 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.287802+0000", 2023-10-09T06:32:55.479 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.018061905 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.287897+0000", 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4599999999999996e-05 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67297 11.b 11:d8877d63:::benchmark_data_smithi107_418363_object67296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.875998+0000", 2023-10-09T06:32:55.480 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 330.34530762200001, 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4162915709999999, 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67297 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.481 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875998+0000", 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875998+0000", 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875998+0000", 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:32:55.482 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876021+0000", 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3614000000000001e-05 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876030+0000", 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1509999999999993e-06 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.483 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876037+0000", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9589999999999998e-06 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876522+0000", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000485019 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.876529+0000", 2023-10-09T06:32:55.484 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0790000000000003e-06 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.860442+0000", 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98391302999999997 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.860448+0000", 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.784e-06 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.485 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.272348+0000", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.411900282 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.273257+0000", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00090868700000000004 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.292164+0000", 2023-10-09T06:32:55.486 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.018906644 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.292290+0000", 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000126153 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125500 11.6 11:66b58078:::benchmark_data_smithi107_418363_object35064:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:32:55.487 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 231.90143913899999, 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3024929169999999, 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125500 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.488 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319865+0000", 2023-10-09T06:32:55.489 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319874+0000", 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9840000000000007e-06 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319910+0000", 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5954000000000002e-05 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.490 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319921+0000", 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1443e-05 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319979+0000", 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7470999999999999e-05 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.491 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319991+0000", 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2768e-05 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381006+0000", 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0610140829999999 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381014+0000", 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5120000000000008e-06 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.492 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.575532+0000", 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1945181199999999 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.580920+0000", 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053880309999999997 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:32:55.493 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.581283+0000", 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00036236999999999999 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.605300+0000", 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024017113 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.612306+0000", 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007006745 2023-10-09T06:32:55.494 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.619553+0000", 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072465910000000001 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.619603+0000", 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0312000000000002e-05 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.495 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.622360+0000", 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027564299999999998 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125514 11.6 11:6f06152b:::benchmark_data_smithi107_418363_object35078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 231.86503213, 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3080845179999998, 2023-10-09T06:32:55.496 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125514 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.497 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356272+0000", 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:32:55.498 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356280+0000", 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0530000000000007e-06 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356291+0000", 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0876000000000001e-05 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356299+0000", 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6820000000000003e-06 2023-10-09T06:32:55.499 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356354+0000", 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5281000000000001e-05 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356364+0000", 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9359999999999994e-06 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.500 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381034+0000", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0246703669999999 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381040+0000", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2890000000000003e-06 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626453+0000", 2023-10-09T06:32:55.501 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.245412467 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626634+0000", 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000180667 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626806+0000", 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172688 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.502 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.636824+0000", 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010017458 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.651097+0000", 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014273447 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.503 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.504 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:32:55.504 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.661502+0000", 2023-10-09T06:32:55.504 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010404958000000001 2023-10-09T06:32:55.504 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.661553+0000", 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1051999999999998e-05 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:32:55.505 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.664358+0000", 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028052569999999998 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:32:55.506 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:32:55.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:55 smithi150 ceph-mon[194178]: pgmap v1216: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 6212/282921 objects degraded (2.196%); 13950/282921 objects misplaced (4.931%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:32:55.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:55 smithi150 ceph-mon[194178]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:55 smithi150 ceph-mon[194178]: failed to encode map e403 with expected crc 2023-10-09T06:32:55.507 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:55 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 6212/282921 objects degraded (2.196%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:32:55.507 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:55.508 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:55.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:55.609 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:32:55.640 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:55.699 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:32:55.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:55.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:32:55.807 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:32:55.897 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:56.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:56 smithi107 ceph-mon[220921]: 12.4 scrub starts 2023-10-09T06:32:56.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:56 smithi107 ceph-mon[212750]: 12.4 scrub starts 2023-10-09T06:32:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:56 smithi150 ceph-mon[194178]: 12.4 scrub starts 2023-10-09T06:32:57.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:57 smithi107 ceph-mon[212750]: pgmap v1218: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 6212/282672 objects degraded (2.198%); 13950/282672 objects misplaced (4.935%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:32:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:57 smithi107 ceph-mon[220921]: pgmap v1218: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 6212/282672 objects degraded (2.198%); 13950/282672 objects misplaced (4.935%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:32:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:57 smithi150 ceph-mon[194178]: pgmap v1218: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 126 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 6212/282672 objects degraded (2.198%); 13950/282672 objects misplaced (4.935%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:32:58.363 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:32:58.370 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:58 smithi107 ceph-mon[220921]: 12.5 scrub starts 2023-10-09T06:32:58.371 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:58 smithi107 ceph-mon[212750]: 12.5 scrub starts 2023-10-09T06:32:58.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:58.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:32:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:58 smithi150 ceph-mon[194178]: 12.5 scrub starts 2023-10-09T06:32:58.474 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:58.546 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:58.680 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:32:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:58.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:32:58.786 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:58.819 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:58.875 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:58.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:58.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:58.983 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.011 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:59.057 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:59.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:59.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:32:59.193 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:59.247 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:32:59.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:59.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:32:59.352 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:32:59.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:32:59 smithi107 ceph-mon[220921]: pgmap v1219: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 9333/278379 objects degraded (3.353%); 13321/278379 objects misplaced (4.785%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:32:59.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:32:59 smithi150 ceph-mon[194178]: pgmap v1219: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 9333/278379 objects degraded (3.353%); 13321/278379 objects misplaced (4.785%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:32:59.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:32:59 smithi107 ceph-mon[212750]: pgmap v1219: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 9333/278379 objects degraded (3.353%); 13321/278379 objects misplaced (4.785%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:32:59.443 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:32:59.489 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:32:59.553 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:32:59.682 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:32:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:32:59.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:32:59.792 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:32:59.945 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:00.186 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:00 smithi107 ceph-mon[212750]: 12.14 scrub starts 2023-10-09T06:33:00.187 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:00 smithi107 ceph-mon[220921]: 12.14 scrub starts 2023-10-09T06:33:00.206 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:00.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:00.319 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:00 smithi150 ceph-mon[194178]: 12.14 scrub starts 2023-10-09T06:33:00.412 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:33:01.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.140 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:33:01.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[212750]: pgmap v1220: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 9089/273594 objects degraded (3.322%); 12889/273594 objects misplaced (4.711%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[212750]: 12.15 scrub starts 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 9089/273594 objects degraded (3.322%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/638647309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[220921]: pgmap v1220: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 9089/273594 objects degraded (3.322%); 12889/273594 objects misplaced (4.711%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[220921]: 12.15 scrub starts 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 9089/273594 objects degraded (3.322%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/638647309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:01.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:01 smithi150 ceph-mon[194178]: pgmap v1220: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 9089/273594 objects degraded (3.322%); 12889/273594 objects misplaced (4.711%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T06:33:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:01 smithi150 ceph-mon[194178]: 12.15 scrub starts 2023-10-09T06:33:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:01 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 9089/273594 objects degraded (3.322%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/638647309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:02.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:02] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:33:02.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[212750]: pgmap v1221: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 9089/270231 objects degraded (3.363%); 12463/270231 objects misplaced (4.612%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[220921]: pgmap v1221: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 9089/270231 objects degraded (3.363%); 12463/270231 objects misplaced (4.612%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:02.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:02 smithi150 ceph-mon[194178]: pgmap v1221: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 9089/270231 objects degraded (3.363%); 12463/270231 objects misplaced (4.612%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:03.890 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:03.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:03 smithi107 ceph-mon[212750]: 12.19 scrub starts 2023-10-09T06:33:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:03.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:03.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:03 smithi107 ceph-mon[220921]: 12.19 scrub starts 2023-10-09T06:33:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:03 smithi150 ceph-mon[194178]: 12.19 scrub starts 2023-10-09T06:33:03.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:04.086 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:04.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:04 smithi107 ceph-mon[212750]: pgmap v1222: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8861/262725 objects degraded (3.373%); 12037/262725 objects misplaced (4.582%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:33:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:04 smithi107 ceph-mon[220921]: pgmap v1222: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8861/262725 objects degraded (3.373%); 12037/262725 objects misplaced (4.582%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:33:04.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:04 smithi150 ceph-mon[194178]: pgmap v1222: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8861/262725 objects degraded (3.373%); 12037/262725 objects misplaced (4.582%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T06:33:05.747 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:05.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:05.855 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:05.906 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:06.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:06.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:06.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:33:06.765 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:06.928 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:06.930 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:06 smithi150 ceph-mon[194178]: pgmap v1223: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 8861/261939 objects degraded (3.383%); 12037/261939 objects misplaced (4.595%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:33:06.930 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:06 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8861/261939 objects degraded (3.383%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:06.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:06 smithi150 ceph-mon[194178]: 12.1 scrub starts 2023-10-09T06:33:06.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:06.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:06.951 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[220921]: pgmap v1223: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 8861/261939 objects degraded (3.383%); 12037/261939 objects misplaced (4.595%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:33:06.951 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8861/261939 objects degraded (3.383%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:06.951 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[220921]: 12.1 scrub starts 2023-10-09T06:33:06.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[212750]: pgmap v1223: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 8861/261939 objects degraded (3.383%); 12037/261939 objects misplaced (4.595%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:33:06.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8861/261939 objects degraded (3.383%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:06.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:06 smithi107 ceph-mon[212750]: 12.1 scrub starts 2023-10-09T06:33:07.032 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:07.136 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:07.601 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:07.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:07.710 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:07.749 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:07.872 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:07.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:07.982 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:08.016 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:08.085 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:08.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:08.190 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:08.232 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:08.346 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:08.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:08.455 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:33:08.456 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:08.481 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:33:08.543 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:08 smithi150 ceph-mon[194178]: pgmap v1224: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 8861/254886 objects degraded (3.476%); 11575/254886 objects misplaced (4.541%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:33:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:08.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:33:08.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:08.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:08.665 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[212750]: pgmap v1224: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 8861/254886 objects degraded (3.476%); 11575/254886 objects misplaced (4.541%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:33:08.665 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:08.665 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:33:08.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[220921]: pgmap v1224: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 8861/254886 objects degraded (3.476%); 11575/254886 objects misplaced (4.541%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T06:33:08.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:08.666 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:33:08.695 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:33:08.765 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:08.796 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:08.841 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:08.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:08.945 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:09.052 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.897 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:09 smithi150 ceph-mon[194178]: 12.9 deep-scrub starts 2023-10-09T06:33:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:09.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[220921]: 12.9 deep-scrub starts 2023-10-09T06:33:09.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2666825570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1447210779' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: osdmap e404: 8 total, 8 up, 8 in 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: failed to encode map e404 with expected crc 2023-10-09T06:33:09.907 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:09 smithi107 ceph-mon[212750]: 12.9 deep-scrub starts 2023-10-09T06:33:10.470 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[212750]: pgmap v1226: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8623/249903 objects degraded (3.451%); 11131/249903 objects misplaced (4.454%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:33:10.470 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[212750]: 12.4 scrub starts 2023-10-09T06:33:10.471 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:10.783 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:10.790 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[220921]: pgmap v1226: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8623/249903 objects degraded (3.451%); 11131/249903 objects misplaced (4.454%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:33:10.790 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[220921]: 12.4 scrub starts 2023-10-09T06:33:10.790 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:10.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:10.891 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:10 smithi150 ceph-mon[194178]: pgmap v1226: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8623/249903 objects degraded (3.451%); 11131/249903 objects misplaced (4.454%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T06:33:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:10 smithi150 ceph-mon[194178]: 12.4 scrub starts 2023-10-09T06:33:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:10.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:10.981 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:10.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:11.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:11.170 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:11.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:11.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:11.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:11.995 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:12.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:12.103 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:12.132 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:12.400 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:12] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-10-09T06:33:12.682 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:12.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:12.768 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:12 smithi107 ceph-mon[212750]: pgmap v1227: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8623/246759 objects degraded (3.495%); 10654/246759 objects misplaced (4.318%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:12.768 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8623/246759 objects degraded (3.495%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:12.768 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:12 smithi107 ceph-mon[220921]: pgmap v1227: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8623/246759 objects degraded (3.495%); 10654/246759 objects misplaced (4.318%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:12.769 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8623/246759 objects degraded (3.495%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:12.793 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:12.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:33:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:12 smithi150 ceph-mon[194178]: pgmap v1227: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 8623/246759 objects degraded (3.495%); 10654/246759 objects misplaced (4.318%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8623/246759 objects degraded (3.495%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:12.912 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:13.090 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:13.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:13.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:13.258 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:13.313 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:13.423 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:33:13.423 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:33:14.318 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:14.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:14.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:14.432 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:14.466 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:14.526 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:14.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:14.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:14.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:14 smithi107 ceph-mon[220921]: pgmap v1228: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 8411/239889 objects degraded (3.506%); 10198/239889 objects misplaced (4.251%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:14.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:14 smithi107 ceph-mon[212750]: pgmap v1228: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 8411/239889 objects degraded (3.506%); 10198/239889 objects misplaced (4.251%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:14.663 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:14.728 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:14.733 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:14 smithi150 ceph-mon[194178]: pgmap v1228: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 8411/239889 objects degraded (3.506%); 10198/239889 objects misplaced (4.251%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:14.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:14.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:14.812 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:33:14.835 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:14.869 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:14.930 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:14.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:15.037 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:15.123 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:15.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:15 smithi107 ceph-mon[212750]: 12.5 scrub starts 2023-10-09T06:33:15.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:15 smithi107 ceph-mon[220921]: 12.5 scrub starts 2023-10-09T06:33:15.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:15 smithi150 ceph-mon[194178]: 12.5 scrub starts 2023-10-09T06:33:16.028 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:16.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:16.143 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:16.256 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:16.259 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:33:16.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:16 smithi107 ceph-mon[212750]: pgmap v1229: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8411/239172 objects degraded (3.517%); 10198/239172 objects misplaced (4.264%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:16.587 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:16 smithi107 ceph-mon[220921]: pgmap v1229: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8411/239172 objects degraded (3.517%); 10198/239172 objects misplaced (4.264%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:16.587 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:16.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:16.697 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:16.784 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:16.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:16 smithi150 ceph-mon[194178]: pgmap v1229: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8411/239172 objects degraded (3.517%); 10198/239172 objects misplaced (4.264%); 6.2 MiB/s, 98 objects/s recovering 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8411/239172 objects degraded (3.517%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:17.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8411/239172 objects degraded (3.517%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:17.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8411/239172 objects degraded (3.517%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:18.050 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:18.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:18.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:18.172 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:18.188 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:18.192 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:18.238 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:18.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:18.345 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:18.361 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:18.444 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:18.617 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:18.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:18.726 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:18.756 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:18.821 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:18.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:18.871 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:18 smithi107 ceph-mon[212750]: pgmap v1230: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8411/233040 objects degraded (3.609%); 9681/233040 objects misplaced (4.154%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:18.871 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:18 smithi107 ceph-mon[220921]: pgmap v1230: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8411/233040 objects degraded (3.609%); 9681/233040 objects misplaced (4.154%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:18 smithi150 ceph-mon[194178]: pgmap v1230: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8411/233040 objects degraded (3.609%); 9681/233040 objects misplaced (4.154%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T06:33:18.928 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:18.957 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:19.015 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:19.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:19.120 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:19.249 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:19.893 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:19.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:20.012 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:20.077 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:21.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:20 smithi107 ceph-mon[212750]: pgmap v1231: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8145/227658 objects degraded (3.578%); 9135/227658 objects misplaced (4.013%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:21.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:20 smithi107 ceph-mon[212750]: 12.14 scrub starts 2023-10-09T06:33:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:20 smithi107 ceph-mon[220921]: pgmap v1231: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8145/227658 objects degraded (3.578%); 9135/227658 objects misplaced (4.013%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:20 smithi107 ceph-mon[220921]: 12.14 scrub starts 2023-10-09T06:33:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:20 smithi150 ceph-mon[194178]: pgmap v1231: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8145/227658 objects degraded (3.578%); 9135/227658 objects misplaced (4.013%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T06:33:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:20 smithi150 ceph-mon[194178]: 12.14 scrub starts 2023-10-09T06:33:22.020 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3935338061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:22.020 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:22] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:33:22.020 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3935338061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:33:22.042 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.043 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:33:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3935338061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:22 smithi107 ceph-mon[212750]: pgmap v1232: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8145/224613 objects degraded (3.626%); 8647/224613 objects misplaced (3.850%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:33:23.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8145/224613 objects degraded (3.626%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:22 smithi107 ceph-mon[220921]: pgmap v1232: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8145/224613 objects degraded (3.626%); 8647/224613 objects misplaced (3.850%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:33:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8145/224613 objects degraded (3.626%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:22 smithi150 ceph-mon[194178]: pgmap v1232: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 8145/224613 objects degraded (3.626%); 8647/224613 objects misplaced (3.850%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T06:33:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8145/224613 objects degraded (3.626%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:23.409 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:23.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:23.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.522 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:23.610 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:23.808 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:23.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:23.919 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:23.959 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:24.019 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:24.023 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:23 smithi150 ceph-mon[194178]: 12.15 scrub starts 2023-10-09T06:33:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:24.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:24.125 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:24.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:23 smithi107 ceph-mon[212750]: 12.15 scrub starts 2023-10-09T06:33:24.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:23 smithi107 ceph-mon[220921]: 12.15 scrub starts 2023-10-09T06:33:24.198 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:25.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:24 smithi107 ceph-mon[212750]: pgmap v1233: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:25.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:24 smithi107 ceph-mon[220921]: pgmap v1233: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:25.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:24 smithi150 ceph-mon[194178]: pgmap v1233: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:25.512 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:25.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:25.619 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:25.708 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:25.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:25 smithi107 ceph-mon[212750]: 12.19 scrub starts 2023-10-09T06:33:25.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:25 smithi107 ceph-mon[220921]: 12.19 scrub starts 2023-10-09T06:33:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:25 smithi150 ceph-mon[194178]: 12.19 scrub starts 2023-10-09T06:33:26.236 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:26.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:26.344 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:26.422 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:26.705 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:26.711 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:33:26.711 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:33:26.711 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:33:26.711 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:33:26.711 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:33:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:26.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:26.815 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:26.843 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:26.901 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:26.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:27.011 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:26 smithi150 ceph-mon[194178]: pgmap v1234: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:33:27.043 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:27.093 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:27.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:26 smithi107 ceph-mon[212750]: pgmap v1234: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:33:27.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:26 smithi107 ceph-mon[220921]: pgmap v1234: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 7889/216624 objects degraded (3.642%); 8091/216624 objects misplaced (3.735%); 6.0 MiB/s, 95 objects/s recovering 2023-10-09T06:33:27.198 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:27.228 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:27.282 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:27.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:27.393 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:27.469 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:27.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2019784225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:27.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:27.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7889/216624 objects degraded (3.642%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:27.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/625327002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:27.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:27.811 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:33:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2019784225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7889/216624 objects degraded (3.642%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/625327002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:28.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2019784225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:28.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:33:28.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7889/216624 objects degraded (3.642%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:28.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/625327002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:28.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:33:28.188 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:33:28.792 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: pgmap v1235: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 7889/213339 objects degraded (3.698%); 7604/213339 objects misplaced (3.564%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:33:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[212750]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: pgmap v1235: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 7889/213339 objects degraded (3.698%); 7604/213339 objects misplaced (3.564%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:33:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:28 smithi107 ceph-mon[220921]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: pgmap v1235: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 7889/213339 objects degraded (3.698%); 7604/213339 objects misplaced (3.564%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: failed to encode map e405 with expected crc 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: osdmap e405: 8 total, 8 up, 8 in 2023-10-09T06:33:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:28 smithi150 ceph-mon[194178]: failed to encode map e405 with expected crc 2023-10-09T06:33:30.830 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:30.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:30.900 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:33:30.945 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:30.979 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:33:31.042 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:31.135 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:30 smithi107 ceph-mon[212750]: pgmap v1237: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7639/205773 objects degraded (3.712%); 6972/205773 objects misplaced (3.388%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:33:31.145 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:30 smithi107 ceph-mon[220921]: pgmap v1237: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7639/205773 objects degraded (3.712%); 6972/205773 objects misplaced (3.388%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:33:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:30 smithi150 ceph-mon[194178]: pgmap v1237: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 7639/205773 objects degraded (3.712%); 6972/205773 objects misplaced (3.388%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T06:33:31.301 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:31.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:31.421 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:31.458 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:31.512 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:31.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:31.619 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:31.645 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:31.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:31.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:31.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:31.821 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:31.880 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:31.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:31.985 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:32.025 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:32.082 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:32.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:32.189 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:32.275 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:32] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-10-09T06:33:32.275 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:32.398 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:32.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:32.516 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:32.517 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:32.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:32.595 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:32.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:32.622 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[212750]: pgmap v1238: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7639/202896 objects degraded (3.765%); 6442/202896 objects misplaced (3.175%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:33:32.622 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:32.622 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:32.623 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7639/202896 objects degraded (3.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:32.695 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:32.705 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:32.742 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:32.819 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:32.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[220921]: pgmap v1238: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7639/202896 objects degraded (3.765%); 6442/202896 objects misplaced (3.175%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:33:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7639/202896 objects degraded (3.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:32.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:32 smithi150 ceph-mon[194178]: pgmap v1238: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 7639/202896 objects degraded (3.765%); 6442/202896 objects misplaced (3.175%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T06:33:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:32.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7639/202896 objects degraded (3.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:32.984 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:32.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:33.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:33.190 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:34.228 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:33:34.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:34 smithi150 ceph-mon[194178]: pgmap v1239: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:34.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:34 smithi107 ceph-mon[212750]: pgmap v1239: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:34.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:34 smithi107 ceph-mon[220921]: pgmap v1239: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:36.893 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:36.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:36 smithi107 ceph-mon[212750]: pgmap v1240: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:36 smithi107 ceph-mon[220921]: pgmap v1240: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:36.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:36 smithi150 ceph-mon[194178]: pgmap v1240: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 7415/195408 objects degraded (3.795%); 5756/195408 objects misplaced (2.946%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:36.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:37.007 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.038 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:37.096 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:37.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:37.203 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:37.241 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:37.300 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:37.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:37.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:37.436 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:37.492 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:37.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:37.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:37.626 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:37.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:37.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7415/195408 objects degraded (3.795%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:37.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7415/195408 objects degraded (3.795%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:37 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7415/195408 objects degraded (3.795%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:38.612 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:38.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:38.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:38.720 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:38.813 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:38 smithi107 ceph-mon[220921]: pgmap v1241: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 7415/192597 objects degraded (3.850%); 5243/192597 objects misplaced (2.722%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:33:38.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:38 smithi107 ceph-mon[212750]: pgmap v1241: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 7415/192597 objects degraded (3.850%); 5243/192597 objects misplaced (2.722%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:33:38.986 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:38.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:39.097 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:39.126 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:38 smithi150 ceph-mon[194178]: pgmap v1241: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 7415/192597 objects degraded (3.850%); 5243/192597 objects misplaced (2.722%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T06:33:39.130 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:39.185 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:39.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:39.295 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:39.324 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:39.349 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:39.377 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:39.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:39.487 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:39.624 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:39.643 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:33:39.716 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:33:39.767 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:33:39.809 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:33:39.857 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:33:39.865 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:39.865 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:33:39.866 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:33:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:39.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:39.909 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:33:39.945 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:33:39.975 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:39.983 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:33:40.000 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.000 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:33:40.024 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:40.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:40.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:40.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/106739410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:40.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:40.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:40.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:40.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/106739410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:40.055 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:33:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:33:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:33:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/106739410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[212750]: pgmap v1242: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 7207/184767 objects degraded (3.901%); 4313/184767 objects misplaced (2.334%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T06:33:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4112029930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:41.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[220921]: pgmap v1242: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 7207/184767 objects degraded (3.901%); 4313/184767 objects misplaced (2.334%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T06:33:41.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4112029930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:41.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:40 smithi150 ceph-mon[194178]: pgmap v1242: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 7207/184767 objects degraded (3.901%); 4313/184767 objects misplaced (2.334%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T06:33:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4112029930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:41 smithi150 ceph-mon[194178]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:33:42.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[212750]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:42] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:33:42.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: osdmap e406: 8 total, 8 up, 8 in 2023-10-09T06:33:42.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:41 smithi107 ceph-mon[220921]: failed to encode map e406 with expected crc 2023-10-09T06:33:42.412 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:33:43.076 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:43.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:43.181 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:43.205 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:42 smithi150 ceph-mon[194178]: pgmap v1244: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7207/182085 objects degraded (3.958%); 3787/182085 objects misplaced (2.080%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:33:43.205 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7207/182085 objects degraded (3.958%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:43.205 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:42 smithi150 ceph-mon[194178]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T06:33:43.205 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:42 smithi150 ceph-mon[194178]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.205 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:42 smithi150 ceph-mon[194178]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.207 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:43.250 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:43.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:43.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:43.358 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:43.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:43.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[212750]: pgmap v1244: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7207/182085 objects degraded (3.958%); 3787/182085 objects misplaced (2.080%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:33:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7207/182085 objects degraded (3.958%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[212750]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T06:33:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[220921]: pgmap v1244: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7207/182085 objects degraded (3.958%); 3787/182085 objects misplaced (2.080%); 8.6 MiB/s, 136 objects/s recovering 2023-10-09T06:33:43.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7207/182085 objects degraded (3.958%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:33:43.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[220921]: osdmap e407: 8 total, 8 up, 8 in 2023-10-09T06:33:43.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[220921]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:42 smithi107 ceph-mon[220921]: failed to encode map e407 with expected crc 2023-10-09T06:33:43.427 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:43.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:43.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:43.532 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:43.559 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:43.599 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:43.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:43.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:43.704 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:43.742 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:43.797 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:43.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:43.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:43.988 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:33:44.011 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:44.387 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:44.389 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.389 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.389 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.389 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[212750]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.390 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:43 smithi107 ceph-mon[220921]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:44.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: failed to encode map e407 with expected crc 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: osdmap e408: 8 total, 8 up, 8 in 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:43 smithi150 ceph-mon[194178]: failed to encode map e408 with expected crc 2023-10-09T06:33:44.493 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:44.531 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:44.574 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:44.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:44.682 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:44.713 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:44.773 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:44.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:44.880 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:44.909 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:44.920 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:33:44.967 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:44.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:44.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:45.073 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:45.141 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:44 smithi107 ceph-mon[220921]: pgmap v1246: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:33:45.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:44 smithi107 ceph-mon[212750]: pgmap v1246: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:33:45.150 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:45.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:44 smithi150 ceph-mon[194178]: pgmap v1246: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T06:33:45.393 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:45.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:45.452 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:45.499 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:45.532 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:45.587 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:45.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:45.696 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:45.719 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:45.724 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:45.776 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:45.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:45.881 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:45.911 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:45.965 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:45.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:46.007 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:33:46.070 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:46.153 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:46.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[212750]: pgmap v1248: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:33:46.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1240875110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:46.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:46.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:46.992 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:33:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:46 smithi150 ceph-mon[194178]: pgmap v1248: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:33:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1240875110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:47.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[220921]: pgmap v1248: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6971/174993 objects degraded (3.984%); 3787/174993 objects misplaced (2.164%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:33:47.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1240875110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:47.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:33:47.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6971/174993 objects degraded (3.984%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6971/174993 objects degraded (3.984%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: osdmap e409: 8 total, 8 up, 8 in 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: failed to encode map e409 with expected crc 2023-10-09T06:33:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6971/174993 objects degraded (3.984%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:49.017 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:49.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:49.120 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:49.154 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:49.222 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:49.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:49.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:49.328 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:49.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:48 smithi107 ceph-mon[212750]: pgmap v1250: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 6971/172239 objects degraded (4.047%); 2957/172239 objects misplaced (1.717%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:33:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:48 smithi150 ceph-mon[194178]: pgmap v1250: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 6971/172239 objects degraded (4.047%); 2957/172239 objects misplaced (1.717%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:33:49.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:48 smithi107 ceph-mon[220921]: pgmap v1250: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 125 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 6971/172239 objects degraded (4.047%); 2957/172239 objects misplaced (1.717%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:33:49.429 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:49.673 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:49.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:49.782 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:49.870 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:49.948 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:50.126 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:50.126 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:33:50.126 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:33:50.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:50.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:50.241 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:50.271 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:50.319 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:50.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:50.429 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:50.469 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:50.534 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:50.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:50.639 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:33:50.713 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:50.810 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:33:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:50.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:33:50.914 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:50.939 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:50.981 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:50.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:51.023 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:33:51.084 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:51.117 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:51.175 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:51.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:51.280 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:51 smithi107 ceph-mon[212750]: pgmap v1251: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6781/165165 objects degraded (4.106%); 2433/165165 objects misplaced (1.473%); 6.8 MiB/s, 126 objects/s recovering 2023-10-09T06:33:51.280 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:33:51.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:51 smithi107 ceph-mon[220921]: pgmap v1251: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6781/165165 objects degraded (4.106%); 2433/165165 objects misplaced (1.473%); 6.8 MiB/s, 126 objects/s recovering 2023-10-09T06:33:51.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:33:51.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:51.352 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:51 smithi150 ceph-mon[194178]: pgmap v1251: 129 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 6781/165165 objects degraded (4.106%); 2433/165165 objects misplaced (1.473%); 6.8 MiB/s, 126 objects/s recovering 2023-10-09T06:33:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:33:51.558 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:33:51.862 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:51.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:51.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:51.968 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:52.030 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:33:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:33:52] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[212750]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:52 smithi107 ceph-mon[220921]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1792539511' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: osdmap e410: 8 total, 8 up, 8 in 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:52 smithi150 ceph-mon[194178]: failed to encode map e410 with expected crc 2023-10-09T06:33:52.421 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:33:53.234 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:53.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:53.302 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:53 smithi107 ceph-mon[212750]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6781/161895 objects degraded (4.189%); 1779/161895 objects misplaced (1.099%); 6.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:53.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:53 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6781/161895 objects degraded (4.189%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:53.303 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:53 smithi107 ceph-mon[220921]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6781/161895 objects degraded (4.189%); 1779/161895 objects misplaced (1.099%); 6.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:53.303 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:53 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6781/161895 objects degraded (4.189%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:53.311 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:33:53.349 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:53.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:53 smithi150 ceph-mon[194178]: pgmap v1253: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6781/161895 objects degraded (4.189%); 1779/161895 objects misplaced (1.099%); 6.7 MiB/s, 123 objects/s recovering 2023-10-09T06:33:53.377 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:53 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6781/161895 objects degraded (4.189%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:33:53.381 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:53.439 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:53.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:53.493 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:53.545 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:33:53.695 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:53.733 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:53.994 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:33:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:33:54.107 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:54.138 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:54.195 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:54.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:54.304 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:33:54.335 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:54.353 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:33:54.393 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:33:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:54.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:33:54.402 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:33:54.453 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:33:54.494 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:33:54.498 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:54.528 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:54.539 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:33:54.575 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:33:54.585 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:54.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:54.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:54.633 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:33:54.663 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.668 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.668 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:33:54.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:54.777 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:55.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:55 smithi107 ceph-mon[220921]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 7.6 MiB/s, 137 objects/s recovering 2023-10-09T06:33:55.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3657529314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:55 smithi107 ceph-mon[212750]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 7.6 MiB/s, 137 objects/s recovering 2023-10-09T06:33:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3657529314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:55.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:55 smithi150 ceph-mon[194178]: pgmap v1254: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 7.6 MiB/s, 137 objects/s recovering 2023-10-09T06:33:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3657529314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:33:56.345 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:56.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:56.451 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:33:56.498 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:56.566 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:33:56.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:56.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:33:56.674 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:56.700 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:56.750 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:56.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:56.856 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:56.926 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:57.219 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:57 smithi107 ceph-mon[212750]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 5.2 MiB/s, 97 objects/s recovering 2023-10-09T06:33:57.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:57 smithi107 ceph-mon[220921]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 5.2 MiB/s, 97 objects/s recovering 2023-10-09T06:33:57.224 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:33:57.258 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:57.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:57.363 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:33:57.389 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:57 smithi150 ceph-mon[194178]: pgmap v1255: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 6539/153939 objects degraded (4.248%); 1779/153939 objects misplaced (1.156%); 5.2 MiB/s, 97 objects/s recovering 2023-10-09T06:33:57.391 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:57.456 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:33:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:57.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:33:57.562 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:33:57.646 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:57.975 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:33:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:57.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:33:58.080 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:58 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6539/153939 objects degraded (4.248%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:33:58.085 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:58.113 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:58.153 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:33:58.153 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:33:58.153 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:33:58.153 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:33:58.153 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:33:58.165 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:58.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:58.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:58.273 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:33:58.301 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:58.359 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:33:58.363 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:58 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6539/153939 objects degraded (4.248%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:33:58.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:58.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:33:58.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:58 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6539/153939 objects degraded (4.248%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:33:58.465 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:33:58.569 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:33:59.077 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:33:59.259 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:33:59.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[220921]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 6539/151050 objects degraded (4.329%); 1143/151050 objects misplaced (0.757%); 5.7 MiB/s, 103 objects/s recovering 2023-10-09T06:33:59.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2545056928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[212750]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 6539/151050 objects degraded (4.329%); 1143/151050 objects misplaced (0.757%); 5.7 MiB/s, 103 objects/s recovering 2023-10-09T06:33:59.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2545056928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.337 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:33:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:59 smithi150 ceph-mon[194178]: pgmap v1256: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 6539/151050 objects degraded (4.329%); 1143/151050 objects misplaced (0.757%); 5.7 MiB/s, 103 objects/s recovering 2023-10-09T06:33:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2545056928' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:33:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:33:59.876 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:33:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:33:59.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:33:59.986 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:00.059 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:00.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:00.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:00.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T06:34:00.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[220921]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:00 smithi107 ceph-mon[212750]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: osdmap e411: 8 total, 8 up, 8 in 2023-10-09T06:34:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:00 smithi150 ceph-mon[194178]: failed to encode map e411 with expected crc 2023-10-09T06:34:00.794 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:00.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:00.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:00.898 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:00.909 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:34:00.927 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:00.986 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:00.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:01.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:01.123 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:01.171 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:01.179 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:01 smithi150 ceph-mon[194178]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 6305/143115 objects degraded (4.406%); 1143/143115 objects misplaced (0.799%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:34:01.179 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:01 smithi150 ceph-mon[194178]: 12.1 scrub starts 2023-10-09T06:34:01.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:01.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:01.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:01 smithi107 ceph-mon[220921]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 6305/143115 objects degraded (4.406%); 1143/143115 objects misplaced (0.799%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:34:01.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:01 smithi107 ceph-mon[220921]: 12.1 scrub starts 2023-10-09T06:34:01.202 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:01 smithi107 ceph-mon[212750]: pgmap v1258: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 6305/143115 objects degraded (4.406%); 1143/143115 objects misplaced (0.799%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T06:34:01.202 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:01 smithi107 ceph-mon[212750]: 12.1 scrub starts 2023-10-09T06:34:01.282 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:01.316 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:01.318 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:01.371 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:01.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:01.477 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:01.575 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:01.775 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:01.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:01.885 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:01.964 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:02.081 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:02] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:34:02.145 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:34:02.207 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:02.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:02.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:02.363 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1804883565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:34:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1804883565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:02 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:34:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1804883565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:34:02.428 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:02.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:02.535 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:02.608 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:02.950 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:34:03.271 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:03.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:03.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:03.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6305/140103 objects degraded (4.500%); 419/140103 objects misplaced (0.299%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:03.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.281 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6305/140103 objects degraded (4.500%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6305/140103 objects degraded (4.500%); 419/140103 objects misplaced (0.299%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.282 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.283 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:03 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6305/140103 objects degraded (4.500%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:03.381 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: pgmap v1259: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6305/140103 objects degraded (4.500%); 419/140103 objects misplaced (0.299%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: osdmap e412: 8 total, 8 up, 8 in 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: failed to encode map e412 with expected crc 2023-10-09T06:34:03.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:03 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6305/140103 objects degraded (4.500%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:03.407 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:03.452 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:03.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:03.560 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:03.594 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:03.648 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:03.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:03.755 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:03.791 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:03.847 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:03.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:03.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:03.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:03.986 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:04.038 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:04.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:04.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:04.099 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:34:04.107 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:04.145 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:04.175 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:04.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:04.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:04.326 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:04.337 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:04.486 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:04.753 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:04.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:04.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:04.860 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:04.879 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:04.913 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:04.914 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:04.951 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:04.979 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:05.006 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:05.032 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:05.056 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:05.061 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.061 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.061 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.061 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:34:05.094 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:05.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:05.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:05.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:34:05.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T06:34:05.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3250114202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3250114202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:05.267 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: pgmap v1261: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: osdmap e413: 8 total, 8 up, 8 in 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: failed to encode map e413 with expected crc 2023-10-09T06:34:05.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3250114202' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:06.233 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:06.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:06.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:06.340 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:06.374 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:06.436 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:06.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: 12.9 scrub starts 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.444 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.445 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:06 smithi150 ceph-mon[194178]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.544 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:06.575 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:06.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:06.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:06.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:34:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T06:34:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: 12.9 scrub starts 2023-10-09T06:34:06.654 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[212750]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: osdmap e414: 8 total, 8 up, 8 in 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: 12.9 scrub starts 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:06 smithi107 ceph-mon[220921]: failed to encode map e414 with expected crc 2023-10-09T06:34:06.749 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:06.756 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:06.805 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:07.610 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:07 smithi107 ceph-mon[212750]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:34:07.610 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:07 smithi107 ceph-mon[220921]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:34:07.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:07 smithi150 ceph-mon[194178]: pgmap v1264: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6067/132594 objects degraded (4.576%); 419/132594 objects misplaced (0.316%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T06:34:07.683 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:07.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:07.789 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:07.827 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:07.886 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:07.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:07.990 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:08.037 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:08.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[212750]: 12.11 scrub starts 2023-10-09T06:34:08.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:08.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6067/132594 objects degraded (4.576%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:08.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[220921]: 12.11 scrub starts 2023-10-09T06:34:08.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:08.408 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:08 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6067/132594 objects degraded (4.576%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:08.430 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:08.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:08.538 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:08.567 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:08 smithi150 ceph-mon[194178]: 12.11 scrub starts 2023-10-09T06:34:08.567 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:08 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:08.567 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:08 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6067/132594 objects degraded (4.576%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:08.570 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:08.636 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:08.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:08.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:08.830 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:08.943 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:08.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:08.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:09.048 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:09.077 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:09.134 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:09.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:09.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:09.242 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:09.272 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:09.329 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:09.334 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:09 smithi150 ceph-mon[194178]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 4524/129642 objects degraded (3.490%); 1260/129642 objects misplaced (0.972%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:09.334 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:09 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:34:09.334 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:09 smithi150 ceph-mon[194178]: 12.0 scrub starts 2023-10-09T06:34:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:09.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:09.386 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:09.386 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[212750]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 4524/129642 objects degraded (3.490%); 1260/129642 objects misplaced (0.972%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:34:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[212750]: 12.0 scrub starts 2023-10-09T06:34:09.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[220921]: pgmap v1265: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 4524/129642 objects degraded (3.490%); 1260/129642 objects misplaced (0.972%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:34:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:09 smithi107 ceph-mon[220921]: 12.0 scrub starts 2023-10-09T06:34:09.436 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:09.521 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:09.626 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:34:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:10 smithi107 ceph-mon[212750]: 12.0 scrub ok 2023-10-09T06:34:10.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:10 smithi107 ceph-mon[212750]: pgmap v1266: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4312/122205 objects degraded (3.528%); 1260/122205 objects misplaced (1.031%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:34:10.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:10 smithi107 ceph-mon[220921]: 12.0 scrub ok 2023-10-09T06:34:10.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:10 smithi107 ceph-mon[220921]: pgmap v1266: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4312/122205 objects degraded (3.528%); 1260/122205 objects misplaced (1.031%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:34:10.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:10 smithi150 ceph-mon[194178]: 12.0 scrub ok 2023-10-09T06:34:10.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:10 smithi150 ceph-mon[194178]: pgmap v1266: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4312/122205 objects degraded (3.528%); 1260/122205 objects misplaced (1.031%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:34:11.177 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:11.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:11.289 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:11.376 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:11.440 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:11 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:11.442 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:11 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:11.541 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:11.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:11.595 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:34:11.624 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:11 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:11.667 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:11.744 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:11.896 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:12.021 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:12.025 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:12.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:12.130 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:12.209 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:12.329 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:12] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-10-09T06:34:12.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:12 smithi107 ceph-mon[212750]: 12.4 scrub starts 2023-10-09T06:34:12.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:12 smithi107 ceph-mon[212750]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 4312/118905 objects degraded (3.626%); 1058/118905 objects misplaced (0.890%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:12.472 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:12.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:12.541 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:12 smithi150 ceph-mon[194178]: 12.4 scrub starts 2023-10-09T06:34:12.541 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:12 smithi150 ceph-mon[194178]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 4312/118905 objects degraded (3.626%); 1058/118905 objects misplaced (0.890%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:12.588 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:12 smithi107 ceph-mon[220921]: 12.4 scrub starts 2023-10-09T06:34:12.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:12 smithi107 ceph-mon[220921]: pgmap v1267: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 4312/118905 objects degraded (3.626%); 1058/118905 objects misplaced (0.890%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:12.668 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:12.916 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:12.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:13.025 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:13.097 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:13.218 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:13.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:13.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:13.322 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:13.342 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:34:13.343 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:34:13.351 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:13.395 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:13.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:13.499 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:13.525 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:13.567 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:13.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4081340874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4312/118905 objects degraded (3.626%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1613755033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4081340874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4312/118905 objects degraded (3.626%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:13.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1613755033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:13 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4081340874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4312/118905 objects degraded (3.626%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1613755033' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:34:13.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:13 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:13.671 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:13.704 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:13.956 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:13.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:14.063 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:14.149 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:14.333 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:14.333 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.333 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:14.333 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:34:14.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T06:34:14.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: pgmap v1269: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:34:14.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.334 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.639 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:14.639 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.639 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:14.639 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:34:14.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T06:34:14.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: pgmap v1269: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:34:14.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.640 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:14 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:14.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: osdmap e415: 8 total, 8 up, 8 in 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: pgmap v1269: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:14 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:14.755 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:14.755 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:15.488 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:34:15.488 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:15.500 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:15.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:15.582 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:34:15.609 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:15.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:15 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.611 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:15 smithi150 ceph-mon[194178]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:15 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:34:15.612 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:15 smithi150 ceph-mon[194178]: 12.5 scrub starts 2023-10-09T06:34:15.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[212750]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:34:15.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[212750]: 12.5 scrub starts 2023-10-09T06:34:15.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[220921]: failed to encode map e415 with expected crc 2023-10-09T06:34:15.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:34:15.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:15 smithi107 ceph-mon[220921]: 12.5 scrub starts 2023-10-09T06:34:15.712 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:15.817 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:15.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:15.924 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:16.012 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:16.013 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:16.102 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:16.143 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:16.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:16.153 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:16.194 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:16.229 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:16.243 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:16.274 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:16.282 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:16.282 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:16.326 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:16.334 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:16.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:16.368 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:16.377 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[212750]: pgmap v1270: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:34:16.377 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/859195640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:16.377 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[212750]: 12.14 scrub starts 2023-10-09T06:34:16.406 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.417 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:34:16.438 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:16.472 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:16.496 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:16 smithi150 ceph-mon[194178]: pgmap v1270: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:34:16.496 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/859195640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:16.496 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:16 smithi150 ceph-mon[194178]: 12.14 scrub starts 2023-10-09T06:34:16.538 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:16.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:16.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:16.647 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[220921]: pgmap v1270: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 4066/111300 objects degraded (3.653%); 1058/111300 objects misplaced (0.951%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:34:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/859195640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:16.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:16 smithi107 ceph-mon[220921]: 12.14 scrub starts 2023-10-09T06:34:16.716 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:16.824 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:16.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:16.933 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:16.962 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:17.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:17.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:17.114 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:17.146 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:17.490 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:17.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:17.563 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:34:17.605 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:17.689 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:17.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:17.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:17.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:17 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:34:17.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:17 smithi150 ceph-mon[194178]: 12.15 scrub starts 2023-10-09T06:34:17.768 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4066/111300 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:34:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[212750]: 12.15 scrub starts 2023-10-09T06:34:17.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4066/111300 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:34:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[220921]: 12.15 scrub starts 2023-10-09T06:34:17.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4066/111300 objects degraded (3.653%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:17.982 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:17.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:18.049 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:34:18.049 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:34:18.049 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:34:18.049 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:34:18.050 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:34:18.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:18.135 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:18.251 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:18.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:18.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:18.358 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:18.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:18.426 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:18.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:18.529 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:18.586 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:18.633 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:18.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:18.737 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:18.765 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:18 smithi150 ceph-mon[194178]: pgmap v1271: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4066/108369 objects degraded (3.752%); 868/108369 objects misplaced (0.801%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:18.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:18 smithi107 ceph-mon[212750]: pgmap v1271: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4066/108369 objects degraded (3.752%); 868/108369 objects misplaced (0.801%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:18.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:18 smithi107 ceph-mon[220921]: pgmap v1271: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4066/108369 objects degraded (3.752%); 868/108369 objects misplaced (0.801%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:34:18.923 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:18.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:18.994 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:34:19.028 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.117 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:19.212 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:19.257 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:19.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:19.363 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:19.455 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:19.477 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:19.649 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:19.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:19.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:19.754 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:19.823 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:19 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:19.823 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:19 smithi107 ceph-mon[212750]: 12.11 scrub starts 2023-10-09T06:34:19.823 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:19 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:19.823 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:19 smithi107 ceph-mon[220921]: 12.11 scrub starts 2023-10-09T06:34:19.837 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:20.028 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:19 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:20.028 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:19 smithi150 ceph-mon[194178]: 12.11 scrub starts 2023-10-09T06:34:20.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:34:20.080 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:20.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:20.187 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:20.218 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:34:20.245 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:34:20.291 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:20.362 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:20.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:20.467 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:20.546 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:21.043 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:20 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:21.043 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:20 smithi150 ceph-mon[194178]: pgmap v1272: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 3820/100524 objects degraded (3.800%); 868/100524 objects misplaced (0.863%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:34:21.043 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:20 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:34:21.043 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:20 smithi150 ceph-mon[194178]: from='client.56416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[212750]: pgmap v1272: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 3820/100524 objects degraded (3.800%); 868/100524 objects misplaced (0.863%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:34:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:34:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[212750]: from='client.56416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[220921]: pgmap v1272: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 3820/100524 objects degraded (3.800%); 868/100524 objects misplaced (0.863%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T06:34:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:34:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:20 smithi107 ceph-mon[220921]: from='client.56416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45650.0:206697 10.e 10:748c2ac0:::benchmark_data_smithi107_363200_object102605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e291)", 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:34:21.301 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 498.02174435900002, 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29428934499999998, 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45650", 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/4197949253", 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 206697 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276440+0000", 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276439+0000", 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276446+0000", 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4529999999999999e-06 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276460+0000", 2023-10-09T06:34:21.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4163e-05 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276466+0000", 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276517+0000", 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1008999999999999e-05 2023-10-09T06:34:21.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276641+0000", 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012383699999999999 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.276760+0000", 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119003 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.280387+0000", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036266850000000002 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.283508+0000", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031211300000000002 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570636+0000", 2023-10-09T06:34:21.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28712860299999998 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570683+0000", 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6950000000000003e-05 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:26:03.570730+0000", 2023-10-09T06:34:21.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6347000000000002e-05 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:49050 11.e 11:7f8fd6d3:::benchmark_data_smithi107_418363_object49049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.02542497799999, 2023-10-09T06:34:21.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31877694499999998, 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 49050 2023-10-09T06:34:21.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272760+0000", 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272758+0000", 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272861+0000", 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010279 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272876+0000", 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.537e-05 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272881+0000", 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0259999999999998e-06 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.272923+0000", 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1832000000000002e-05 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.273063+0000", 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013991399999999999 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:34:21.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.273209+0000", 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014560199999999999 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.277138+0000", 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003928941 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.277154+0000", 2023-10-09T06:34:21.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6611000000000001e-05 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591443+0000", 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31428866700000002 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591495+0000", 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1518999999999999e-05 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:06.591537+0000", 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2082000000000001e-05 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117048 11.d 11:b97729d1:::benchmark_data_smithi107_418363_object26612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:34:21.318 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.64073433300001, 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.039298464, 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117048 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657450+0000", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657449+0000", 2023-10-09T06:34:21.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657457+0000", 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8329999999999994e-06 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657470+0000", 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3081999999999999e-05 2023-10-09T06:34:21.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657477+0000", 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8279999999999997e-06 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657537+0000", 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0040000000000001e-05 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.657546+0000", 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4749999999999999e-06 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.661510+0000", 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.003963478 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.663496+0000", 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019859529999999999 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.663578+0000", 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2823000000000003e-05 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676713+0000", 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013134866 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696643+0000", 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019930184 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696673+0000", 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9291000000000001e-05 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696712+0000", 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9456000000000003e-05 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696749+0000", 2023-10-09T06:34:21.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.663e-05 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117050 11.d 11:b78e64bd:::benchmark_data_smithi107_418363_object26614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.63379885099999, 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0325316689999999, 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.327 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117050 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664386+0000", 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664384+0000", 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664391+0000", 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.447e-06 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664417+0000", 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5969000000000002e-05 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664438+0000", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0896000000000001e-05 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664529+0000", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0749999999999997e-05 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.664539+0000", 2023-10-09T06:34:21.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0315e-05 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666048+0000", 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0015085319999999 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666153+0000", 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105499 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.666223+0000", 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9479000000000003e-05 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676722+0000", 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010498802 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696799+0000", 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020077426999999998 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696845+0000", 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5605999999999997e-05 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696884+0000", 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8911000000000001e-05 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696917+0000", 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3878999999999999e-05 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117064 11.d 11:be7c3437:::benchmark_data_smithi107_418363_object26628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.61812989100002, 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0170446479999999, 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117064 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680055+0000", 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680053+0000", 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680063+0000", 2023-10-09T06:34:21.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.057e-05 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680083+0000", 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9009999999999999e-05 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.680091+0000", 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6989999999999992e-06 2023-10-09T06:34:21.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.682293+0000", 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022022460000000002 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.682300+0000", 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:34:21.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668425+0000", 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98612509199999998 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668545+0000", 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119864 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.668673+0000", 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012771700000000001 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682177+0000", 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013504063 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696961+0000", 2023-10-09T06:34:21.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014783918 2023-10-09T06:34:21.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697034+0000", 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3361000000000005e-05 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697068+0000", 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4257000000000001e-05 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697099+0000", 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1103999999999999e-05 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117099 11.d 11:b81849c2:::benchmark_data_smithi107_418363_object26663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:34:21.347 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.55835111300001, 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95753685700000002, 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117099 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739834+0000", 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739832+0000", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739840+0000", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4880000000000007e-06 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739857+0000", 2023-10-09T06:34:21.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7003e-05 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739865+0000", 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3950000000000003e-06 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739924+0000", 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9728000000000001e-05 2023-10-09T06:34:21.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.739933+0000", 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2740000000000007e-06 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671421+0000", 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93148836300000004 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671546+0000", 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012488199999999999 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.671666+0000", 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120091 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682184+0000", 2023-10-09T06:34:21.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010518323 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696999+0000", 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014814475000000001 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697307+0000", 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000308654 2023-10-09T06:34:21.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697341+0000", 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3426999999999997e-05 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697370+0000", 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9592999999999999e-05 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.355 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117109 11.d 11:bb11f552:::benchmark_data_smithi107_418363_object26673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.54580970500001, 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94509762100000005, 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.356 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117109 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752375+0000", 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752373+0000", 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752383+0000", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4390000000000006e-06 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752402+0000", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9057999999999999e-05 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752409+0000", 2023-10-09T06:34:21.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8220000000000001e-06 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752463+0000", 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4107000000000002e-05 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.752471+0000", 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3499999999999997e-06 2023-10-09T06:34:21.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674412+0000", 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92194090200000001 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674531+0000", 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011925399999999999 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.674654+0000", 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012284199999999999 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692198+0000", 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017543665999999999 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697138+0000", 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049405960000000002 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697408+0000", 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00026925000000000001 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697442+0000", 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4663000000000003e-05 2023-10-09T06:34:21.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697473+0000", 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0385000000000001e-05 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117124 11.d 11:b2d54711:::benchmark_data_smithi107_418363_object26688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:34:21.364 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.520775259, 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92015732900000002, 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117124 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777409+0000", 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777408+0000", 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777416+0000", 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7450000000000001e-06 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777446+0000", 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9723999999999999e-05 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777454+0000", 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.323e-06 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777510+0000", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6020000000000002e-05 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.777517+0000", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5700000000000004e-06 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676769+0000", 2023-10-09T06:34:21.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89925138699999996 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676871+0000", 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000102178 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.676940+0000", 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9088e-05 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692205+0000", 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015265261 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697174+0000", 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049690460000000004 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697505+0000", 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00033072700000000001 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697537+0000", 2023-10-09T06:34:21.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2613000000000001e-05 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697567+0000", 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9298e-05 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.373 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117132 11.d 11:b198dabb:::benchmark_data_smithi107_418363_object26696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.50878899000003, 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91594003000000002, 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117132 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.374 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789396+0000", 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789394+0000", 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789404+0000", 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8099999999999992e-06 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789427+0000", 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2948000000000001e-05 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789435+0000", 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9920000000000007e-06 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789488+0000", 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2843999999999998e-05 2023-10-09T06:34:21.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789498+0000", 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9620000000000005e-06 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679459+0000", 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88996140499999998 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679533+0000", 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4139000000000003e-05 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.679616+0000", 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2577999999999994e-05 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696206+0000", 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016589967000000001 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697210+0000", 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001004128 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705228+0000", 2023-10-09T06:34:21.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080186179999999999 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705281+0000", 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2667000000000002e-05 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705336+0000", 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4673000000000003e-05 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.381 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117133 11.d 11:b5b48777:::benchmark_data_smithi107_418363_object26697:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.50849161500003, 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91576685000000002, 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.382 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117133 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789693+0000", 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789692+0000", 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789698+0000", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3300000000000004e-06 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789712+0000", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.379e-05 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789720+0000", 2023-10-09T06:34:21.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3119999999999996e-06 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789756+0000", 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6071000000000001e-05 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.789761+0000", 2023-10-09T06:34:21.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4310000000000002e-06 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682210+0000", 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89244917300000004 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682327+0000", 2023-10-09T06:34:21.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011647800000000001 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.682398+0000", 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1798000000000006e-05 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696212+0000", 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013813997 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697245+0000", 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010323960000000001 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705380+0000", 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0081353640000000008 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705424+0000", 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4078e-05 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705460+0000", 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5547999999999999e-05 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117137 11.d 11:ba9f3cee:::benchmark_data_smithi107_418363_object26701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.50334411, 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.91075980999999995, 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.391 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117137 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794841+0000", 2023-10-09T06:34:21.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794840+0000", 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794845+0000", 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1250000000000001e-06 2023-10-09T06:34:21.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794858+0000", 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3273e-05 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.794865+0000", 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.883e-06 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.795916+0000", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010514109999999999 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.795929+0000", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.289e-05 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685114+0000", 2023-10-09T06:34:21.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88918451899999995 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685228+0000", 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000114472 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.685348+0000", 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011943300000000001 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696220+0000", 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010872527999999999 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.697273+0000", 2023-10-09T06:34:21.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010526730000000001 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705519+0000", 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082458070000000008 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705577+0000", 2023-10-09T06:34:21.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7926000000000001e-05 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705600+0000", 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3782e-05 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.399 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117147 11.d 11:b7f258a5:::benchmark_data_smithi107_418363_object26711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.46900392499998, 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.87652914699999995, 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117147 2023-10-09T06:34:21.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829181+0000", 2023-10-09T06:34:21.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829179+0000", 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829189+0000", 2023-10-09T06:34:21.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8519999999999999e-06 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829214+0000", 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4949000000000001e-05 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829223+0000", 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4599999999999992e-06 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829291+0000", 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8280000000000004e-05 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.829304+0000", 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2408e-05 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688474+0000", 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.85917075399999998 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688581+0000", 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106147 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.688687+0000", 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106715 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.696345+0000", 2023-10-09T06:34:21.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0076581649999999998 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.700894+0000", 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.004548837 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705634+0000", 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047392659999999998 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705672+0000", 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8198999999999998e-05 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705710+0000", 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8099999999999998e-05 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.408 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:117155 11.d 11:bfe8c66f:::benchmark_data_smithi107_418363_object26719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 332.43248737200003, 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84011032600000002, 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.409 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 117155 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865697+0000", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865696+0000", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865714+0000", 2023-10-09T06:34:21.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7901e-05 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865733+0000", 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9585999999999998e-05 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865743+0000", 2023-10-09T06:34:21.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6849999999999996e-06 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865819+0000", 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6369e-05 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:48.865827+0000", 2023-10-09T06:34:21.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7170000000000008e-06 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692234+0000", 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.82640717200000002 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692351+0000", 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117275 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.692432+0000", 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0427999999999996e-05 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.699389+0000", 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0069568240000000003 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.700923+0000", 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015340040000000001 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705740+0000", 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048176399999999998 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705765+0000", 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4825e-05 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:28:49.705808+0000", 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2391000000000001e-05 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:32545 12.0 12:01fec15d:::benchmark_data_smithi107_479504_object32544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 173.14145047299999, 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.280403281, 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.418 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 32545 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156733+0000", 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156799+0000", 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5947999999999999e-05 2023-10-09T06:34:21.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156811+0000", 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2396e-05 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156818+0000", 2023-10-09T06:34:21.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3509999999999998e-06 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156855+0000", 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7357000000000002e-05 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156935+0000", 2023-10-09T06:34:21.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9637999999999999e-05 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.157020+0000", 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5840000000000005e-05 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158089+0000", 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010683859999999999 2023-10-09T06:34:21.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158849+0000", 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00076036400000000003 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437051+0000", 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27820162700000001 2023-10-09T06:34:21.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437100+0000", 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9347e-05 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437137+0000", 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7234000000000003e-05 2023-10-09T06:34:21.425 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:132696 12.2 12:473b2252:::benchmark_data_smithi107_479504_object33085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e405)", 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:33:29.985490+0000", 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 51.312694393999998, 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.132120604, 2023-10-09T06:34:21.426 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 132696 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985490+0000", 2023-10-09T06:34:21.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985490+0000", 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985489+0000", 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985495+0000", 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5010000000000004e-06 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985509+0000", 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4416999999999999e-05 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985516+0000", 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5630000000000003e-06 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985567+0000", 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0785999999999998e-05 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.985660+0000", 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2985000000000005e-05 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.986038+0000", 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037813700000000002 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.987932+0000", 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001893804 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:29.988221+0000", 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00028857299999999998 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:30.117548+0000", 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.12932743199999999 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:30.117584+0000", 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5704000000000002e-05 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:30.117611+0000", 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6933000000000001e-05 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:153022 12.0 12:03cdd803:::benchmark_data_smithi107_479504_object53411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:33:59.004964+0000", 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 22.293220517000002, 2023-10-09T06:34:21.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23560020300000001, 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 153022 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004964+0000", 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004964+0000", 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004964+0000", 2023-10-09T06:34:21.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004967+0000", 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0819999999999999e-06 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004974+0000", 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0129999999999996e-06 2023-10-09T06:34:21.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004978+0000", 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0339999999999998e-06 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005028+0000", 2023-10-09T06:34:21.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0114000000000002e-05 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005150+0000", 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012197399999999999 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005589+0000", 2023-10-09T06:34:21.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00043955300000000002 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.008009+0000", 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024191769999999998 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.009946+0000", 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019376980000000001 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.240468+0000", 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.230521435 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.240525+0000", 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7627999999999998e-05 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.240564+0000", 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8915999999999998e-05 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:153821 12.2 12:4504e905:::benchmark_data_smithi107_479504_object54210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:00.300344+0000", 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 20.997840385, 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.162971648, 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.444 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 153821 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300344+0000", 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300344+0000", 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300343+0000", 2023-10-09T06:34:21.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300350+0000", 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5979999999999999e-06 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300367+0000", 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7082e-05 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300374+0000", 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2999999999999998e-06 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300410+0000", 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6656999999999998e-05 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300520+0000", 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010957 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.300960+0000", 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00043983200000000003 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.303400+0000", 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024403879999999999 2023-10-09T06:34:21.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.305497+0000", 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020963729999999999 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463216+0000", 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15771943799999999 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463272+0000", 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6348999999999999e-05 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463316+0000", 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3582999999999999e-05 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:153832 12.2 12:4311f653:::benchmark_data_smithi107_479504_object54221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:00.310417+0000", 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 20.987767170000001, 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.153039802, 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 153832 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310417+0000", 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310417+0000", 2023-10-09T06:34:21.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310416+0000", 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310421+0000", 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9640000000000002e-06 2023-10-09T06:34:21.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310433+0000", 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.199e-05 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310438+0000", 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0170000000000004e-06 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310477+0000", 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8572000000000003e-05 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310571+0000", 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3912000000000005e-05 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.310944+0000", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037280800000000003 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.313106+0000", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021627209999999998 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.313602+0000", 2023-10-09T06:34:21.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00049537799999999999 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463399+0000", 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14979746899999999 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463433+0000", 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3374000000000003e-05 2023-10-09T06:34:21.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:00.463457+0000", 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4635999999999999e-05 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:155770 12.2 12:47058fdc:::benchmark_data_smithi107_479504_object56159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e412)", 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:02.940357+0000", 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 18.357827790999998, 2023-10-09T06:34:21.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23819786500000001, 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 155770 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940357+0000", 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940357+0000", 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940356+0000", 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940361+0000", 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7069999999999997e-06 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940374+0000", 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2775999999999999e-05 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940393+0000", 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8624e-05 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940427+0000", 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4570000000000003e-05 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940532+0000", 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104489 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.940977+0000", 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044548299999999999 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.943250+0000", 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022726579999999999 2023-10-09T06:34:21.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:02.943753+0000", 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00050309200000000001 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:03.178465+0000", 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23471228 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:03.178515+0000", 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9897e-05 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:03.178555+0000", 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9487999999999999e-05 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:165632 12.0 12:00da5945:::benchmark_data_smithi107_479504_object66021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:16.900211+0000", 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.3979734309999996, 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15060358700000001, 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165632 2023-10-09T06:34:21.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900211+0000", 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:21.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900211+0000", 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900210+0000", 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900217+0000", 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9789999999999996e-06 2023-10-09T06:34:21.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900230+0000", 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3040000000000001e-05 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900235+0000", 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4750000000000001e-06 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900281+0000", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5788999999999998e-05 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900360+0000", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8509000000000003e-05 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.900771+0000", 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041106699999999999 2023-10-09T06:34:21.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.902693+0000", 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019225990000000001 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:16.906457+0000", 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037634510000000001 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:21.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:17.050711+0000", 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.144254149 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:17.050765+0000", 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3743999999999999e-05 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:17.050815+0000", 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0037999999999998e-05 2023-10-09T06:34:21.473 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:21.474 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:21.474 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:34:21.641 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:21.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:21.744 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:21.788 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:21.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:21 smithi107 ceph-mon[212750]: 12.19 scrub starts 2023-10-09T06:34:21.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:21 smithi107 ceph-mon[220921]: 12.19 scrub starts 2023-10-09T06:34:22.095 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1272,"stamp":"2023-10-09T06:34:19.393579+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2194773672,"num_objects":33508,"num_object_clones":0,"num_object_copies":100524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3820,"num_objects_misplaced":868,"num_objects_unfound":0,"num_objects_dirty":33508,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160819,"num_write_kb":6175998,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8751,"num_bytes_recovered":561982204,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201873,"ondisk_log_size":201873,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":395,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14233896,"kb_used_data":7590312,"kb_used_omap":20,"kb_used_meta":6643371,"kb_avail":735727320,"statfs":{"total":767960285184,"available":753384775680,"internally_reserved":0,"allocated":7772479488,"data_stored":7760732323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6802812558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,0,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-473661440,"num_objects":-7227,"num_object_clones":0,"num_object_copies":-21681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-492,"num_objects_misplaced":-392,"num_objects_unfound":0,"num_objects_dirty":-7227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2251,"num_write_kb":-198944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":638,"num_bytes_recovered":41811968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003940"},"pg_stats":[{"pgid":"12.15","version":"415'8071","reported_seq":8154,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.376867+0000","last_change":"2023-10-09T06:34:17.052386+0000","last_active":"2023-10-09T06:34:17.376867+0000","last_peered":"2023-10-09T06:34:17.376867+0000","last_clean":"2023-10-09T06:34:17.376867+0000","last_became_active":"2023-10-09T06:32:33.082121+0000","last_became_peered":"2023-10-09T06:32:33.082121+0000","last_unstale":"2023-10-09T06:34:17.376867+0000","last_undegraded":"2023-10-09T06:34:17.376867+0000","last_fullsized":"2023-10-09T06:34:17.376867+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8071,"log_dups_size":0,"ondisk_log_size":8071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68583424,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":199136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":770,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:07.777890+0000","last_change":"2023-10-09T06:34:07.777859+0000","last_active":"2023-10-09T06:34:07.777890+0000","last_peered":"2023-10-09T06:34:07.777890+0000","last_clean":"2023-10-09T06:34:07.777890+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:34:07.777890+0000","last_undegraded":"2023-10-09T06:34:07.777890+0000","last_fullsized":"2023-10-09T06:34:07.777890+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":394,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178152+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:34:05.178152+0000","last_peered":"2023-10-09T06:34:05.178152+0000","last_clean":"2023-10-09T06:34:05.178152+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:34:05.178152+0000","last_undegraded":"2023-10-09T06:34:05.178152+0000","last_fullsized":"2023-10-09T06:34:05.178152+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167097+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:34:05.167097+0000","last_peered":"2023-10-09T06:34:05.167097+0000","last_clean":"2023-10-09T06:34:05.167097+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:34:05.167097+0000","last_undegraded":"2023-10-09T06:34:05.167097+0000","last_fullsized":"2023-10-09T06:34:05.167097+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"415'8042","reported_seq":8114,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.428145+0000","last_change":"2023-10-09T06:32:33.003181+0000","last_active":"2023-10-09T06:34:17.428145+0000","last_peered":"2023-10-09T06:34:17.428145+0000","last_clean":"2023-10-09T06:34:17.428145+0000","last_became_active":"2023-10-09T06:32:32.998596+0000","last_became_peered":"2023-10-09T06:32:32.998596+0000","last_unstale":"2023-10-09T06:34:17.428145+0000","last_undegraded":"2023-10-09T06:34:17.428145+0000","last_fullsized":"2023-10-09T06:34:17.428145+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8042,"log_dups_size":0,"ondisk_log_size":8042,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69337088,"num_objects":1058,"num_object_clones":0,"num_object_copies":3174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":733,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.788684+0000","last_change":"2023-10-09T06:34:13.788684+0000","last_active":"2023-10-09T06:34:13.788684+0000","last_peered":"2023-10-09T06:34:13.788684+0000","last_clean":"2023-10-09T06:34:13.788684+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:34:13.788684+0000","last_undegraded":"2023-10-09T06:34:13.788684+0000","last_fullsized":"2023-10-09T06:34:13.788684+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":408,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172574+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:34:05.172574+0000","last_peered":"2023-10-09T06:34:05.172574+0000","last_clean":"2023-10-09T06:34:05.172574+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:34:05.172574+0000","last_undegraded":"2023-10-09T06:34:05.172574+0000","last_fullsized":"2023-10-09T06:34:05.172574+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":413,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.595110+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:34:13.595110+0000","last_peered":"2023-10-09T06:34:13.595110+0000","last_clean":"2023-10-09T06:34:13.595110+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:34:13.595110+0000","last_undegraded":"2023-10-09T06:34:13.595110+0000","last_fullsized":"2023-10-09T06:34:13.595110+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"415'7995","reported_seq":8068,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.525086+0000","last_change":"2023-10-09T06:32:33.092190+0000","last_active":"2023-10-09T06:34:17.525086+0000","last_peered":"2023-10-09T06:34:17.525086+0000","last_clean":"2023-10-09T06:34:17.525086+0000","last_became_active":"2023-10-09T06:32:33.084295+0000","last_became_peered":"2023-10-09T06:32:33.084295+0000","last_unstale":"2023-10-09T06:34:17.525086+0000","last_undegraded":"2023-10-09T06:34:17.525086+0000","last_fullsized":"2023-10-09T06:34:17.525086+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7995,"log_dups_size":0,"ondisk_log_size":7995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60719104,"num_objects":927,"num_object_clones":0,"num_object_copies":2781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5151,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":662,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177686+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:34:05.177686+0000","last_peered":"2023-10-09T06:34:05.177686+0000","last_clean":"2023-10-09T06:34:05.177686+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:34:05.177686+0000","last_undegraded":"2023-10-09T06:34:05.177686+0000","last_fullsized":"2023-10-09T06:34:05.177686+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166507+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:34:05.166507+0000","last_peered":"2023-10-09T06:34:05.166507+0000","last_clean":"2023-10-09T06:34:05.166507+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:34:05.166507+0000","last_undegraded":"2023-10-09T06:34:05.166507+0000","last_fullsized":"2023-10-09T06:34:05.166507+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":404,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176376+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:34:05.176376+0000","last_peered":"2023-10-09T06:34:05.176376+0000","last_clean":"2023-10-09T06:34:05.176376+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:34:05.176376+0000","last_undegraded":"2023-10-09T06:34:05.176376+0000","last_fullsized":"2023-10-09T06:34:05.176376+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"415'7973","reported_seq":8045,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:18.192405+0000","last_change":"2023-10-09T06:32:33.080997+0000","last_active":"2023-10-09T06:34:18.192405+0000","last_peered":"2023-10-09T06:34:18.192405+0000","last_clean":"2023-10-09T06:34:18.192405+0000","last_became_active":"2023-10-09T06:32:33.077785+0000","last_became_peered":"2023-10-09T06:32:33.077785+0000","last_unstale":"2023-10-09T06:34:18.192405+0000","last_undegraded":"2023-10-09T06:34:18.192405+0000","last_fullsized":"2023-10-09T06:34:18.192405+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70647808,"num_objects":1078,"num_object_clones":0,"num_object_copies":3234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5108,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":523,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173408+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:34:05.173408+0000","last_peered":"2023-10-09T06:34:05.173408+0000","last_clean":"2023-10-09T06:34:05.173408+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:34:05.173408+0000","last_undegraded":"2023-10-09T06:34:05.173408+0000","last_fullsized":"2023-10-09T06:34:05.173408+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.892097+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:34:05.892097+0000","last_peered":"2023-10-09T06:34:05.892097+0000","last_clean":"2023-10-09T06:34:05.892097+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:34:05.892097+0000","last_undegraded":"2023-10-09T06:34:05.892097+0000","last_fullsized":"2023-10-09T06:34:05.892097+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166959+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:34:05.166959+0000","last_peered":"2023-10-09T06:34:05.166959+0000","last_clean":"2023-10-09T06:34:05.166959+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:34:05.166959+0000","last_undegraded":"2023-10-09T06:34:05.166959+0000","last_fullsized":"2023-10-09T06:34:05.166959+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"415'7980","reported_seq":8057,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.376709+0000","last_change":"2023-10-09T06:33:24.993946+0000","last_active":"2023-10-09T06:34:17.376709+0000","last_peered":"2023-10-09T06:34:17.376709+0000","last_clean":"2023-10-09T06:34:17.376709+0000","last_became_active":"2023-10-09T06:32:33.074690+0000","last_became_peered":"2023-10-09T06:32:33.074690+0000","last_unstale":"2023-10-09T06:34:17.376709+0000","last_undegraded":"2023-10-09T06:34:17.376709+0000","last_fullsized":"2023-10-09T06:34:17.376709+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7980,"log_dups_size":0,"ondisk_log_size":7980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66158592,"num_objects":1010,"num_object_clones":0,"num_object_copies":3030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5112,"num_write_kb":195872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":560,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172715+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:34:05.172715+0000","last_peered":"2023-10-09T06:34:05.172715+0000","last_clean":"2023-10-09T06:34:05.172715+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:34:05.172715+0000","last_undegraded":"2023-10-09T06:34:05.172715+0000","last_fullsized":"2023-10-09T06:34:05.172715+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179586+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:34:05.179586+0000","last_peered":"2023-10-09T06:34:05.179586+0000","last_clean":"2023-10-09T06:34:05.179586+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:34:05.179586+0000","last_undegraded":"2023-10-09T06:34:05.179586+0000","last_fullsized":"2023-10-09T06:34:05.179586+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168978+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:34:05.168978+0000","last_peered":"2023-10-09T06:34:05.168978+0000","last_clean":"2023-10-09T06:34:05.168978+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:34:05.168978+0000","last_undegraded":"2023-10-09T06:34:05.168978+0000","last_fullsized":"2023-10-09T06:34:05.168978+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":610,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:16.838424+0000","last_change":"2023-10-09T06:34:16.838424+0000","last_active":"2023-10-09T06:34:16.838424+0000","last_peered":"2023-10-09T06:34:16.838424+0000","last_clean":"2023-10-09T06:34:16.838424+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:34:16.838424+0000","last_undegraded":"2023-10-09T06:34:16.838424+0000","last_fullsized":"2023-10-09T06:34:16.838424+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":296,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173713+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:34:05.173713+0000","last_peered":"2023-10-09T06:34:05.173713+0000","last_clean":"2023-10-09T06:34:05.173713+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:34:05.173713+0000","last_undegraded":"2023-10-09T06:34:05.173713+0000","last_fullsized":"2023-10-09T06:34:05.173713+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":350,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173080+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:34:05.173080+0000","last_peered":"2023-10-09T06:34:05.173080+0000","last_clean":"2023-10-09T06:34:05.173080+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:34:05.173080+0000","last_undegraded":"2023-10-09T06:34:05.173080+0000","last_fullsized":"2023-10-09T06:34:05.173080+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":521,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173531+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:34:05.173531+0000","last_peered":"2023-10-09T06:34:05.173531+0000","last_clean":"2023-10-09T06:34:05.173531+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:34:05.173531+0000","last_undegraded":"2023-10-09T06:34:05.173531+0000","last_fullsized":"2023-10-09T06:34:05.173531+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":421,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178565+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:34:05.178565+0000","last_peered":"2023-10-09T06:34:05.178565+0000","last_clean":"2023-10-09T06:34:05.178565+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:34:05.178565+0000","last_undegraded":"2023-10-09T06:34:05.178565+0000","last_fullsized":"2023-10-09T06:34:05.178565+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":385,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178213+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:34:05.178213+0000","last_peered":"2023-10-09T06:34:05.178213+0000","last_clean":"2023-10-09T06:34:05.178213+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:34:05.178213+0000","last_undegraded":"2023-10-09T06:34:05.178213+0000","last_fullsized":"2023-10-09T06:34:05.178213+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":685,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.163660+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:34:05.163660+0000","last_peered":"2023-10-09T06:34:05.163660+0000","last_clean":"2023-10-09T06:34:05.163660+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:34:05.163660+0000","last_undegraded":"2023-10-09T06:34:05.163660+0000","last_fullsized":"2023-10-09T06:34:05.163660+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":264,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172816+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:34:05.172816+0000","last_peered":"2023-10-09T06:34:05.172816+0000","last_clean":"2023-10-09T06:34:05.172816+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:34:05.172816+0000","last_undegraded":"2023-10-09T06:34:05.172816+0000","last_fullsized":"2023-10-09T06:34:05.172816+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168148+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:34:05.168148+0000","last_peered":"2023-10-09T06:34:05.168148+0000","last_clean":"2023-10-09T06:34:05.168148+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:34:05.168148+0000","last_undegraded":"2023-10-09T06:34:05.168148+0000","last_fullsized":"2023-10-09T06:34:05.168148+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":529,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179174+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:34:05.179174+0000","last_peered":"2023-10-09T06:34:05.179174+0000","last_clean":"2023-10-09T06:34:05.179174+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:34:05.179174+0000","last_undegraded":"2023-10-09T06:34:05.179174+0000","last_fullsized":"2023-10-09T06:34:05.179174+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":411,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168141+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:34:05.168141+0000","last_peered":"2023-10-09T06:34:05.168141+0000","last_clean":"2023-10-09T06:34:05.168141+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:34:05.168141+0000","last_undegraded":"2023-10-09T06:34:05.168141+0000","last_fullsized":"2023-10-09T06:34:05.168141+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179725+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:34:05.179725+0000","last_peered":"2023-10-09T06:34:05.179725+0000","last_clean":"2023-10-09T06:34:05.179725+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:34:05.179725+0000","last_undegraded":"2023-10-09T06:34:05.179725+0000","last_fullsized":"2023-10-09T06:34:05.179725+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":740,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.552748+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:34:12.552748+0000","last_peered":"2023-10-09T06:34:12.552748+0000","last_clean":"2023-10-09T06:34:12.552748+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:34:12.552748+0000","last_undegraded":"2023-10-09T06:34:12.552748+0000","last_fullsized":"2023-10-09T06:34:12.552748+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173215+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:34:05.173215+0000","last_peered":"2023-10-09T06:34:05.173215+0000","last_clean":"2023-10-09T06:34:05.173215+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:34:05.173215+0000","last_undegraded":"2023-10-09T06:34:05.173215+0000","last_fullsized":"2023-10-09T06:34:05.173215+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":88,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.892038+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:34:05.892038+0000","last_peered":"2023-10-09T06:34:05.892038+0000","last_clean":"2023-10-09T06:34:05.892038+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:34:05.892038+0000","last_undegraded":"2023-10-09T06:34:05.892038+0000","last_fullsized":"2023-10-09T06:34:05.892038+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":529,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179799+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:34:05.179799+0000","last_peered":"2023-10-09T06:34:05.179799+0000","last_clean":"2023-10-09T06:34:05.179799+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:34:05.179799+0000","last_undegraded":"2023-10-09T06:34:05.179799+0000","last_fullsized":"2023-10-09T06:34:05.179799+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":493,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.175675+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:34:05.175675+0000","last_peered":"2023-10-09T06:34:05.175675+0000","last_clean":"2023-10-09T06:34:05.175675+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:34:05.175675+0000","last_undegraded":"2023-10-09T06:34:05.175675+0000","last_fullsized":"2023-10-09T06:34:05.175675+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167718+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:34:05.167718+0000","last_peered":"2023-10-09T06:34:05.167718+0000","last_clean":"2023-10-09T06:34:05.167718+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:34:05.167718+0000","last_undegraded":"2023-10-09T06:34:05.167718+0000","last_fullsized":"2023-10-09T06:34:05.167718+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":542,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173461+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:34:05.173461+0000","last_peered":"2023-10-09T06:34:05.173461+0000","last_clean":"2023-10-09T06:34:05.173461+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:34:05.173461+0000","last_undegraded":"2023-10-09T06:34:05.173461+0000","last_fullsized":"2023-10-09T06:34:05.173461+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"415'8122","reported_seq":8254,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.427922+0000","last_change":"2023-10-09T06:34:08.464064+0000","last_active":"2023-10-09T06:34:17.427922+0000","last_peered":"2023-10-09T06:34:17.427922+0000","last_clean":"2023-10-09T06:34:17.427922+0000","last_became_active":"2023-10-09T06:32:33.069345+0000","last_became_peered":"2023-10-09T06:32:33.069345+0000","last_unstale":"2023-10-09T06:34:17.427922+0000","last_undegraded":"2023-10-09T06:34:17.427922+0000","last_fullsized":"2023-10-09T06:34:17.427922+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'7928","last_scrub_stamp":"2023-10-09T06:34:08.463956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:34:08.463956+0000","objects_scrubbed":1240,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:39.228319+0000","scrub_duration":0.275394735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68485120,"num_objects":1045,"num_object_clones":0,"num_object_copies":3135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":195,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":300,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173224+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:34:05.173224+0000","last_peered":"2023-10-09T06:34:05.173224+0000","last_clean":"2023-10-09T06:34:05.173224+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:34:05.173224+0000","last_undegraded":"2023-10-09T06:34:05.173224+0000","last_fullsized":"2023-10-09T06:34:05.173224+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179265+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:34:05.179265+0000","last_peered":"2023-10-09T06:34:05.179265+0000","last_clean":"2023-10-09T06:34:05.179265+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:34:05.179265+0000","last_undegraded":"2023-10-09T06:34:05.179265+0000","last_fullsized":"2023-10-09T06:34:05.179265+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":799,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254857+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:34:08.254857+0000","last_peered":"2023-10-09T06:34:08.254857+0000","last_clean":"2023-10-09T06:34:08.254857+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:34:08.254857+0000","last_undegraded":"2023-10-09T06:34:08.254857+0000","last_fullsized":"2023-10-09T06:34:08.254857+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"415'8156","reported_seq":8238,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.388293+0000","last_change":"2023-10-09T06:34:00.106009+0000","last_active":"2023-10-09T06:34:17.388293+0000","last_peered":"2023-10-09T06:34:17.388293+0000","last_clean":"2023-10-09T06:34:17.388293+0000","last_became_active":"2023-10-09T06:32:33.000165+0000","last_became_peered":"2023-10-09T06:32:33.000165+0000","last_unstale":"2023-10-09T06:34:17.388293+0000","last_undegraded":"2023-10-09T06:34:17.388293+0000","last_fullsized":"2023-10-09T06:34:17.388293+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67764224,"num_objects":1034,"num_object_clones":0,"num_object_copies":3102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5248,"num_write_kb":201024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":262,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173055+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:34:05.173055+0000","last_peered":"2023-10-09T06:34:05.173055+0000","last_clean":"2023-10-09T06:34:05.173055+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:34:05.173055+0000","last_undegraded":"2023-10-09T06:34:05.173055+0000","last_fullsized":"2023-10-09T06:34:05.173055+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":412,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177869+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:34:05.177869+0000","last_peered":"2023-10-09T06:34:05.177869+0000","last_clean":"2023-10-09T06:34:05.177869+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:34:05.177869+0000","last_undegraded":"2023-10-09T06:34:05.177869+0000","last_fullsized":"2023-10-09T06:34:05.177869+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":716,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176231+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:34:05.176231+0000","last_peered":"2023-10-09T06:34:05.176231+0000","last_clean":"2023-10-09T06:34:05.176231+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:34:05.176231+0000","last_undegraded":"2023-10-09T06:34:05.176231+0000","last_fullsized":"2023-10-09T06:34:05.176231+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"415'8103","reported_seq":8188,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.417475+0000","last_change":"2023-10-09T06:34:13.198883+0000","last_active":"2023-10-09T06:34:17.417475+0000","last_peered":"2023-10-09T06:34:17.417475+0000","last_clean":"2023-10-09T06:34:17.417475+0000","last_became_active":"2023-10-09T06:32:32.996849+0000","last_became_peered":"2023-10-09T06:32:32.996849+0000","last_unstale":"2023-10-09T06:34:17.417475+0000","last_undegraded":"2023-10-09T06:34:17.417475+0000","last_fullsized":"2023-10-09T06:34:17.417475+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8103,"log_dups_size":0,"ondisk_log_size":8103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68648960,"num_objects":1047,"num_object_clones":0,"num_object_copies":3141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5199,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":318,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173581+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:34:05.173581+0000","last_peered":"2023-10-09T06:34:05.173581+0000","last_clean":"2023-10-09T06:34:05.173581+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:34:05.173581+0000","last_undegraded":"2023-10-09T06:34:05.173581+0000","last_fullsized":"2023-10-09T06:34:05.173581+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":391,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177016+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:34:05.177016+0000","last_peered":"2023-10-09T06:34:05.177016+0000","last_clean":"2023-10-09T06:34:05.177016+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:34:05.177016+0000","last_undegraded":"2023-10-09T06:34:05.177016+0000","last_fullsized":"2023-10-09T06:34:05.177016+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":539,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178116+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:34:05.178116+0000","last_peered":"2023-10-09T06:34:05.178116+0000","last_clean":"2023-10-09T06:34:05.178116+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:34:05.178116+0000","last_undegraded":"2023-10-09T06:34:05.178116+0000","last_fullsized":"2023-10-09T06:34:05.178116+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"415'8062","reported_seq":8136,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.853757+0000","last_change":"2023-10-09T06:32:32.995971+0000","last_active":"2023-10-09T06:34:15.853757+0000","last_peered":"2023-10-09T06:34:15.853757+0000","last_clean":"2023-10-09T06:34:15.853757+0000","last_became_active":"2023-10-09T06:32:32.993707+0000","last_became_peered":"2023-10-09T06:32:32.993707+0000","last_unstale":"2023-10-09T06:34:15.853757+0000","last_undegraded":"2023-10-09T06:34:15.853757+0000","last_fullsized":"2023-10-09T06:34:15.853757+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8062,"log_dups_size":0,"ondisk_log_size":8062,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71991296,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5170,"num_write_kb":200608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179395+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:34:05.179395+0000","last_peered":"2023-10-09T06:34:05.179395+0000","last_clean":"2023-10-09T06:34:05.179395+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:34:05.179395+0000","last_undegraded":"2023-10-09T06:34:05.179395+0000","last_fullsized":"2023-10-09T06:34:05.179395+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":375,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.170414+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:34:05.170414+0000","last_peered":"2023-10-09T06:34:05.170414+0000","last_clean":"2023-10-09T06:34:05.170414+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:34:05.170414+0000","last_undegraded":"2023-10-09T06:34:05.170414+0000","last_fullsized":"2023-10-09T06:34:05.170414+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"415'7992","reported_seq":8072,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.352918+0000","last_change":"2023-10-09T06:34:11.068769+0000","last_active":"2023-10-09T06:34:17.352918+0000","last_peered":"2023-10-09T06:34:17.352918+0000","last_clean":"2023-10-09T06:34:17.352918+0000","last_became_active":"2023-10-09T06:32:32.995923+0000","last_became_peered":"2023-10-09T06:32:32.995923+0000","last_unstale":"2023-10-09T06:34:17.352918+0000","last_undegraded":"2023-10-09T06:34:17.352918+0000","last_fullsized":"2023-10-09T06:34:17.352918+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5120,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":767,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173067+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:34:05.173067+0000","last_peered":"2023-10-09T06:34:05.173067+0000","last_clean":"2023-10-09T06:34:05.173067+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:34:05.173067+0000","last_undegraded":"2023-10-09T06:34:05.173067+0000","last_fullsized":"2023-10-09T06:34:05.173067+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":140,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167488+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:34:05.167488+0000","last_peered":"2023-10-09T06:34:05.167488+0000","last_clean":"2023-10-09T06:34:05.167488+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:34:05.167488+0000","last_undegraded":"2023-10-09T06:34:05.167488+0000","last_fullsized":"2023-10-09T06:34:05.167488+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":395,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178466+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:34:05.178466+0000","last_peered":"2023-10-09T06:34:05.178466+0000","last_clean":"2023-10-09T06:34:05.178466+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:34:05.178466+0000","last_undegraded":"2023-10-09T06:34:05.178466+0000","last_fullsized":"2023-10-09T06:34:05.178466+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"415'8088","reported_seq":8168,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.430906+0000","last_change":"2023-10-09T06:34:14.119582+0000","last_active":"2023-10-09T06:34:17.430906+0000","last_peered":"2023-10-09T06:34:17.430906+0000","last_clean":"2023-10-09T06:34:17.430906+0000","last_became_active":"2023-10-09T06:32:33.044765+0000","last_became_peered":"2023-10-09T06:32:33.044765+0000","last_unstale":"2023-10-09T06:34:17.430906+0000","last_undegraded":"2023-10-09T06:34:17.430906+0000","last_fullsized":"2023-10-09T06:34:17.430906+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8088,"log_dups_size":0,"ondisk_log_size":8088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68780032,"num_objects":1049,"num_object_clones":0,"num_object_copies":3147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5193,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":747,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179229+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:34:05.179229+0000","last_peered":"2023-10-09T06:34:05.179229+0000","last_clean":"2023-10-09T06:34:05.179229+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:34:05.179229+0000","last_undegraded":"2023-10-09T06:34:05.179229+0000","last_fullsized":"2023-10-09T06:34:05.179229+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":372,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176940+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:34:05.176940+0000","last_peered":"2023-10-09T06:34:05.176940+0000","last_clean":"2023-10-09T06:34:05.176940+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:34:05.176940+0000","last_undegraded":"2023-10-09T06:34:05.176940+0000","last_fullsized":"2023-10-09T06:34:05.176940+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":392,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178190+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:34:05.178190+0000","last_peered":"2023-10-09T06:34:05.178190+0000","last_clean":"2023-10-09T06:34:05.178190+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:34:05.178190+0000","last_undegraded":"2023-10-09T06:34:05.178190+0000","last_fullsized":"2023-10-09T06:34:05.178190+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":749,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:07.538387+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:34:07.538387+0000","last_peered":"2023-10-09T06:34:07.538387+0000","last_clean":"2023-10-09T06:34:07.538387+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:34:07.538387+0000","last_undegraded":"2023-10-09T06:34:07.538387+0000","last_fullsized":"2023-10-09T06:34:07.538387+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"415'8110","reported_seq":8182,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.510537+0000","last_change":"2023-10-09T06:32:32.976279+0000","last_active":"2023-10-09T06:34:17.510537+0000","last_peered":"2023-10-09T06:34:17.510537+0000","last_clean":"2023-10-09T06:34:17.510537+0000","last_became_active":"2023-10-09T06:32:32.973171+0000","last_became_peered":"2023-10-09T06:32:32.973171+0000","last_unstale":"2023-10-09T06:34:17.510537+0000","last_undegraded":"2023-10-09T06:34:17.510537+0000","last_fullsized":"2023-10-09T06:34:17.510537+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8110,"log_dups_size":0,"ondisk_log_size":8110,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5226,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177683+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:34:05.177683+0000","last_peered":"2023-10-09T06:34:05.177683+0000","last_clean":"2023-10-09T06:34:05.177683+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:34:05.177683+0000","last_undegraded":"2023-10-09T06:34:05.177683+0000","last_fullsized":"2023-10-09T06:34:05.177683+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":375,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178340+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:34:05.178340+0000","last_peered":"2023-10-09T06:34:05.178340+0000","last_clean":"2023-10-09T06:34:05.178340+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:34:05.178340+0000","last_undegraded":"2023-10-09T06:34:05.178340+0000","last_fullsized":"2023-10-09T06:34:05.178340+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"415'7859","reported_seq":7933,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.507861+0000","last_change":"2023-10-09T06:32:33.079508+0000","last_active":"2023-10-09T06:34:17.507861+0000","last_peered":"2023-10-09T06:34:17.507861+0000","last_clean":"2023-10-09T06:34:17.507861+0000","last_became_active":"2023-10-09T06:32:33.075123+0000","last_became_peered":"2023-10-09T06:32:33.075123+0000","last_unstale":"2023-10-09T06:34:17.507861+0000","last_undegraded":"2023-10-09T06:34:17.507861+0000","last_fullsized":"2023-10-09T06:34:17.507861+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7859,"log_dups_size":0,"ondisk_log_size":7859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74743808,"num_objects":1140,"num_object_clones":0,"num_object_copies":3420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5014,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":725,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179429+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:34:05.179429+0000","last_peered":"2023-10-09T06:34:05.179429+0000","last_clean":"2023-10-09T06:34:05.179429+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:34:05.179429+0000","last_undegraded":"2023-10-09T06:34:05.179429+0000","last_fullsized":"2023-10-09T06:34:05.179429+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254719+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:34:08.254719+0000","last_peered":"2023-10-09T06:34:08.254719+0000","last_clean":"2023-10-09T06:34:08.254719+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:34:08.254719+0000","last_undegraded":"2023-10-09T06:34:08.254719+0000","last_fullsized":"2023-10-09T06:34:08.254719+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173759+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:34:05.173759+0000","last_peered":"2023-10-09T06:34:05.173759+0000","last_clean":"2023-10-09T06:34:05.173759+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:34:05.173759+0000","last_undegraded":"2023-10-09T06:34:05.173759+0000","last_fullsized":"2023-10-09T06:34:05.173759+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"415'10301","reported_seq":10365,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.876573+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:34:15.876573+0000","last_peered":"2023-10-09T06:34:15.876573+0000","last_clean":"2023-10-09T06:34:15.876573+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:34:15.876573+0000","last_undegraded":"2023-10-09T06:34:15.876573+0000","last_fullsized":"2023-10-09T06:34:15.876573+0000","mapping_epoch":361,"log_start":"372'3700","ondisk_log_start":"372'3700","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10301,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":707,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176449+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:34:05.176449+0000","last_peered":"2023-10-09T06:34:05.176449+0000","last_clean":"2023-10-09T06:34:05.176449+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:34:05.176449+0000","last_undegraded":"2023-10-09T06:34:05.176449+0000","last_fullsized":"2023-10-09T06:34:05.176449+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":420,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173148+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:34:05.173148+0000","last_peered":"2023-10-09T06:34:05.173148+0000","last_clean":"2023-10-09T06:34:05.173148+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:34:05.173148+0000","last_undegraded":"2023-10-09T06:34:05.173148+0000","last_fullsized":"2023-10-09T06:34:05.173148+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179678+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:34:05.179678+0000","last_peered":"2023-10-09T06:34:05.179678+0000","last_clean":"2023-10-09T06:34:05.179678+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:34:05.179678+0000","last_undegraded":"2023-10-09T06:34:05.179678+0000","last_fullsized":"2023-10-09T06:34:05.179678+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":354,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172807+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:34:05.172807+0000","last_peered":"2023-10-09T06:34:05.172807+0000","last_clean":"2023-10-09T06:34:05.172807+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:34:05.172807+0000","last_undegraded":"2023-10-09T06:34:05.172807+0000","last_fullsized":"2023-10-09T06:34:05.172807+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"415'10321","reported_seq":10384,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.876408+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:34:15.876408+0000","last_peered":"2023-10-09T06:34:15.876408+0000","last_clean":"2023-10-09T06:34:15.876408+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:34:15.876408+0000","last_undegraded":"2023-10-09T06:34:15.876408+0000","last_fullsized":"2023-10-09T06:34:15.876408+0000","mapping_epoch":361,"log_start":"372'3700","ondisk_log_start":"372'3700","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142016512,"num_objects":2167,"num_object_clones":0,"num_object_copies":6501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10321,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":761,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179358+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:34:05.179358+0000","last_peered":"2023-10-09T06:34:05.179358+0000","last_clean":"2023-10-09T06:34:05.179358+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:34:05.179358+0000","last_undegraded":"2023-10-09T06:34:05.179358+0000","last_fullsized":"2023-10-09T06:34:05.179358+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179749+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:34:05.179749+0000","last_peered":"2023-10-09T06:34:05.179749+0000","last_clean":"2023-10-09T06:34:05.179749+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:34:05.179749+0000","last_undegraded":"2023-10-09T06:34:05.179749+0000","last_fullsized":"2023-10-09T06:34:05.179749+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":396,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178130+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:34:05.178130+0000","last_peered":"2023-10-09T06:34:05.178130+0000","last_clean":"2023-10-09T06:34:05.178130+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:34:05.178130+0000","last_undegraded":"2023-10-09T06:34:05.178130+0000","last_fullsized":"2023-10-09T06:34:05.178130+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":384,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173073+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:34:05.173073+0000","last_peered":"2023-10-09T06:34:05.173073+0000","last_clean":"2023-10-09T06:34:05.173073+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:34:05.173073+0000","last_undegraded":"2023-10-09T06:34:05.173073+0000","last_fullsized":"2023-10-09T06:34:05.173073+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"415'10467","reported_seq":10529,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.402012+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:34:17.402012+0000","last_peered":"2023-10-09T06:34:17.402012+0000","last_clean":"2023-10-09T06:34:17.402012+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:34:17.402012+0000","last_undegraded":"2023-10-09T06:34:17.402012+0000","last_fullsized":"2023-10-09T06:34:17.402012+0000","mapping_epoch":361,"log_start":"380'4600","ondisk_log_start":"380'4600","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135725056,"num_objects":2071,"num_object_clones":0,"num_object_copies":6213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10467,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":757,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172903+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:34:05.172903+0000","last_peered":"2023-10-09T06:34:05.172903+0000","last_clean":"2023-10-09T06:34:05.172903+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:34:05.172903+0000","last_undegraded":"2023-10-09T06:34:05.172903+0000","last_fullsized":"2023-10-09T06:34:05.172903+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":405,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168588+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:34:05.168588+0000","last_peered":"2023-10-09T06:34:05.168588+0000","last_clean":"2023-10-09T06:34:05.168588+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:34:05.168588+0000","last_undegraded":"2023-10-09T06:34:05.168588+0000","last_fullsized":"2023-10-09T06:34:05.168588+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":392,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173267+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:34:05.173267+0000","last_peered":"2023-10-09T06:34:05.173267+0000","last_clean":"2023-10-09T06:34:05.173267+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:34:05.173267+0000","last_undegraded":"2023-10-09T06:34:05.173267+0000","last_fullsized":"2023-10-09T06:34:05.173267+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":414,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173826+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:34:05.173826+0000","last_peered":"2023-10-09T06:34:05.173826+0000","last_clean":"2023-10-09T06:34:05.173826+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:34:05.173826+0000","last_undegraded":"2023-10-09T06:34:05.173826+0000","last_fullsized":"2023-10-09T06:34:05.173826+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"415'10544","reported_seq":10607,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.415750+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:34:17.415750+0000","last_peered":"2023-10-09T06:34:17.415750+0000","last_clean":"2023-10-09T06:34:17.415750+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:34:17.415750+0000","last_undegraded":"2023-10-09T06:34:17.415750+0000","last_fullsized":"2023-10-09T06:34:17.415750+0000","mapping_epoch":361,"log_start":"383'5149","ondisk_log_start":"383'5149","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5395,"log_dups_size":0,"ondisk_log_size":5395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135594008,"num_objects":2070,"num_object_clones":0,"num_object_copies":6210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2070,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10544,"num_write_kb":403585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179473+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:34:05.179473+0000","last_peered":"2023-10-09T06:34:05.179473+0000","last_clean":"2023-10-09T06:34:05.179473+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:34:05.179473+0000","last_undegraded":"2023-10-09T06:34:05.179473+0000","last_fullsized":"2023-10-09T06:34:05.179473+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":738,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173308+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:34:05.173308+0000","last_peered":"2023-10-09T06:34:05.173308+0000","last_clean":"2023-10-09T06:34:05.173308+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:34:05.173308+0000","last_undegraded":"2023-10-09T06:34:05.173308+0000","last_fullsized":"2023-10-09T06:34:05.173308+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":124,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.170733+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:34:05.170733+0000","last_peered":"2023-10-09T06:34:05.170733+0000","last_clean":"2023-10-09T06:34:05.170733+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:34:05.170733+0000","last_undegraded":"2023-10-09T06:34:05.170733+0000","last_fullsized":"2023-10-09T06:34:05.170733+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":377,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176926+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:34:05.176926+0000","last_peered":"2023-10-09T06:34:05.176926+0000","last_clean":"2023-10-09T06:34:05.176926+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:34:05.176926+0000","last_undegraded":"2023-10-09T06:34:05.176926+0000","last_fullsized":"2023-10-09T06:34:05.176926+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"415'10486","reported_seq":10550,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.891611+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:34:15.891611+0000","last_peered":"2023-10-09T06:34:15.891611+0000","last_clean":"2023-10-09T06:34:15.891611+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:34:15.891611+0000","last_undegraded":"2023-10-09T06:34:15.891611+0000","last_fullsized":"2023-10-09T06:34:15.891611+0000","mapping_epoch":361,"log_start":"373'3900","ondisk_log_start":"373'3900","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142213120,"num_objects":2170,"num_object_clones":0,"num_object_copies":6510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10486,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":723,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.174398+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:34:05.174398+0000","last_peered":"2023-10-09T06:34:05.174398+0000","last_clean":"2023-10-09T06:34:05.174398+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:34:05.174398+0000","last_undegraded":"2023-10-09T06:34:05.174398+0000","last_fullsized":"2023-10-09T06:34:05.174398+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":278,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173177+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:34:05.173177+0000","last_peered":"2023-10-09T06:34:05.173177+0000","last_clean":"2023-10-09T06:34:05.173177+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:34:05.173177+0000","last_undegraded":"2023-10-09T06:34:05.173177+0000","last_fullsized":"2023-10-09T06:34:05.173177+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":88,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.163857+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:34:05.163857+0000","last_peered":"2023-10-09T06:34:05.163857+0000","last_clean":"2023-10-09T06:34:05.163857+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:34:05.163857+0000","last_undegraded":"2023-10-09T06:34:05.163857+0000","last_fullsized":"2023-10-09T06:34:05.163857+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":399,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.595210+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:34:13.595210+0000","last_peered":"2023-10-09T06:34:13.595210+0000","last_clean":"2023-10-09T06:34:13.595210+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:34:13.595210+0000","last_undegraded":"2023-10-09T06:34:13.595210+0000","last_fullsized":"2023-10-09T06:34:13.595210+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"415'10263","reported_seq":10327,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:18.230149+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:34:18.230149+0000","last_peered":"2023-10-09T06:34:18.230149+0000","last_clean":"2023-10-09T06:34:18.230149+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:34:18.230149+0000","last_undegraded":"2023-10-09T06:34:18.230149+0000","last_fullsized":"2023-10-09T06:34:18.230149+0000","mapping_epoch":361,"log_start":"383'5009","ondisk_log_start":"383'5009","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10263,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":737,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.175933+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:34:05.175933+0000","last_peered":"2023-10-09T06:34:05.175933+0000","last_clean":"2023-10-09T06:34:05.175933+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:34:05.175933+0000","last_undegraded":"2023-10-09T06:34:05.175933+0000","last_fullsized":"2023-10-09T06:34:05.175933+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":278,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173654+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:34:05.173654+0000","last_peered":"2023-10-09T06:34:05.173654+0000","last_clean":"2023-10-09T06:34:05.173654+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:34:05.173654+0000","last_undegraded":"2023-10-09T06:34:05.173654+0000","last_fullsized":"2023-10-09T06:34:05.173654+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":410,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173002+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:34:05.173002+0000","last_peered":"2023-10-09T06:34:05.173002+0000","last_clean":"2023-10-09T06:34:05.173002+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:34:05.173002+0000","last_undegraded":"2023-10-09T06:34:05.173002+0000","last_fullsized":"2023-10-09T06:34:05.173002+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":397,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.170656+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:34:05.170656+0000","last_peered":"2023-10-09T06:34:05.170656+0000","last_clean":"2023-10-09T06:34:05.170656+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:34:05.170656+0000","last_undegraded":"2023-10-09T06:34:05.170656+0000","last_fullsized":"2023-10-09T06:34:05.170656+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"415'8108","reported_seq":8182,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:18.213076+0000","last_change":"2023-10-09T06:32:47.459790+0000","last_active":"2023-10-09T06:34:18.213076+0000","last_peered":"2023-10-09T06:34:18.213076+0000","last_clean":"2023-10-09T06:34:18.213076+0000","last_became_active":"2023-10-09T06:32:33.070761+0000","last_became_peered":"2023-10-09T06:32:33.070761+0000","last_unstale":"2023-10-09T06:34:18.213076+0000","last_undegraded":"2023-10-09T06:34:18.213076+0000","last_fullsized":"2023-10-09T06:34:18.213076+0000","mapping_epoch":361,"log_start":"365'299","ondisk_log_start":"365'299","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7809,"log_dups_size":0,"ondisk_log_size":7809,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5243,"num_write_kb":197824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":738,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178041+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:34:05.178041+0000","last_peered":"2023-10-09T06:34:05.178041+0000","last_clean":"2023-10-09T06:34:05.178041+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:34:05.178041+0000","last_undegraded":"2023-10-09T06:34:05.178041+0000","last_fullsized":"2023-10-09T06:34:05.178041+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":145,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254940+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:34:08.254940+0000","last_peered":"2023-10-09T06:34:08.254940+0000","last_clean":"2023-10-09T06:34:08.254940+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:34:08.254940+0000","last_undegraded":"2023-10-09T06:34:08.254940+0000","last_fullsized":"2023-10-09T06:34:08.254940+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":377,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173708+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:34:05.173708+0000","last_peered":"2023-10-09T06:34:05.173708+0000","last_clean":"2023-10-09T06:34:05.173708+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:34:05.173708+0000","last_undegraded":"2023-10-09T06:34:05.173708+0000","last_fullsized":"2023-10-09T06:34:05.173708+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166698+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:34:05.166698+0000","last_peered":"2023-10-09T06:34:05.166698+0000","last_clean":"2023-10-09T06:34:05.166698+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:34:05.166698+0000","last_undegraded":"2023-10-09T06:34:05.166698+0000","last_fullsized":"2023-10-09T06:34:05.166698+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"415'8010","reported_seq":8091,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.249098+0000","last_change":"2023-10-09T06:34:05.172567+0000","last_active":"2023-10-09T06:34:17.249098+0000","last_peered":"2023-10-09T06:34:17.249098+0000","last_clean":"2023-10-09T06:34:17.249098+0000","last_became_active":"2023-10-09T06:32:33.044852+0000","last_became_peered":"2023-10-09T06:32:33.044852+0000","last_unstale":"2023-10-09T06:34:17.249098+0000","last_undegraded":"2023-10-09T06:34:17.249098+0000","last_fullsized":"2023-10-09T06:34:17.249098+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8010,"log_dups_size":0,"ondisk_log_size":8010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68386816,"num_objects":1043,"num_object_clones":0,"num_object_copies":3129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":197920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":767,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168295+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:34:05.168295+0000","last_peered":"2023-10-09T06:34:05.168295+0000","last_clean":"2023-10-09T06:34:05.168295+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:34:05.168295+0000","last_undegraded":"2023-10-09T06:34:05.168295+0000","last_fullsized":"2023-10-09T06:34:05.168295+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":399,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176578+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:34:05.176578+0000","last_peered":"2023-10-09T06:34:05.176578+0000","last_clean":"2023-10-09T06:34:05.176578+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:34:05.176578+0000","last_undegraded":"2023-10-09T06:34:05.176578+0000","last_fullsized":"2023-10-09T06:34:05.176578+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":380,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594987+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:34:13.594987+0000","last_peered":"2023-10-09T06:34:13.594987+0000","last_clean":"2023-10-09T06:34:13.594987+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:34:13.594987+0000","last_undegraded":"2023-10-09T06:34:13.594987+0000","last_fullsized":"2023-10-09T06:34:13.594987+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177866+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:34:05.177866+0000","last_peered":"2023-10-09T06:34:05.177866+0000","last_clean":"2023-10-09T06:34:05.177866+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:34:05.177866+0000","last_undegraded":"2023-10-09T06:34:05.177866+0000","last_fullsized":"2023-10-09T06:34:05.177866+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"415'8044","reported_seq":8129,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.404576+0000","last_change":"2023-10-09T06:34:16.084491+0000","last_active":"2023-10-09T06:34:17.404576+0000","last_peered":"2023-10-09T06:34:17.404576+0000","last_clean":"2023-10-09T06:34:17.404576+0000","last_became_active":"2023-10-09T06:32:33.005074+0000","last_became_peered":"2023-10-09T06:32:33.005074+0000","last_unstale":"2023-10-09T06:34:17.404576+0000","last_undegraded":"2023-10-09T06:34:17.404576+0000","last_fullsized":"2023-10-09T06:34:17.404576+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8044,"log_dups_size":0,"ondisk_log_size":8044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67108864,"num_objects":1024,"num_object_clones":0,"num_object_copies":3072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":693,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168352+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:34:05.168352+0000","last_peered":"2023-10-09T06:34:05.168352+0000","last_clean":"2023-10-09T06:34:05.168352+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:34:05.168352+0000","last_undegraded":"2023-10-09T06:34:05.168352+0000","last_fullsized":"2023-10-09T06:34:05.168352+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":381,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.170577+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:34:05.170577+0000","last_peered":"2023-10-09T06:34:05.170577+0000","last_clean":"2023-10-09T06:34:05.170577+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:34:05.170577+0000","last_undegraded":"2023-10-09T06:34:05.170577+0000","last_fullsized":"2023-10-09T06:34:05.170577+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":403,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172928+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:34:05.172928+0000","last_peered":"2023-10-09T06:34:05.172928+0000","last_clean":"2023-10-09T06:34:05.172928+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:34:05.172928+0000","last_undegraded":"2023-10-09T06:34:05.172928+0000","last_fullsized":"2023-10-09T06:34:05.172928+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.13","version":"415'8040","reported_seq":9133,"reported_epoch":415,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:15.891514+0000","last_change":"2023-10-09T06:34:03.204624+0000","last_active":"2023-10-09T06:34:15.891514+0000","last_peered":"2023-10-09T06:34:15.891514+0000","last_clean":"2023-10-09T06:32:51.037365+0000","last_became_active":"2023-10-09T06:32:53.531678+0000","last_became_peered":"2023-10-09T06:32:53.531678+0000","last_unstale":"2023-10-09T06:34:15.891514+0000","last_undegraded":"2023-10-09T06:32:53.443414+0000","last_fullsized":"2023-10-09T06:32:53.427850+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8040,"log_dups_size":0,"ondisk_log_size":8040,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71467008,"num_objects":1091,"num_object_clones":0,"num_object_copies":3273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1578,"num_objects_misplaced":868,"num_objects_unfound":0,"num_objects_dirty":1091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5149,"num_write_kb":199648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":507,"num_bytes_recovered":33226752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":868},{"shards":"0,7","objects":-1355}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":624,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594715+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:34:13.594715+0000","last_peered":"2023-10-09T06:34:13.594715+0000","last_clean":"2023-10-09T06:34:13.594715+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:34:13.594715+0000","last_undegraded":"2023-10-09T06:34:13.594715+0000","last_fullsized":"2023-10-09T06:34:13.594715+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":106,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179491+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:34:05.179491+0000","last_peered":"2023-10-09T06:34:05.179491+0000","last_clean":"2023-10-09T06:34:05.179491+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:34:05.179491+0000","last_undegraded":"2023-10-09T06:34:05.179491+0000","last_fullsized":"2023-10-09T06:34:05.179491+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178064+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:34:05.178064+0000","last_peered":"2023-10-09T06:34:05.178064+0000","last_clean":"2023-10-09T06:34:05.178064+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:34:05.178064+0000","last_undegraded":"2023-10-09T06:34:05.178064+0000","last_fullsized":"2023-10-09T06:34:05.178064+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"415'8185","reported_seq":14297,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:18.248734+0000","last_change":"2023-10-09T06:32:52.162166+0000","last_active":"2023-10-09T06:34:18.248734+0000","last_peered":"2023-10-09T06:34:18.248734+0000","last_clean":"2023-10-09T06:32:48.657566+0000","last_became_active":"2023-10-09T06:32:52.155917+0000","last_became_peered":"2023-10-09T06:32:52.155917+0000","last_unstale":"2023-10-09T06:34:18.248734+0000","last_undegraded":"2023-10-09T06:32:52.150896+0000","last_fullsized":"2023-10-09T06:32:52.051991+0000","mapping_epoch":400,"log_start":"365'252","ondisk_log_start":"365'252","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7933,"log_dups_size":0,"ondisk_log_size":7933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63668224,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2242,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5282,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2012,"num_bytes_recovered":131858432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"1,3","objects":-6},{"shards":"3,4,7","objects":978}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":106,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179590+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:34:05.179590+0000","last_peered":"2023-10-09T06:34:05.179590+0000","last_clean":"2023-10-09T06:34:05.179590+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:34:05.179590+0000","last_undegraded":"2023-10-09T06:34:05.179590+0000","last_fullsized":"2023-10-09T06:34:05.179590+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":432,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178292+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:34:05.178292+0000","last_peered":"2023-10-09T06:34:05.178292+0000","last_clean":"2023-10-09T06:34:05.178292+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:34:05.178292+0000","last_undegraded":"2023-10-09T06:34:05.178292+0000","last_fullsized":"2023-10-09T06:34:05.178292+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"415'8105","reported_seq":17579,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:17.365013+0000","last_change":"2023-10-09T06:34:06.782665+0000","last_active":"2023-10-09T06:34:17.365013+0000","last_peered":"2023-10-09T06:34:17.365013+0000","last_clean":"2023-10-09T06:34:17.365013+0000","last_became_active":"2023-10-09T06:33:43.622574+0000","last_became_peered":"2023-10-09T06:33:43.622574+0000","last_unstale":"2023-10-09T06:34:17.365013+0000","last_undegraded":"2023-10-09T06:34:17.365013+0000","last_fullsized":"2023-10-09T06:34:17.365013+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8105,"log_dups_size":0,"ondisk_log_size":8105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5197,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":196673536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":771,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172667+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:34:05.172667+0000","last_peered":"2023-10-09T06:34:05.172667+0000","last_clean":"2023-10-09T06:34:05.172667+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:34:05.172667+0000","last_undegraded":"2023-10-09T06:34:05.172667+0000","last_fullsized":"2023-10-09T06:34:05.172667+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":115,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167379+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:34:05.167379+0000","last_peered":"2023-10-09T06:34:05.167379+0000","last_clean":"2023-10-09T06:34:05.167379+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:34:05.167379+0000","last_undegraded":"2023-10-09T06:34:05.167379+0000","last_fullsized":"2023-10-09T06:34:05.167379+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168860+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:34:05.168860+0000","last_peered":"2023-10-09T06:34:05.168860+0000","last_clean":"2023-10-09T06:34:05.168860+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:34:05.168860+0000","last_undegraded":"2023-10-09T06:34:05.168860+0000","last_fullsized":"2023-10-09T06:34:05.168860+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"415'8009","reported_seq":20280,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.736550+0000","last_change":"2023-10-09T06:34:05.191255+0000","last_active":"2023-10-09T06:34:15.736550+0000","last_peered":"2023-10-09T06:34:15.736550+0000","last_clean":"2023-10-09T06:34:15.736550+0000","last_became_active":"2023-10-09T06:34:05.174880+0000","last_became_peered":"2023-10-09T06:34:05.174880+0000","last_unstale":"2023-10-09T06:34:15.736550+0000","last_undegraded":"2023-10-09T06:34:15.736550+0000","last_fullsized":"2023-10-09T06:34:15.736550+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8009,"log_dups_size":0,"ondisk_log_size":8009,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73826304,"num_objects":1127,"num_object_clones":0,"num_object_copies":3381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5121,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3046,"num_bytes_recovered":199622656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":535,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173525+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:34:05.173525+0000","last_peered":"2023-10-09T06:34:05.173525+0000","last_clean":"2023-10-09T06:34:05.173525+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:34:05.173525+0000","last_undegraded":"2023-10-09T06:34:05.173525+0000","last_fullsized":"2023-10-09T06:34:05.173525+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":410,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168076+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:34:05.168076+0000","last_peered":"2023-10-09T06:34:05.168076+0000","last_clean":"2023-10-09T06:34:05.168076+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:34:05.168076+0000","last_undegraded":"2023-10-09T06:34:05.168076+0000","last_fullsized":"2023-10-09T06:34:05.168076+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":375,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177659+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:34:05.177659+0000","last_peered":"2023-10-09T06:34:05.177659+0000","last_clean":"2023-10-09T06:34:05.177659+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:34:05.177659+0000","last_undegraded":"2023-10-09T06:34:05.177659+0000","last_fullsized":"2023-10-09T06:34:05.177659+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":2194309144,"num_objects":33484,"num_object_clones":0,"num_object_copies":100452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3820,"num_objects_misplaced":868,"num_objects_unfound":0,"num_objects_dirty":33484,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160758,"num_write_kb":6175969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8695,"num_bytes_recovered":561577984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7655927808,"data_stored":7655915592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":196827,"ondisk_log_size":196827,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234444,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2106996,"kb_used_data":1194476,"kb_used_omap":0,"kb_used_meta":912512,"kb_avail":91638156,"statfs":{"total":95995035648,"available":93837471744,"internally_reserved":0,"allocated":1223143424,"data_stored":1222105697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934412288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":5.7720000000000002,"15min":6.1849999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":25.225000000000001,"5min":196.28899999999999,"15min":219.41900000000001},"last":2.0529999999999999},{"interface":"front","average":{"1min":3.097,"5min":5.9139999999999997,"15min":6.3200000000000003},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":25.248000000000001,"5min":196.333,"15min":219.399},"last":2.0649999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":4.516,"15min":4.5620000000000003},"min":{"1min":0.33000000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":8.0540000000000003,"5min":107.87,"15min":215.61699999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.8999999999999999,"5min":3.9540000000000002,"15min":4.1829999999999998},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.23300000000000001},"max":{"1min":7.8220000000000001,"5min":107.904,"15min":216.22200000000001},"last":0.60999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":3.7879999999999998,"15min":6.718},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27700000000000002},"max":{"1min":5.9809999999999999,"5min":198.94200000000001,"15min":216.34299999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.895,"5min":3.948,"15min":6.8929999999999998},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":5.8239999999999998,"5min":198.92099999999999,"15min":216.25200000000001},"last":1.9239999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":9.1150000000000002,"5min":6.282,"15min":4.5720000000000001},"min":{"1min":0.437,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":97.441999999999993,"5min":187.34200000000001,"15min":215.922},"last":0.65000000000000002},{"interface":"front","average":{"1min":9.0139999999999993,"5min":6.375,"15min":4.641},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":97.162000000000006,"5min":187.08199999999999,"15min":215.94900000000001},"last":0.72599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.2310000000000001,"15min":2.504},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":1.821,"5min":9.6880000000000006,"15min":218.446},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.111,"15min":2.3599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":2.0619999999999998,"5min":9.7699999999999996,"15min":216.172},"last":0.91900000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.226,"15min":2.472},"min":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":3.1920000000000002,"5min":8.8170000000000002,"15min":216.37200000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.373,"15min":2.4009999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.28499999999999998},"max":{"1min":3.4860000000000002,"5min":7.8739999999999997,"15min":218.15899999999999},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.464,"15min":2.3039999999999998},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":1.756,"5min":15.162000000000001,"15min":218.19999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.0740000000000001,"15min":2.1030000000000002},"min":{"1min":0.36099999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":2.1200000000000001,"5min":8.2919999999999998,"15min":218.05500000000001},"last":0.69199999999999995}]}]},{"osd":6,"up_from":46,"seq":197568496080,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2005080,"kb_used_data":1106064,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91740072,"statfs":{"total":95995035648,"available":93941833728,"internally_reserved":0,"allocated":1132609536,"data_stored":1131599179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":3.2309999999999999,"15min":3.5110000000000001},"min":{"1min":0.24299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":42.32,"5min":75.061999999999998,"15min":110.926},"last":0.372},{"interface":"front","average":{"1min":3.8159999999999998,"5min":3.6080000000000001,"15min":3.7879999999999998},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":42.307000000000002,"5min":75.021000000000001,"15min":110.911},"last":0.29999999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":4.4669999999999996,"15min":5.242},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":41.912999999999997,"5min":87.147999999999996,"15min":292.92200000000003},"last":0.255},{"interface":"front","average":{"1min":4.0049999999999999,"5min":4.5730000000000004,"15min":5.1159999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":41.927,"5min":87.191000000000003,"15min":292.904},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.8220000000000001,"15min":3.1840000000000002},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":47.484000000000002,"5min":74.748999999999995,"15min":75.191999999999993},"last":0.42599999999999999},{"interface":"front","average":{"1min":4.5880000000000001,"5min":3.4820000000000002,"15min":2.9079999999999999},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":47.494999999999997,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.48499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:33:17 2023","interfaces":[{"interface":"back","average":{"1min":6.915,"5min":5.2699999999999996,"15min":3.702},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":62.223999999999997,"5min":91.174000000000007,"15min":110.358},"last":0.436},{"interface":"front","average":{"1min":6.0570000000000004,"5min":4.9889999999999999,"15min":3.7109999999999999},"min":{"1min":0.36299999999999999,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":61.637,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.74099999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.365,"15min":1.278},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.2690000000000001,"5min":12.666,"15min":17.643999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.1000000000000001,"15min":1.1259999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.223},"max":{"1min":6.7859999999999996,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.2769999999999999,"15min":1.377},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29199999999999998},"max":{"1min":5.7539999999999996,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5620000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":6.8140000000000001,"5min":16.536999999999999,"15min":24.401},"last":0.52100000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.3180000000000001,"15min":2.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":7.4029999999999996,"5min":19.295999999999999,"15min":122.937},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.7270000000000001,"15min":2.6800000000000002},"min":{"1min":0.378,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.4729999999999999,"5min":19.584,"15min":122.92100000000001},"last":0.497}]}]},{"osd":1,"up_from":110,"seq":472446402817,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505376,"kb_used_data":765592,"kb_used_omap":4,"kb_used_meta":739707,"kb_avail":92239776,"statfs":{"total":95995035648,"available":94453530624,"internally_reserved":0,"allocated":783966208,"data_stored":782071857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":757460878},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":5.7990000000000004,"15min":6.1040000000000001},"min":{"1min":0.221,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":26.59,"5min":143.465,"15min":269.79199999999997},"last":0.25700000000000001},{"interface":"front","average":{"1min":3.8290000000000002,"5min":6.0309999999999997,"15min":6.5880000000000001},"min":{"1min":0.22700000000000001,"5min":0.21199999999999999,"15min":0.17899999999999999},"max":{"1min":28.992999999999999,"5min":143.66399999999999,"15min":271.20999999999998},"last":5.4450000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":3.9830000000000001,"5min":3.7130000000000001,"15min":3.6230000000000002},"min":{"1min":0.186,"5min":0.184,"15min":0.184},"max":{"1min":22.614999999999998,"5min":138.452,"15min":139.75399999999999},"last":1.742},{"interface":"front","average":{"1min":3.6629999999999998,"5min":4.202,"15min":4.3220000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":24.372,"5min":138.47499999999999,"15min":140.29900000000001},"last":5.4050000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:33:25 2023","interfaces":[{"interface":"back","average":{"1min":5.4560000000000004,"5min":3.2639999999999998,"15min":3.8450000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":27.064,"5min":27.064,"15min":153.18199999999999},"last":5.8179999999999996},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.1299999999999999,"15min":2.7250000000000001},"min":{"1min":0.23899999999999999,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":26.626999999999999,"5min":26.626999999999999,"15min":152.52000000000001},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.5630000000000002,"15min":3.7360000000000002},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.9930000000000003,"5min":38.835999999999999,"15min":199.43799999999999},"last":5.7619999999999996},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.1840000000000002,"15min":3.3879999999999999},"min":{"1min":0.43099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.5,"5min":38.819000000000003,"15min":199.32900000000001},"last":1.8260000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.536,"15min":2.2330000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.406000000000001,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.438},{"interface":"front","average":{"1min":1.5189999999999999,"5min":2.766,"15min":2.6869999999999998},"min":{"1min":0.48799999999999999,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":4.6150000000000002,"5min":52.664999999999999,"15min":77.388000000000005},"last":1.79}]},{"osd":6,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":3.9660000000000002,"15min":3.1949999999999998},"min":{"1min":0.46800000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4199999999999999,"5min":107.27200000000001,"15min":107.27200000000001},"last":1.849},{"interface":"front","average":{"1min":2.3599999999999999,"5min":4.4320000000000004,"15min":3.4399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.9329999999999998,"5min":107.252,"15min":107.252},"last":5.7210000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":4.1920000000000002,"15min":3.6099999999999999},"min":{"1min":0.47899999999999998,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":8.0229999999999997,"5min":104.07299999999999,"15min":104.07299999999999},"last":5.79},{"interface":"front","average":{"1min":1.159,"5min":3.3900000000000001,"15min":2.9169999999999998},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3609999999999998,"5min":104.056,"15min":104.056},"last":0.40799999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024250,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1851228,"kb_used_data":909588,"kb_used_omap":5,"kb_used_meta":941626,"kb_avail":91893924,"statfs":{"total":95995035648,"available":94099378176,"internally_reserved":0,"allocated":931418112,"data_stored":929238838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":964225658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.726,"15min":3.4729999999999999},"min":{"1min":0.19700000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":7.4909999999999997,"5min":14.977,"15min":191.446},"last":6.0540000000000003},{"interface":"front","average":{"1min":2.6429999999999998,"5min":2.2080000000000002,"15min":3.7410000000000001},"min":{"1min":0.22900000000000001,"5min":0.187,"15min":0.187},"max":{"1min":9.2599999999999998,"5min":15.129,"15min":191.393},"last":5.899}]},{"osd":2,"last update":"Mon Oct 9 06:33:52 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.5009999999999999,"15min":2.5350000000000001},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":9.1859999999999999,"5min":16.936,"15min":57.162999999999997},"last":3.407},{"interface":"front","average":{"1min":2.8889999999999998,"5min":3.3210000000000002,"15min":2.9889999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.20999999999999999},"max":{"1min":8.9450000000000003,"5min":17.962,"15min":57.585000000000001},"last":0.874}]},{"osd":3,"last update":"Mon Oct 9 06:34:11 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":6.5800000000000001,"15min":5.9820000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":15.413,"5min":187.172,"15min":208.11199999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":3.9609999999999999,"5min":6.2960000000000003,"15min":5.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":15.266,"5min":194.851,"15min":208.25899999999999},"last":3.3879999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.78,"15min":1.593},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":5.6059999999999999,"5min":9.7159999999999993,"15min":11.162000000000001},"last":3.2050000000000001},{"interface":"front","average":{"1min":2.2749999999999999,"5min":2.2509999999999999,"15min":1.929},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":8.2880000000000003,"5min":18.081,"15min":18.081},"last":3.3159999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.883,"15min":1.907},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":6.2809999999999997,"5min":9.7720000000000002,"15min":62.984999999999999},"last":3.2450000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.492,"15min":2.3450000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":6.1529999999999996,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.89800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":3.4790000000000001,"15min":2.9540000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":6.2069999999999999,"5min":85.525999999999996,"15min":120.572},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.8370000000000002,"15min":2.4460000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.30299999999999999},"max":{"1min":5.6440000000000001,"5min":85.734999999999999,"15min":120.64400000000001},"last":3.2799999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.5299999999999998,"15min":2.6579999999999999},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.32300000000000001},"max":{"1min":8.3249999999999993,"5min":18.041,"15min":122.018},"last":3.3620000000000001},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.9119999999999999,"15min":2.972},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":7.7089999999999996,"5min":26.416,"15min":122.32299999999999},"last":0.92300000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814119,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2086364,"kb_used_data":1388692,"kb_used_omap":7,"kb_used_meta":697592,"kb_avail":91658788,"statfs":{"total":95995035648,"available":93858598912,"internally_reserved":0,"allocated":1422020608,"data_stored":1420186046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":714334289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7160000000000002,"5min":5.1029999999999998,"15min":4.9429999999999996},"min":{"1min":0.309,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":72.125,"5min":82.751000000000005,"15min":162.21100000000001},"last":4.7610000000000001},{"interface":"front","average":{"1min":5.9829999999999997,"5min":4.8460000000000001,"15min":4.4989999999999997},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":75.456000000000003,"5min":81.960999999999999,"15min":168.49700000000001},"last":6.2960000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":4.0499999999999998,"15min":4.1390000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20599999999999999},"max":{"1min":16.152999999999999,"5min":74.744,"15min":205.148},"last":3.2770000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":4.5,"15min":4.335},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":8.9819999999999993,"5min":75.759,"15min":205.267},"last":3.4550000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2880000000000003,"5min":3.968,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":75.498999999999995,"5min":77.692999999999998,"15min":91.644000000000005},"last":8.5980000000000008},{"interface":"front","average":{"1min":6.2220000000000004,"5min":4.3140000000000001,"15min":3.9409999999999998},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":69.784000000000006,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.42999999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":2.1699999999999999,"15min":2.2669999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":6.2839999999999998,"5min":44.314,"15min":90.938999999999993},"last":3.423},{"interface":"front","average":{"1min":2.3690000000000002,"5min":2.492,"15min":2.5880000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":8.6099999999999994,"5min":43.896000000000001,"15min":90.950000000000003},"last":3.3090000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":2.0339999999999998,"15min":1.7070000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.3140000000000001,"5min":13.102,"15min":17.800000000000001},"last":3.3769999999999998},{"interface":"front","average":{"1min":2.4670000000000001,"5min":2.2389999999999999,"15min":2.0089999999999999},"min":{"1min":0.39100000000000001,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":8.6829999999999998,"5min":11.285,"15min":11.285},"last":3.2519999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":4.2460000000000004,"15min":3.2999999999999998},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":8.6600000000000001,"5min":123.47799999999999,"15min":138.291},"last":3.3999999999999999},{"interface":"front","average":{"1min":3.677,"5min":4.133,"15min":3.093},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":16.012,"5min":122.517,"15min":138.01599999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5150000000000001,"5min":2.875,"15min":2.3399999999999999},"min":{"1min":0.36499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":16.064,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.46,"5min":2.5910000000000002,"15min":2.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":6.3460000000000001,"5min":47.206000000000003,"15min":55.731000000000002},"last":3.3330000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127317,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849676,"kb_used_data":1181124,"kb_used_omap":2,"kb_used_meta":668541,"kb_avail":91895476,"statfs":{"total":95995035648,"available":94100967424,"internally_reserved":0,"allocated":1209470976,"data_stored":1207711982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":684586037},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":7.5149999999999997,"15min":5.7229999999999999},"min":{"1min":0.22500000000000001,"5min":0.19,"15min":0.19},"max":{"1min":8.577,"5min":161.821,"15min":187.852},"last":3.2789999999999999},{"interface":"front","average":{"1min":1.9299999999999999,"5min":6.9020000000000001,"15min":5.7880000000000003},"min":{"1min":0.24299999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":8.3260000000000005,"5min":163.52000000000001,"15min":187.923},"last":0.58999999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":4.931,"15min":4.3029999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.9619999999999997,"5min":155.23099999999999,"15min":187.89099999999999},"last":4.0720000000000001},{"interface":"front","average":{"1min":3.8090000000000002,"5min":6.2720000000000002,"15min":4.0469999999999997},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":14.507999999999999,"5min":155.24799999999999,"15min":155.24799999999999},"last":3.6890000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":5.0880000000000001,"15min":4.0890000000000004},"min":{"1min":0.254,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.2750000000000004,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.3170000000000002,"5min":5.7389999999999999,"15min":4.048},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":11.744999999999999,"5min":168.16900000000001,"15min":187.79499999999999},"last":3.3740000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":2.9399999999999999,"15min":1.9610000000000001},"min":{"1min":0.35699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.4020000000000001,"5min":48.249000000000002,"15min":48.249000000000002},"last":3.2069999999999999},{"interface":"front","average":{"1min":2.117,"5min":2.867,"15min":2.5369999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.9689999999999994,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.371}]},{"osd":5,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":2.7050000000000001,"15min":2.6200000000000001},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.29899999999999999},"max":{"1min":9.0259999999999998,"5min":48.162999999999997,"15min":187.833},"last":1.8240000000000001},{"interface":"front","average":{"1min":1.778,"5min":2.2679999999999998,"15min":2.3599999999999999},"min":{"1min":0.35099999999999998,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":6.7649999999999997,"5min":48.106000000000002,"15min":187.952},"last":1.7869999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":4.3639999999999999,"15min":3.2309999999999999},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.33700000000000002},"max":{"1min":9.0579999999999998,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.433},{"interface":"front","average":{"1min":1.778,"5min":4.0309999999999997,"15min":2.9119999999999999},"min":{"1min":0.375,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":6.7249999999999996,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":4.5819999999999999,"15min":3.27},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":4.4329999999999998,"5min":121.613,"15min":192.20400000000001},"last":3.2530000000000001},{"interface":"front","average":{"1min":2.173,"5min":4.8730000000000002,"15min":3.3479999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":9.0869999999999997,"5min":121.559,"15min":187.86500000000001},"last":0.47099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019353,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597280,"kb_used_data":702296,"kb_used_omap":0,"kb_used_meta":894976,"kb_avail":92147872,"statfs":{"total":95995035648,"available":94359420928,"internally_reserved":0,"allocated":719151104,"data_stored":718132932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916455424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":11.006,"5min":11.769,"15min":5.6020000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":118.98999999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":11.007,"5min":11.597,"15min":5.8220000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":118.956,"5min":287.57100000000003,"15min":287.57100000000003},"last":0.39300000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4100000000000001,"5min":4.4550000000000001,"15min":3.1779999999999999},"min":{"1min":0.30099999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":25.879000000000001,"5min":128.24299999999999,"15min":128.24299999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.2810000000000001,"5min":4.6719999999999997,"15min":3.3700000000000001},"min":{"1min":0.41799999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":25.617999999999999,"5min":128.21600000000001,"15min":128.21600000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":3.411,"15min":3.0619999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":7.7039999999999997,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.498},{"interface":"front","average":{"1min":1.833,"5min":3.1469999999999998,"15min":3.1179999999999999},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":6.8070000000000004,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":9.234,"5min":10.218999999999999,"15min":5.6870000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":144.399,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":9.2699999999999996,"5min":9.9489999999999998,"15min":5.4020000000000001},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28699999999999998},"max":{"1min":143.84100000000001,"5min":161.934,"15min":161.934},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.90500000000000003,"15min":1.008},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.5070000000000001,"5min":6.25,"15min":10.673},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.1120000000000001,"15min":1.0429999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":3.282,"5min":7.2249999999999996,"15min":12.552},"last":0.439}]},{"osd":6,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.4620000000000002,"15min":1.2929999999999999},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":2.5179999999999998,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.641,"15min":1.6419999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":2.9340000000000002,"5min":163.381,"15min":163.381},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:33:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":1.8009999999999999,"15min":1.7},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":0.79200000000000004,"5min":70.247,"15min":70.247},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.845,"15min":1.6379999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":5.1440000000000001,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.47499999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757718,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1231896,"kb_used_data":342480,"kb_used_omap":0,"kb_used_meta":889408,"kb_avail":92513256,"statfs":{"total":95995035648,"available":94733574144,"internally_reserved":0,"allocated":350699520,"data_stored":349685792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910753792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":6.1609999999999996,"15min":4.9950000000000001},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.216},"max":{"1min":8.6940000000000008,"5min":294.89699999999999,"15min":294.89699999999999},"last":2.8050000000000002},{"interface":"front","average":{"1min":1.47,"5min":5.9009999999999998,"15min":4.8280000000000003},"min":{"1min":0.34999999999999998,"5min":0.24399999999999999,"15min":0.22},"max":{"1min":5.6539999999999999,"5min":294.30399999999997,"15min":294.30399999999997},"last":2.794}]},{"osd":1,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":3.2090000000000001,"15min":2.8700000000000001},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4160000000000004,"5min":87.795000000000002,"15min":126.71899999999999},"last":4.806},{"interface":"front","average":{"1min":1.357,"5min":3.4300000000000002,"15min":3.02},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":3.2320000000000002,"5min":87.856999999999999,"15min":126.931},"last":4.9349999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.625,"15min":2.5019999999999998},"min":{"1min":0.311,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.6589999999999998,"5min":64.150000000000006,"15min":81.260000000000005},"last":3.8380000000000001},{"interface":"front","average":{"1min":1.893,"5min":2.4489999999999998,"15min":2.4049999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":8.1470000000000002,"5min":64.507000000000005,"15min":81.245000000000005},"last":1.4350000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":5.6289999999999996,"15min":4.6790000000000003},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":38.582000000000001,"5min":144.56399999999999,"15min":144.56399999999999},"last":3.1920000000000002},{"interface":"front","average":{"1min":3.9409999999999998,"5min":5.3700000000000001,"15min":4.5099999999999998},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":38.447000000000003,"5min":144.62799999999999,"15min":144.62799999999999},"last":2.496}]},{"osd":4,"last update":"Mon Oct 9 06:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":1.0600000000000001,"15min":1.1839999999999999},"min":{"1min":0.26100000000000001,"5min":0.25,"15min":0.25},"max":{"1min":1.7150000000000001,"5min":10.529999999999999,"15min":32.850999999999999},"last":1.9990000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.871,"15min":1.0509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":0.93500000000000005,"5min":6.2610000000000001,"15min":32.82},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.7250000000000001,"15min":1.6200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7839999999999998,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.9810000000000001,"15min":1.7450000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0630000000000002,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.375}]},{"osd":7,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.454,"15min":1.397},"min":{"1min":0.374,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.96,"5min":10.461,"15min":30.658999999999999},"last":2.141},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.218,"15min":1.4099999999999999},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":3.4289999999999998,"5min":6.2320000000000002,"15min":30.683},"last":0.436}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":920059904,"data_stored":920059904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":765661184,"data_stored":765657112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1393627136,"data_stored":1393623064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1188888576,"data_stored":1188888576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":713359360,"data_stored":713359360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":342360064,"data_stored":342360064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1117257728,"data_stored":1117257728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1214713856,"data_stored":1214709784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:22.104 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:22.105 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:21 smithi150 ceph-mon[194178]: 12.19 scrub starts 2023-10-09T06:34:22.105 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:22.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:22.112 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:34:22.212 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:22.241 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:22.305 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:22.309 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:22] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:34:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:22.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:22.417 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:22.514 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:22.522 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:22.598 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:22.647 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:22.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:22.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:22.763 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:22.793 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:22.852 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:22.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[220921]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 3820/97644 objects degraded (3.912%); 611/97644 objects misplaced (0.626%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3820/97644 objects degraded (3.912%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[212750]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 3820/97644 objects degraded (3.912%); 611/97644 objects misplaced (0.626%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:22.919 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3820/97644 objects degraded (3.912%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:22.958 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:23.051 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:23.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:22 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:23.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:22 smithi150 ceph-mon[194178]: pgmap v1273: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 3820/97644 objects degraded (3.912%); 611/97644 objects misplaced (0.626%); 4.3 MiB/s, 69 objects/s recovering 2023-10-09T06:34:23.122 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3820/97644 objects degraded (3.912%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:23.385 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:23.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:23.502 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:23.530 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:23.576 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:23.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:23.681 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:23.692 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:34:23.709 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:23.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:34:23.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:34:24.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:34:24.024 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:24.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:24.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:24.130 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:24.345 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:24.658 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:34:24.658 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:34:24.658 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:34:24.658 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:34:25.068 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:25.069 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: pgmap v1274: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:25.069 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:25.069 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.069 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:34:25.069 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: pgmap v1274: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T06:34:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.071 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:24 smithi107 ceph-mon[220921]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:25.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:25.150 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: pgmap v1274: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3171489043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: osdmap e416: 8 total, 8 up, 8 in 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:24 smithi150 ceph-mon[194178]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.172 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:25.247 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:25.404 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:25.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:25.515 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:25.563 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:34:25.590 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:25.769 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:25.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:25.860 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:25.891 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:25.894 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:25.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:25 smithi150 ceph-mon[194178]: failed to encode map e416 with expected crc 2023-10-09T06:34:25.935 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:25.941 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:25.979 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:25.997 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:26.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:26.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:25 smithi107 ceph-mon[212750]: failed to encode map e416 with expected crc 2023-10-09T06:34:26.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:25 smithi107 ceph-mon[220921]: failed to encode map e416 with expected crc 2023-10-09T06:34:26.019 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:26.057 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:26.060 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:26.106 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:26.116 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:26.150 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:26.152 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:26.164 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.164 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.164 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.164 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.165 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.165 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.165 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.165 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:34:26.242 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:26.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:26.350 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:26.426 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:26.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:26 smithi107 ceph-mon[220921]: pgmap v1276: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:26.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/428063008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:26 smithi150 ceph-mon[194178]: pgmap v1276: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/428063008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:26 smithi107 ceph-mon[212750]: pgmap v1276: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 3600/89967 objects degraded (4.001%); 611/89967 objects misplaced (0.679%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T06:34:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/428063008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:27.194 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:34:27.455 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:27.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:27.565 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:27.671 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:28.010 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:28.011 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:28.028 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3600/89967 objects degraded (4.001%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:28.028 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3600/89967 objects degraded (4.001%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:28.032 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3600/89967 objects degraded (4.001%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:28.111 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:28.222 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:28.265 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:28.331 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:28.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:28.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:28.468 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:28.483 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:28.661 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:28.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:28.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:28.745 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:28 smithi107 ceph-mon[220921]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3600/87234 objects degraded (4.127%); 359/87234 objects misplaced (0.412%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:34:28.775 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:28.807 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:28.925 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:28.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:28.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:28.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:34:28.998 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:28 smithi150 ceph-mon[194178]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3600/87234 objects degraded (4.127%); 359/87234 objects misplaced (0.412%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:34:29.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:29.080 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:29.138 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:29.148 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:28 smithi107 ceph-mon[212750]: pgmap v1277: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 3600/87234 objects degraded (4.127%); 359/87234 objects misplaced (0.412%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:34:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:29.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:29.231 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:34:29.248 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:34:29.250 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:29.287 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:29.407 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:29.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:29.445 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:29.515 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:29.581 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:29.760 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:29 smithi107 ceph-mon[212750]: from='client.56428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:29.761 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:29 smithi107 ceph-mon[220921]: from='client.56428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:29.992 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:29 smithi150 ceph-mon[194178]: from='client.56428 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:30.110 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:34:30.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:30 smithi107 ceph-mon[212750]: pgmap v1278: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3428/79950 objects degraded (4.288%); 359/79950 objects misplaced (0.449%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:30 smithi107 ceph-mon[220921]: pgmap v1278: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3428/79950 objects degraded (4.288%); 359/79950 objects misplaced (0.449%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:30 smithi150 ceph-mon[194178]: pgmap v1278: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 3428/79950 objects degraded (4.288%); 359/79950 objects misplaced (0.449%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T06:34:31.415 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1277,"stamp":"2023-10-09T06:34:27.396272+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1904449192,"num_objects":29078,"num_object_clones":0,"num_object_copies":87234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3600,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":29078,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165249,"num_write_kb":6175998,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9370,"num_bytes_recovered":602548988,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203750,"ondisk_log_size":203750,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":395,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13313856,"kb_used_data":6674496,"kb_used_omap":20,"kb_used_meta":6639275,"kb_avail":736647360,"statfs":{"total":767960285184,"available":754326896640,"internally_reserved":0,"allocated":6834683904,"data_stored":6822929261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6798618254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,3,4,12,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-461701120,"num_objects":-7045,"num_object_clones":0,"num_object_copies":-21135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-466,"num_objects_misplaced":-509,"num_objects_unfound":0,"num_objects_dirty":-7045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7045,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":742,"num_bytes_recovered":48627712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003245"},"pg_stats":[{"pgid":"12.15","version":"415'8187","reported_seq":8270,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.403618+0000","last_change":"2023-10-09T06:34:17.052386+0000","last_active":"2023-10-09T06:34:22.403618+0000","last_peered":"2023-10-09T06:34:22.403618+0000","last_clean":"2023-10-09T06:34:22.403618+0000","last_became_active":"2023-10-09T06:32:33.082121+0000","last_became_peered":"2023-10-09T06:32:33.082121+0000","last_unstale":"2023-10-09T06:34:22.403618+0000","last_undegraded":"2023-10-09T06:34:22.403618+0000","last_fullsized":"2023-10-09T06:34:22.403618+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60981248,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5293,"num_write_kb":199136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":773,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:19.873202+0000","last_change":"2023-10-09T06:34:19.873202+0000","last_active":"2023-10-09T06:34:19.873202+0000","last_peered":"2023-10-09T06:34:19.873202+0000","last_clean":"2023-10-09T06:34:19.873202+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:34:19.873202+0000","last_undegraded":"2023-10-09T06:34:19.873202+0000","last_fullsized":"2023-10-09T06:34:19.873202+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":394,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178152+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:34:05.178152+0000","last_peered":"2023-10-09T06:34:05.178152+0000","last_clean":"2023-10-09T06:34:05.178152+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:34:05.178152+0000","last_undegraded":"2023-10-09T06:34:05.178152+0000","last_fullsized":"2023-10-09T06:34:05.178152+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167097+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:34:05.167097+0000","last_peered":"2023-10-09T06:34:05.167097+0000","last_clean":"2023-10-09T06:34:05.167097+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:34:05.167097+0000","last_undegraded":"2023-10-09T06:34:05.167097+0000","last_fullsized":"2023-10-09T06:34:05.167097+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"415'8143","reported_seq":8215,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.534053+0000","last_change":"2023-10-09T06:32:33.003181+0000","last_active":"2023-10-09T06:34:22.534053+0000","last_peered":"2023-10-09T06:34:22.534053+0000","last_clean":"2023-10-09T06:34:22.534053+0000","last_became_active":"2023-10-09T06:32:32.998596+0000","last_became_peered":"2023-10-09T06:32:32.998596+0000","last_unstale":"2023-10-09T06:34:22.534053+0000","last_undegraded":"2023-10-09T06:34:22.534053+0000","last_fullsized":"2023-10-09T06:34:22.534053+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8143,"log_dups_size":0,"ondisk_log_size":8143,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62717952,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5259,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":733,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.788684+0000","last_change":"2023-10-09T06:34:13.788684+0000","last_active":"2023-10-09T06:34:13.788684+0000","last_peered":"2023-10-09T06:34:13.788684+0000","last_clean":"2023-10-09T06:34:13.788684+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:34:13.788684+0000","last_undegraded":"2023-10-09T06:34:13.788684+0000","last_fullsized":"2023-10-09T06:34:13.788684+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":408,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172574+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:34:05.172574+0000","last_peered":"2023-10-09T06:34:05.172574+0000","last_clean":"2023-10-09T06:34:05.172574+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:34:05.172574+0000","last_undegraded":"2023-10-09T06:34:05.172574+0000","last_fullsized":"2023-10-09T06:34:05.172574+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":414,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.904810+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:34:15.904810+0000","last_peered":"2023-10-09T06:34:15.904810+0000","last_clean":"2023-10-09T06:34:15.904810+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:34:15.904810+0000","last_undegraded":"2023-10-09T06:34:15.904810+0000","last_fullsized":"2023-10-09T06:34:15.904810+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"415'8105","reported_seq":8178,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.476322+0000","last_change":"2023-10-09T06:32:33.092190+0000","last_active":"2023-10-09T06:34:22.476322+0000","last_peered":"2023-10-09T06:34:22.476322+0000","last_clean":"2023-10-09T06:34:22.476322+0000","last_became_active":"2023-10-09T06:32:33.084295+0000","last_became_peered":"2023-10-09T06:32:33.084295+0000","last_unstale":"2023-10-09T06:34:22.476322+0000","last_undegraded":"2023-10-09T06:34:22.476322+0000","last_fullsized":"2023-10-09T06:34:22.476322+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8105,"log_dups_size":0,"ondisk_log_size":8105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53510144,"num_objects":817,"num_object_clones":0,"num_object_copies":2451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5261,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":662,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177686+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:34:05.177686+0000","last_peered":"2023-10-09T06:34:05.177686+0000","last_clean":"2023-10-09T06:34:05.177686+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:34:05.177686+0000","last_undegraded":"2023-10-09T06:34:05.177686+0000","last_fullsized":"2023-10-09T06:34:05.177686+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166507+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:34:05.166507+0000","last_peered":"2023-10-09T06:34:05.166507+0000","last_clean":"2023-10-09T06:34:05.166507+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:34:05.166507+0000","last_undegraded":"2023-10-09T06:34:05.166507+0000","last_fullsized":"2023-10-09T06:34:05.166507+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":404,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176376+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:34:05.176376+0000","last_peered":"2023-10-09T06:34:05.176376+0000","last_clean":"2023-10-09T06:34:05.176376+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:34:05.176376+0000","last_undegraded":"2023-10-09T06:34:05.176376+0000","last_fullsized":"2023-10-09T06:34:05.176376+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"415'8069","reported_seq":8141,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:23.164921+0000","last_change":"2023-10-09T06:32:33.080997+0000","last_active":"2023-10-09T06:34:23.164921+0000","last_peered":"2023-10-09T06:34:23.164921+0000","last_clean":"2023-10-09T06:34:23.164921+0000","last_became_active":"2023-10-09T06:32:33.077785+0000","last_became_peered":"2023-10-09T06:32:33.077785+0000","last_unstale":"2023-10-09T06:34:23.164921+0000","last_undegraded":"2023-10-09T06:34:23.164921+0000","last_fullsized":"2023-10-09T06:34:23.164921+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8069,"log_dups_size":0,"ondisk_log_size":8069,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64356352,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5204,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":523,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173408+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:34:05.173408+0000","last_peered":"2023-10-09T06:34:05.173408+0000","last_clean":"2023-10-09T06:34:05.173408+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:34:05.173408+0000","last_undegraded":"2023-10-09T06:34:05.173408+0000","last_fullsized":"2023-10-09T06:34:05.173408+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":89,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594322+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:34:13.594322+0000","last_peered":"2023-10-09T06:34:13.594322+0000","last_clean":"2023-10-09T06:34:13.594322+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:34:13.594322+0000","last_undegraded":"2023-10-09T06:34:13.594322+0000","last_fullsized":"2023-10-09T06:34:13.594322+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166959+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:34:05.166959+0000","last_peered":"2023-10-09T06:34:05.166959+0000","last_clean":"2023-10-09T06:34:05.166959+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:34:05.166959+0000","last_undegraded":"2023-10-09T06:34:05.166959+0000","last_fullsized":"2023-10-09T06:34:05.166959+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"415'8105","reported_seq":8185,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.387517+0000","last_change":"2023-10-09T06:34:21.023111+0000","last_active":"2023-10-09T06:34:22.387517+0000","last_peered":"2023-10-09T06:34:22.387517+0000","last_clean":"2023-10-09T06:34:22.387517+0000","last_became_active":"2023-10-09T06:32:33.074690+0000","last_became_peered":"2023-10-09T06:32:33.074690+0000","last_unstale":"2023-10-09T06:34:22.387517+0000","last_undegraded":"2023-10-09T06:34:22.387517+0000","last_fullsized":"2023-10-09T06:34:22.387517+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8105,"log_dups_size":0,"ondisk_log_size":8105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57966592,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5237,"num_write_kb":195872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":560,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172715+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:34:05.172715+0000","last_peered":"2023-10-09T06:34:05.172715+0000","last_clean":"2023-10-09T06:34:05.172715+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:34:05.172715+0000","last_undegraded":"2023-10-09T06:34:05.172715+0000","last_fullsized":"2023-10-09T06:34:05.172715+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179586+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:34:05.179586+0000","last_peered":"2023-10-09T06:34:05.179586+0000","last_clean":"2023-10-09T06:34:05.179586+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:34:05.179586+0000","last_undegraded":"2023-10-09T06:34:05.179586+0000","last_fullsized":"2023-10-09T06:34:05.179586+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168978+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:34:05.168978+0000","last_peered":"2023-10-09T06:34:05.168978+0000","last_clean":"2023-10-09T06:34:05.168978+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:34:05.168978+0000","last_undegraded":"2023-10-09T06:34:05.168978+0000","last_fullsized":"2023-10-09T06:34:05.168978+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":610,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:16.838424+0000","last_change":"2023-10-09T06:34:16.838424+0000","last_active":"2023-10-09T06:34:16.838424+0000","last_peered":"2023-10-09T06:34:16.838424+0000","last_clean":"2023-10-09T06:34:16.838424+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:34:16.838424+0000","last_undegraded":"2023-10-09T06:34:16.838424+0000","last_fullsized":"2023-10-09T06:34:16.838424+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":296,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173713+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:34:05.173713+0000","last_peered":"2023-10-09T06:34:05.173713+0000","last_clean":"2023-10-09T06:34:05.173713+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:34:05.173713+0000","last_undegraded":"2023-10-09T06:34:05.173713+0000","last_fullsized":"2023-10-09T06:34:05.173713+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":350,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173080+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:34:05.173080+0000","last_peered":"2023-10-09T06:34:05.173080+0000","last_clean":"2023-10-09T06:34:05.173080+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:34:05.173080+0000","last_undegraded":"2023-10-09T06:34:05.173080+0000","last_fullsized":"2023-10-09T06:34:05.173080+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":521,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173531+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:34:05.173531+0000","last_peered":"2023-10-09T06:34:05.173531+0000","last_clean":"2023-10-09T06:34:05.173531+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:34:05.173531+0000","last_undegraded":"2023-10-09T06:34:05.173531+0000","last_fullsized":"2023-10-09T06:34:05.173531+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":421,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178565+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:34:05.178565+0000","last_peered":"2023-10-09T06:34:05.178565+0000","last_clean":"2023-10-09T06:34:05.178565+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:34:05.178565+0000","last_undegraded":"2023-10-09T06:34:05.178565+0000","last_fullsized":"2023-10-09T06:34:05.178565+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":385,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178213+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:34:05.178213+0000","last_peered":"2023-10-09T06:34:05.178213+0000","last_clean":"2023-10-09T06:34:05.178213+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:34:05.178213+0000","last_undegraded":"2023-10-09T06:34:05.178213+0000","last_fullsized":"2023-10-09T06:34:05.178213+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":687,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:20.932581+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:34:20.932581+0000","last_peered":"2023-10-09T06:34:20.932581+0000","last_clean":"2023-10-09T06:34:20.932581+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:34:20.932581+0000","last_undegraded":"2023-10-09T06:34:20.932581+0000","last_fullsized":"2023-10-09T06:34:20.932581+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":264,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172816+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:34:05.172816+0000","last_peered":"2023-10-09T06:34:05.172816+0000","last_clean":"2023-10-09T06:34:05.172816+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:34:05.172816+0000","last_undegraded":"2023-10-09T06:34:05.172816+0000","last_fullsized":"2023-10-09T06:34:05.172816+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168148+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:34:05.168148+0000","last_peered":"2023-10-09T06:34:05.168148+0000","last_clean":"2023-10-09T06:34:05.168148+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:34:05.168148+0000","last_undegraded":"2023-10-09T06:34:05.168148+0000","last_fullsized":"2023-10-09T06:34:05.168148+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":529,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179174+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:34:05.179174+0000","last_peered":"2023-10-09T06:34:05.179174+0000","last_clean":"2023-10-09T06:34:05.179174+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:34:05.179174+0000","last_undegraded":"2023-10-09T06:34:05.179174+0000","last_fullsized":"2023-10-09T06:34:05.179174+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":412,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.592391+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:34:13.592391+0000","last_peered":"2023-10-09T06:34:13.592391+0000","last_clean":"2023-10-09T06:34:13.592391+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:34:13.592391+0000","last_undegraded":"2023-10-09T06:34:13.592391+0000","last_fullsized":"2023-10-09T06:34:13.592391+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179725+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:34:05.179725+0000","last_peered":"2023-10-09T06:34:05.179725+0000","last_clean":"2023-10-09T06:34:05.179725+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:34:05.179725+0000","last_undegraded":"2023-10-09T06:34:05.179725+0000","last_fullsized":"2023-10-09T06:34:05.179725+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":740,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:12.552748+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:34:12.552748+0000","last_peered":"2023-10-09T06:34:12.552748+0000","last_clean":"2023-10-09T06:34:12.552748+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:34:12.552748+0000","last_undegraded":"2023-10-09T06:34:12.552748+0000","last_fullsized":"2023-10-09T06:34:12.552748+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":258,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173215+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:34:05.173215+0000","last_peered":"2023-10-09T06:34:05.173215+0000","last_clean":"2023-10-09T06:34:05.173215+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:34:05.173215+0000","last_undegraded":"2023-10-09T06:34:05.173215+0000","last_fullsized":"2023-10-09T06:34:05.173215+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594857+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:34:13.594857+0000","last_peered":"2023-10-09T06:34:13.594857+0000","last_clean":"2023-10-09T06:34:13.594857+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:34:13.594857+0000","last_undegraded":"2023-10-09T06:34:13.594857+0000","last_fullsized":"2023-10-09T06:34:13.594857+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":529,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179799+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:34:05.179799+0000","last_peered":"2023-10-09T06:34:05.179799+0000","last_clean":"2023-10-09T06:34:05.179799+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:34:05.179799+0000","last_undegraded":"2023-10-09T06:34:05.179799+0000","last_fullsized":"2023-10-09T06:34:05.179799+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":493,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.175675+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:34:05.175675+0000","last_peered":"2023-10-09T06:34:05.175675+0000","last_clean":"2023-10-09T06:34:05.175675+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:34:05.175675+0000","last_undegraded":"2023-10-09T06:34:05.175675+0000","last_fullsized":"2023-10-09T06:34:05.175675+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":114,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167718+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:34:05.167718+0000","last_peered":"2023-10-09T06:34:05.167718+0000","last_clean":"2023-10-09T06:34:05.167718+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:34:05.167718+0000","last_undegraded":"2023-10-09T06:34:05.167718+0000","last_fullsized":"2023-10-09T06:34:05.167718+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":542,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173461+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:34:05.173461+0000","last_peered":"2023-10-09T06:34:05.173461+0000","last_clean":"2023-10-09T06:34:05.173461+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:34:05.173461+0000","last_undegraded":"2023-10-09T06:34:05.173461+0000","last_fullsized":"2023-10-09T06:34:05.173461+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"415'8225","reported_seq":8357,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.352681+0000","last_change":"2023-10-09T06:34:08.464064+0000","last_active":"2023-10-09T06:34:22.352681+0000","last_peered":"2023-10-09T06:34:22.352681+0000","last_clean":"2023-10-09T06:34:22.352681+0000","last_became_active":"2023-10-09T06:32:33.069345+0000","last_became_peered":"2023-10-09T06:32:33.069345+0000","last_unstale":"2023-10-09T06:34:22.352681+0000","last_undegraded":"2023-10-09T06:34:22.352681+0000","last_fullsized":"2023-10-09T06:34:22.352681+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'7928","last_scrub_stamp":"2023-10-09T06:34:08.463956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:34:08.463956+0000","objects_scrubbed":1240,"log_size":8225,"log_dups_size":0,"ondisk_log_size":8225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:39.228319+0000","scrub_duration":0.275394735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61734912,"num_objects":942,"num_object_clones":0,"num_object_copies":2826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":298,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":300,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173224+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:34:05.173224+0000","last_peered":"2023-10-09T06:34:05.173224+0000","last_clean":"2023-10-09T06:34:05.173224+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:34:05.173224+0000","last_undegraded":"2023-10-09T06:34:05.173224+0000","last_fullsized":"2023-10-09T06:34:05.173224+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179265+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:34:05.179265+0000","last_peered":"2023-10-09T06:34:05.179265+0000","last_clean":"2023-10-09T06:34:05.179265+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:34:05.179265+0000","last_undegraded":"2023-10-09T06:34:05.179265+0000","last_fullsized":"2023-10-09T06:34:05.179265+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":799,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254857+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:34:08.254857+0000","last_peered":"2023-10-09T06:34:08.254857+0000","last_clean":"2023-10-09T06:34:08.254857+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:34:08.254857+0000","last_undegraded":"2023-10-09T06:34:08.254857+0000","last_fullsized":"2023-10-09T06:34:08.254857+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"415'8265","reported_seq":8347,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.413819+0000","last_change":"2023-10-09T06:34:00.106009+0000","last_active":"2023-10-09T06:34:22.413819+0000","last_peered":"2023-10-09T06:34:22.413819+0000","last_clean":"2023-10-09T06:34:22.413819+0000","last_became_active":"2023-10-09T06:32:33.000165+0000","last_became_peered":"2023-10-09T06:32:33.000165+0000","last_unstale":"2023-10-09T06:34:22.413819+0000","last_undegraded":"2023-10-09T06:34:22.413819+0000","last_fullsized":"2023-10-09T06:34:22.413819+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8265,"log_dups_size":0,"ondisk_log_size":8265,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5357,"num_write_kb":201024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":262,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173055+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:34:05.173055+0000","last_peered":"2023-10-09T06:34:05.173055+0000","last_clean":"2023-10-09T06:34:05.173055+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:34:05.173055+0000","last_undegraded":"2023-10-09T06:34:05.173055+0000","last_fullsized":"2023-10-09T06:34:05.173055+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":412,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177869+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:34:05.177869+0000","last_peered":"2023-10-09T06:34:05.177869+0000","last_clean":"2023-10-09T06:34:05.177869+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:34:05.177869+0000","last_undegraded":"2023-10-09T06:34:05.177869+0000","last_fullsized":"2023-10-09T06:34:05.177869+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":716,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176231+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:34:05.176231+0000","last_peered":"2023-10-09T06:34:05.176231+0000","last_clean":"2023-10-09T06:34:05.176231+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:34:05.176231+0000","last_undegraded":"2023-10-09T06:34:05.176231+0000","last_fullsized":"2023-10-09T06:34:05.176231+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"415'8207","reported_seq":8301,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.376313+0000","last_change":"2023-10-09T06:34:21.334474+0000","last_active":"2023-10-09T06:34:22.376313+0000","last_peered":"2023-10-09T06:34:22.376313+0000","last_clean":"2023-10-09T06:34:22.376313+0000","last_became_active":"2023-10-09T06:32:32.996849+0000","last_became_peered":"2023-10-09T06:32:32.996849+0000","last_unstale":"2023-10-09T06:34:22.376313+0000","last_undegraded":"2023-10-09T06:34:22.376313+0000","last_fullsized":"2023-10-09T06:34:22.376313+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8207,"log_dups_size":0,"ondisk_log_size":8207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61833216,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5303,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":318,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173581+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:34:05.173581+0000","last_peered":"2023-10-09T06:34:05.173581+0000","last_clean":"2023-10-09T06:34:05.173581+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:34:05.173581+0000","last_undegraded":"2023-10-09T06:34:05.173581+0000","last_fullsized":"2023-10-09T06:34:05.173581+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":391,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177016+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:34:05.177016+0000","last_peered":"2023-10-09T06:34:05.177016+0000","last_clean":"2023-10-09T06:34:05.177016+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:34:05.177016+0000","last_undegraded":"2023-10-09T06:34:05.177016+0000","last_fullsized":"2023-10-09T06:34:05.177016+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":539,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178116+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:34:05.178116+0000","last_peered":"2023-10-09T06:34:05.178116+0000","last_clean":"2023-10-09T06:34:05.178116+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:34:05.178116+0000","last_undegraded":"2023-10-09T06:34:05.178116+0000","last_fullsized":"2023-10-09T06:34:05.178116+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"416'8264","reported_seq":8339,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:25.869971+0000","last_change":"2023-10-09T06:32:32.995971+0000","last_active":"2023-10-09T06:34:25.869971+0000","last_peered":"2023-10-09T06:34:25.869971+0000","last_clean":"2023-10-09T06:34:25.869971+0000","last_became_active":"2023-10-09T06:32:32.993707+0000","last_became_peered":"2023-10-09T06:32:32.993707+0000","last_unstale":"2023-10-09T06:34:25.869971+0000","last_undegraded":"2023-10-09T06:34:25.869971+0000","last_fullsized":"2023-10-09T06:34:25.869971+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8264,"log_dups_size":0,"ondisk_log_size":8264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58753024,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5372,"num_write_kb":200608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179395+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:34:05.179395+0000","last_peered":"2023-10-09T06:34:05.179395+0000","last_clean":"2023-10-09T06:34:05.179395+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:34:05.179395+0000","last_undegraded":"2023-10-09T06:34:05.179395+0000","last_fullsized":"2023-10-09T06:34:05.179395+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":376,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.593675+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:34:13.593675+0000","last_peered":"2023-10-09T06:34:13.593675+0000","last_clean":"2023-10-09T06:34:13.593675+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:34:13.593675+0000","last_undegraded":"2023-10-09T06:34:13.593675+0000","last_fullsized":"2023-10-09T06:34:13.593675+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"415'8117","reported_seq":8197,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.359767+0000","last_change":"2023-10-09T06:34:11.068769+0000","last_active":"2023-10-09T06:34:22.359767+0000","last_peered":"2023-10-09T06:34:22.359767+0000","last_clean":"2023-10-09T06:34:22.359767+0000","last_became_active":"2023-10-09T06:32:32.995923+0000","last_became_peered":"2023-10-09T06:32:32.995923+0000","last_unstale":"2023-10-09T06:34:22.359767+0000","last_undegraded":"2023-10-09T06:34:22.359767+0000","last_fullsized":"2023-10-09T06:34:22.359767+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8117,"log_dups_size":0,"ondisk_log_size":8117,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61669376,"num_objects":941,"num_object_clones":0,"num_object_copies":2823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5245,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":767,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173067+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:34:05.173067+0000","last_peered":"2023-10-09T06:34:05.173067+0000","last_clean":"2023-10-09T06:34:05.173067+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:34:05.173067+0000","last_undegraded":"2023-10-09T06:34:05.173067+0000","last_fullsized":"2023-10-09T06:34:05.173067+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":141,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.595315+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:34:13.595315+0000","last_peered":"2023-10-09T06:34:13.595315+0000","last_clean":"2023-10-09T06:34:13.595315+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:34:13.595315+0000","last_undegraded":"2023-10-09T06:34:13.595315+0000","last_fullsized":"2023-10-09T06:34:13.595315+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":395,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178466+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:34:05.178466+0000","last_peered":"2023-10-09T06:34:05.178466+0000","last_clean":"2023-10-09T06:34:05.178466+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:34:05.178466+0000","last_undegraded":"2023-10-09T06:34:05.178466+0000","last_fullsized":"2023-10-09T06:34:05.178466+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"415'8211","reported_seq":8291,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.403494+0000","last_change":"2023-10-09T06:34:14.119582+0000","last_active":"2023-10-09T06:34:22.403494+0000","last_peered":"2023-10-09T06:34:22.403494+0000","last_clean":"2023-10-09T06:34:22.403494+0000","last_became_active":"2023-10-09T06:32:33.044765+0000","last_became_peered":"2023-10-09T06:32:33.044765+0000","last_unstale":"2023-10-09T06:34:22.403494+0000","last_undegraded":"2023-10-09T06:34:22.403494+0000","last_fullsized":"2023-10-09T06:34:22.403494+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8211,"log_dups_size":0,"ondisk_log_size":8211,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60719104,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5316,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":747,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179229+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:34:05.179229+0000","last_peered":"2023-10-09T06:34:05.179229+0000","last_clean":"2023-10-09T06:34:05.179229+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:34:05.179229+0000","last_undegraded":"2023-10-09T06:34:05.179229+0000","last_fullsized":"2023-10-09T06:34:05.179229+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":372,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176940+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:34:05.176940+0000","last_peered":"2023-10-09T06:34:05.176940+0000","last_clean":"2023-10-09T06:34:05.176940+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:34:05.176940+0000","last_undegraded":"2023-10-09T06:34:05.176940+0000","last_fullsized":"2023-10-09T06:34:05.176940+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":392,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178190+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:34:05.178190+0000","last_peered":"2023-10-09T06:34:05.178190+0000","last_clean":"2023-10-09T06:34:05.178190+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:34:05.178190+0000","last_undegraded":"2023-10-09T06:34:05.178190+0000","last_fullsized":"2023-10-09T06:34:05.178190+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":749,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:07.538387+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:34:07.538387+0000","last_peered":"2023-10-09T06:34:07.538387+0000","last_clean":"2023-10-09T06:34:07.538387+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:34:07.538387+0000","last_undegraded":"2023-10-09T06:34:07.538387+0000","last_fullsized":"2023-10-09T06:34:07.538387+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"415'8189","reported_seq":8262,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.451713+0000","last_change":"2023-10-09T06:32:32.976279+0000","last_active":"2023-10-09T06:34:22.451713+0000","last_peered":"2023-10-09T06:34:22.451713+0000","last_clean":"2023-10-09T06:34:22.451713+0000","last_became_active":"2023-10-09T06:32:32.973171+0000","last_became_peered":"2023-10-09T06:32:32.973171+0000","last_unstale":"2023-10-09T06:34:22.451713+0000","last_undegraded":"2023-10-09T06:34:22.451713+0000","last_fullsized":"2023-10-09T06:34:22.451713+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8189,"log_dups_size":0,"ondisk_log_size":8189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61800448,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5305,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177683+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:34:05.177683+0000","last_peered":"2023-10-09T06:34:05.177683+0000","last_clean":"2023-10-09T06:34:05.177683+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:34:05.177683+0000","last_undegraded":"2023-10-09T06:34:05.177683+0000","last_fullsized":"2023-10-09T06:34:05.177683+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":375,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178340+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:34:05.178340+0000","last_peered":"2023-10-09T06:34:05.178340+0000","last_clean":"2023-10-09T06:34:05.178340+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:34:05.178340+0000","last_undegraded":"2023-10-09T06:34:05.178340+0000","last_fullsized":"2023-10-09T06:34:05.178340+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"415'7982","reported_seq":8056,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.424543+0000","last_change":"2023-10-09T06:32:33.079508+0000","last_active":"2023-10-09T06:34:22.424543+0000","last_peered":"2023-10-09T06:34:22.424543+0000","last_clean":"2023-10-09T06:34:22.424543+0000","last_became_active":"2023-10-09T06:32:33.075123+0000","last_became_peered":"2023-10-09T06:32:33.075123+0000","last_unstale":"2023-10-09T06:34:22.424543+0000","last_undegraded":"2023-10-09T06:34:22.424543+0000","last_fullsized":"2023-10-09T06:34:22.424543+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7982,"log_dups_size":0,"ondisk_log_size":7982,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66682880,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":725,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179429+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:34:05.179429+0000","last_peered":"2023-10-09T06:34:05.179429+0000","last_clean":"2023-10-09T06:34:05.179429+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:34:05.179429+0000","last_undegraded":"2023-10-09T06:34:05.179429+0000","last_fullsized":"2023-10-09T06:34:05.179429+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254719+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:34:08.254719+0000","last_peered":"2023-10-09T06:34:08.254719+0000","last_clean":"2023-10-09T06:34:08.254719+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:34:08.254719+0000","last_undegraded":"2023-10-09T06:34:08.254719+0000","last_fullsized":"2023-10-09T06:34:08.254719+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173759+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:34:05.173759+0000","last_peered":"2023-10-09T06:34:05.173759+0000","last_clean":"2023-10-09T06:34:05.173759+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:34:05.173759+0000","last_undegraded":"2023-10-09T06:34:05.173759+0000","last_fullsized":"2023-10-09T06:34:05.173759+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"416'10714","reported_seq":10779,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:25.927559+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:34:25.927559+0000","last_peered":"2023-10-09T06:34:25.927559+0000","last_clean":"2023-10-09T06:34:25.927559+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:34:25.927559+0000","last_undegraded":"2023-10-09T06:34:25.927559+0000","last_fullsized":"2023-10-09T06:34:25.927559+0000","mapping_epoch":361,"log_start":"374'4100","ondisk_log_start":"374'4100","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114294784,"num_objects":1744,"num_object_clones":0,"num_object_copies":5232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10714,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":707,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176449+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:34:05.176449+0000","last_peered":"2023-10-09T06:34:05.176449+0000","last_clean":"2023-10-09T06:34:05.176449+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:34:05.176449+0000","last_undegraded":"2023-10-09T06:34:05.176449+0000","last_fullsized":"2023-10-09T06:34:05.176449+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":420,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173148+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:34:05.173148+0000","last_peered":"2023-10-09T06:34:05.173148+0000","last_clean":"2023-10-09T06:34:05.173148+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:34:05.173148+0000","last_undegraded":"2023-10-09T06:34:05.173148+0000","last_fullsized":"2023-10-09T06:34:05.173148+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179678+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:34:05.179678+0000","last_peered":"2023-10-09T06:34:05.179678+0000","last_clean":"2023-10-09T06:34:05.179678+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:34:05.179678+0000","last_undegraded":"2023-10-09T06:34:05.179678+0000","last_fullsized":"2023-10-09T06:34:05.179678+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":354,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172807+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:34:05.172807+0000","last_peered":"2023-10-09T06:34:05.172807+0000","last_clean":"2023-10-09T06:34:05.172807+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:34:05.172807+0000","last_undegraded":"2023-10-09T06:34:05.172807+0000","last_fullsized":"2023-10-09T06:34:05.172807+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"416'10732","reported_seq":10797,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:25.896772+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:34:25.896772+0000","last_peered":"2023-10-09T06:34:25.896772+0000","last_clean":"2023-10-09T06:34:25.896772+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:34:25.896772+0000","last_undegraded":"2023-10-09T06:34:25.896772+0000","last_fullsized":"2023-10-09T06:34:25.896772+0000","mapping_epoch":361,"log_start":"376'4200","ondisk_log_start":"376'4200","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115081216,"num_objects":1756,"num_object_clones":0,"num_object_copies":5268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10732,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":761,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179358+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:34:05.179358+0000","last_peered":"2023-10-09T06:34:05.179358+0000","last_clean":"2023-10-09T06:34:05.179358+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:34:05.179358+0000","last_undegraded":"2023-10-09T06:34:05.179358+0000","last_fullsized":"2023-10-09T06:34:05.179358+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179749+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:34:05.179749+0000","last_peered":"2023-10-09T06:34:05.179749+0000","last_clean":"2023-10-09T06:34:05.179749+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:34:05.179749+0000","last_undegraded":"2023-10-09T06:34:05.179749+0000","last_fullsized":"2023-10-09T06:34:05.179749+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":396,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178130+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:34:05.178130+0000","last_peered":"2023-10-09T06:34:05.178130+0000","last_clean":"2023-10-09T06:34:05.178130+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:34:05.178130+0000","last_undegraded":"2023-10-09T06:34:05.178130+0000","last_fullsized":"2023-10-09T06:34:05.178130+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":384,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173073+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:34:05.173073+0000","last_peered":"2023-10-09T06:34:05.173073+0000","last_clean":"2023-10-09T06:34:05.173073+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:34:05.173073+0000","last_undegraded":"2023-10-09T06:34:05.173073+0000","last_fullsized":"2023-10-09T06:34:05.173073+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"415'10684","reported_seq":10746,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.412660+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:34:22.412660+0000","last_peered":"2023-10-09T06:34:22.412660+0000","last_clean":"2023-10-09T06:34:22.412660+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:34:22.412660+0000","last_undegraded":"2023-10-09T06:34:22.412660+0000","last_fullsized":"2023-10-09T06:34:22.412660+0000","mapping_epoch":361,"log_start":"382'4900","ondisk_log_start":"382'4900","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121503744,"num_objects":1854,"num_object_clones":0,"num_object_copies":5562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10684,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":757,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172903+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:34:05.172903+0000","last_peered":"2023-10-09T06:34:05.172903+0000","last_clean":"2023-10-09T06:34:05.172903+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:34:05.172903+0000","last_undegraded":"2023-10-09T06:34:05.172903+0000","last_fullsized":"2023-10-09T06:34:05.172903+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":405,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168588+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:34:05.168588+0000","last_peered":"2023-10-09T06:34:05.168588+0000","last_clean":"2023-10-09T06:34:05.168588+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:34:05.168588+0000","last_undegraded":"2023-10-09T06:34:05.168588+0000","last_fullsized":"2023-10-09T06:34:05.168588+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":392,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173267+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:34:05.173267+0000","last_peered":"2023-10-09T06:34:05.173267+0000","last_clean":"2023-10-09T06:34:05.173267+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:34:05.173267+0000","last_undegraded":"2023-10-09T06:34:05.173267+0000","last_fullsized":"2023-10-09T06:34:05.173267+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":414,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173826+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:34:05.173826+0000","last_peered":"2023-10-09T06:34:05.173826+0000","last_clean":"2023-10-09T06:34:05.173826+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:34:05.173826+0000","last_undegraded":"2023-10-09T06:34:05.173826+0000","last_fullsized":"2023-10-09T06:34:05.173826+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"415'10781","reported_seq":10844,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.430940+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:34:22.430940+0000","last_peered":"2023-10-09T06:34:22.430940+0000","last_clean":"2023-10-09T06:34:22.430940+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:34:22.430940+0000","last_undegraded":"2023-10-09T06:34:22.430940+0000","last_fullsized":"2023-10-09T06:34:22.430940+0000","mapping_epoch":361,"log_start":"387'5349","ondisk_log_start":"387'5349","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120061976,"num_objects":1833,"num_object_clones":0,"num_object_copies":5499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1833,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10781,"num_write_kb":403585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179473+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:34:05.179473+0000","last_peered":"2023-10-09T06:34:05.179473+0000","last_clean":"2023-10-09T06:34:05.179473+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:34:05.179473+0000","last_undegraded":"2023-10-09T06:34:05.179473+0000","last_fullsized":"2023-10-09T06:34:05.179473+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":738,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173308+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:34:05.173308+0000","last_peered":"2023-10-09T06:34:05.173308+0000","last_clean":"2023-10-09T06:34:05.173308+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:34:05.173308+0000","last_undegraded":"2023-10-09T06:34:05.173308+0000","last_fullsized":"2023-10-09T06:34:05.173308+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":124,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.170733+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:34:05.170733+0000","last_peered":"2023-10-09T06:34:05.170733+0000","last_clean":"2023-10-09T06:34:05.170733+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:34:05.170733+0000","last_undegraded":"2023-10-09T06:34:05.170733+0000","last_fullsized":"2023-10-09T06:34:05.170733+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":377,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176926+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:34:05.176926+0000","last_peered":"2023-10-09T06:34:05.176926+0000","last_clean":"2023-10-09T06:34:05.176926+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:34:05.176926+0000","last_undegraded":"2023-10-09T06:34:05.176926+0000","last_fullsized":"2023-10-09T06:34:05.176926+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"416'10907","reported_seq":10970,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:25.931166+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:34:25.931166+0000","last_peered":"2023-10-09T06:34:25.931166+0000","last_clean":"2023-10-09T06:34:25.931166+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:34:25.931166+0000","last_undegraded":"2023-10-09T06:34:25.931166+0000","last_fullsized":"2023-10-09T06:34:25.931166+0000","mapping_epoch":361,"log_start":"376'4300","ondisk_log_start":"376'4300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10907,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":723,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.174398+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:34:05.174398+0000","last_peered":"2023-10-09T06:34:05.174398+0000","last_clean":"2023-10-09T06:34:05.174398+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:34:05.174398+0000","last_undegraded":"2023-10-09T06:34:05.174398+0000","last_fullsized":"2023-10-09T06:34:05.174398+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":278,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173177+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:34:05.173177+0000","last_peered":"2023-10-09T06:34:05.173177+0000","last_clean":"2023-10-09T06:34:05.173177+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:34:05.173177+0000","last_undegraded":"2023-10-09T06:34:05.173177+0000","last_fullsized":"2023-10-09T06:34:05.173177+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.592063+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:34:13.592063+0000","last_peered":"2023-10-09T06:34:13.592063+0000","last_clean":"2023-10-09T06:34:13.592063+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:34:13.592063+0000","last_undegraded":"2023-10-09T06:34:13.592063+0000","last_fullsized":"2023-10-09T06:34:13.592063+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":400,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.904834+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:34:15.904834+0000","last_peered":"2023-10-09T06:34:15.904834+0000","last_clean":"2023-10-09T06:34:15.904834+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:34:15.904834+0000","last_undegraded":"2023-10-09T06:34:15.904834+0000","last_fullsized":"2023-10-09T06:34:15.904834+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"415'10489","reported_seq":10553,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:23.258403+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:34:23.258403+0000","last_peered":"2023-10-09T06:34:23.258403+0000","last_clean":"2023-10-09T06:34:23.258403+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:34:23.258403+0000","last_undegraded":"2023-10-09T06:34:23.258403+0000","last_fullsized":"2023-10-09T06:34:23.258403+0000","mapping_epoch":361,"log_start":"387'5309","ondisk_log_start":"387'5309","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5180,"log_dups_size":0,"ondisk_log_size":5180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115015680,"num_objects":1755,"num_object_clones":0,"num_object_copies":5265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10489,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":737,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.175933+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:34:05.175933+0000","last_peered":"2023-10-09T06:34:05.175933+0000","last_clean":"2023-10-09T06:34:05.175933+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:34:05.175933+0000","last_undegraded":"2023-10-09T06:34:05.175933+0000","last_fullsized":"2023-10-09T06:34:05.175933+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":278,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173654+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:34:05.173654+0000","last_peered":"2023-10-09T06:34:05.173654+0000","last_clean":"2023-10-09T06:34:05.173654+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:34:05.173654+0000","last_undegraded":"2023-10-09T06:34:05.173654+0000","last_fullsized":"2023-10-09T06:34:05.173654+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":410,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173002+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:34:05.173002+0000","last_peered":"2023-10-09T06:34:05.173002+0000","last_clean":"2023-10-09T06:34:05.173002+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:34:05.173002+0000","last_undegraded":"2023-10-09T06:34:05.173002+0000","last_fullsized":"2023-10-09T06:34:05.173002+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":398,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.593887+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:34:13.593887+0000","last_peered":"2023-10-09T06:34:13.593887+0000","last_clean":"2023-10-09T06:34:13.593887+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:34:13.593887+0000","last_undegraded":"2023-10-09T06:34:13.593887+0000","last_fullsized":"2023-10-09T06:34:13.593887+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"415'8236","reported_seq":8310,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:23.259595+0000","last_change":"2023-10-09T06:32:47.459790+0000","last_active":"2023-10-09T06:34:23.259595+0000","last_peered":"2023-10-09T06:34:23.259595+0000","last_clean":"2023-10-09T06:34:23.259595+0000","last_became_active":"2023-10-09T06:32:33.070761+0000","last_became_peered":"2023-10-09T06:32:33.070761+0000","last_unstale":"2023-10-09T06:34:23.259595+0000","last_undegraded":"2023-10-09T06:34:23.259595+0000","last_fullsized":"2023-10-09T06:34:23.259595+0000","mapping_epoch":361,"log_start":"365'544","ondisk_log_start":"365'544","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7692,"log_dups_size":0,"ondisk_log_size":7692,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53149696,"num_objects":811,"num_object_clones":0,"num_object_copies":2433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5371,"num_write_kb":197824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":738,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178041+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:34:05.178041+0000","last_peered":"2023-10-09T06:34:05.178041+0000","last_clean":"2023-10-09T06:34:05.178041+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:34:05.178041+0000","last_undegraded":"2023-10-09T06:34:05.178041+0000","last_fullsized":"2023-10-09T06:34:05.178041+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":145,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:08.254940+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:34:08.254940+0000","last_peered":"2023-10-09T06:34:08.254940+0000","last_clean":"2023-10-09T06:34:08.254940+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:34:08.254940+0000","last_undegraded":"2023-10-09T06:34:08.254940+0000","last_fullsized":"2023-10-09T06:34:08.254940+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":377,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173708+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:34:05.173708+0000","last_peered":"2023-10-09T06:34:05.173708+0000","last_clean":"2023-10-09T06:34:05.173708+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:34:05.173708+0000","last_undegraded":"2023-10-09T06:34:05.173708+0000","last_fullsized":"2023-10-09T06:34:05.173708+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.166698+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:34:05.166698+0000","last_peered":"2023-10-09T06:34:05.166698+0000","last_clean":"2023-10-09T06:34:05.166698+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:34:05.166698+0000","last_undegraded":"2023-10-09T06:34:05.166698+0000","last_fullsized":"2023-10-09T06:34:05.166698+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"415'8130","reported_seq":8211,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.342210+0000","last_change":"2023-10-09T06:34:05.172567+0000","last_active":"2023-10-09T06:34:22.342210+0000","last_peered":"2023-10-09T06:34:22.342210+0000","last_clean":"2023-10-09T06:34:22.342210+0000","last_became_active":"2023-10-09T06:32:33.044852+0000","last_became_peered":"2023-10-09T06:32:33.044852+0000","last_unstale":"2023-10-09T06:34:22.342210+0000","last_undegraded":"2023-10-09T06:34:22.342210+0000","last_fullsized":"2023-10-09T06:34:22.342210+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60522496,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5261,"num_write_kb":197920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":767,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168295+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:34:05.168295+0000","last_peered":"2023-10-09T06:34:05.168295+0000","last_clean":"2023-10-09T06:34:05.168295+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:34:05.168295+0000","last_undegraded":"2023-10-09T06:34:05.168295+0000","last_fullsized":"2023-10-09T06:34:05.168295+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":399,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.176578+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:34:05.176578+0000","last_peered":"2023-10-09T06:34:05.176578+0000","last_clean":"2023-10-09T06:34:05.176578+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:34:05.176578+0000","last_undegraded":"2023-10-09T06:34:05.176578+0000","last_fullsized":"2023-10-09T06:34:05.176578+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":381,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:15.904792+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:34:15.904792+0000","last_peered":"2023-10-09T06:34:15.904792+0000","last_clean":"2023-10-09T06:34:15.904792+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:34:15.904792+0000","last_undegraded":"2023-10-09T06:34:15.904792+0000","last_fullsized":"2023-10-09T06:34:15.904792+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177866+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:34:05.177866+0000","last_peered":"2023-10-09T06:34:05.177866+0000","last_clean":"2023-10-09T06:34:05.177866+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:34:05.177866+0000","last_undegraded":"2023-10-09T06:34:05.177866+0000","last_fullsized":"2023-10-09T06:34:05.177866+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"415'8137","reported_seq":8222,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.427939+0000","last_change":"2023-10-09T06:34:16.084491+0000","last_active":"2023-10-09T06:34:22.427939+0000","last_peered":"2023-10-09T06:34:22.427939+0000","last_clean":"2023-10-09T06:34:22.427939+0000","last_became_active":"2023-10-09T06:32:33.005074+0000","last_became_peered":"2023-10-09T06:32:33.005074+0000","last_unstale":"2023-10-09T06:34:22.427939+0000","last_undegraded":"2023-10-09T06:34:22.427939+0000","last_fullsized":"2023-10-09T06:34:22.427939+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8137,"log_dups_size":0,"ondisk_log_size":8137,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5265,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":693,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168352+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:34:05.168352+0000","last_peered":"2023-10-09T06:34:05.168352+0000","last_clean":"2023-10-09T06:34:05.168352+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:34:05.168352+0000","last_undegraded":"2023-10-09T06:34:05.168352+0000","last_fullsized":"2023-10-09T06:34:05.168352+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":382,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.593815+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:34:13.593815+0000","last_peered":"2023-10-09T06:34:13.593815+0000","last_clean":"2023-10-09T06:34:13.593815+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:34:13.593815+0000","last_undegraded":"2023-10-09T06:34:13.593815+0000","last_fullsized":"2023-10-09T06:34:13.593815+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":403,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172928+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:34:05.172928+0000","last_peered":"2023-10-09T06:34:05.172928+0000","last_clean":"2023-10-09T06:34:05.172928+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:34:05.172928+0000","last_undegraded":"2023-10-09T06:34:05.172928+0000","last_fullsized":"2023-10-09T06:34:05.172928+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.13","version":"416'8263","reported_seq":10375,"reported_epoch":416,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:25.926615+0000","last_change":"2023-10-09T06:34:03.204624+0000","last_active":"2023-10-09T06:34:25.926615+0000","last_peered":"2023-10-09T06:34:25.926615+0000","last_clean":"2023-10-09T06:32:51.037365+0000","last_became_active":"2023-10-09T06:32:53.531678+0000","last_became_peered":"2023-10-09T06:32:53.531678+0000","last_unstale":"2023-10-09T06:34:25.926615+0000","last_undegraded":"2023-10-09T06:32:53.443414+0000","last_fullsized":"2023-10-09T06:32:53.427850+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8263,"log_dups_size":0,"ondisk_log_size":8263,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56852480,"num_objects":868,"num_object_clones":0,"num_object_copies":2604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1578,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5372,"num_write_kb":199648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1016,"num_bytes_recovered":66584576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":359},{"shards":"0,7","objects":-1069}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":624,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594715+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:34:13.594715+0000","last_peered":"2023-10-09T06:34:13.594715+0000","last_clean":"2023-10-09T06:34:13.594715+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:34:13.594715+0000","last_undegraded":"2023-10-09T06:34:13.594715+0000","last_fullsized":"2023-10-09T06:34:13.594715+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":106,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179491+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:34:05.179491+0000","last_peered":"2023-10-09T06:34:05.179491+0000","last_clean":"2023-10-09T06:34:05.179491+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:34:05.179491+0000","last_undegraded":"2023-10-09T06:34:05.179491+0000","last_fullsized":"2023-10-09T06:34:05.179491+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178064+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:34:05.178064+0000","last_peered":"2023-10-09T06:34:05.178064+0000","last_clean":"2023-10-09T06:34:05.178064+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:34:05.178064+0000","last_undegraded":"2023-10-09T06:34:05.178064+0000","last_fullsized":"2023-10-09T06:34:05.178064+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"415'8293","reported_seq":14737,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:23.263410+0000","last_change":"2023-10-09T06:32:52.162166+0000","last_active":"2023-10-09T06:34:23.263410+0000","last_peered":"2023-10-09T06:34:23.263410+0000","last_clean":"2023-10-09T06:32:48.657566+0000","last_became_active":"2023-10-09T06:32:52.155917+0000","last_became_peered":"2023-10-09T06:32:52.155917+0000","last_unstale":"2023-10-09T06:34:23.263410+0000","last_undegraded":"2023-10-09T06:32:52.150896+0000","last_fullsized":"2023-10-09T06:32:52.051991+0000","mapping_epoch":400,"log_start":"365'460","ondisk_log_start":"365'460","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7833,"log_dups_size":0,"ondisk_log_size":7833,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56590336,"num_objects":864,"num_object_clones":0,"num_object_copies":2592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2022,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5390,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139067392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"1,3","objects":-4},{"shards":"3,4,7","objects":868}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":106,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.179590+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:34:05.179590+0000","last_peered":"2023-10-09T06:34:05.179590+0000","last_clean":"2023-10-09T06:34:05.179590+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:34:05.179590+0000","last_undegraded":"2023-10-09T06:34:05.179590+0000","last_fullsized":"2023-10-09T06:34:05.179590+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":432,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.178292+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:34:05.178292+0000","last_peered":"2023-10-09T06:34:05.178292+0000","last_clean":"2023-10-09T06:34:05.178292+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:34:05.178292+0000","last_undegraded":"2023-10-09T06:34:05.178292+0000","last_fullsized":"2023-10-09T06:34:05.178292+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"415'8221","reported_seq":17698,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:22.288501+0000","last_change":"2023-10-09T06:34:18.854131+0000","last_active":"2023-10-09T06:34:22.288501+0000","last_peered":"2023-10-09T06:34:22.288501+0000","last_clean":"2023-10-09T06:34:22.288501+0000","last_became_active":"2023-10-09T06:33:43.622574+0000","last_became_peered":"2023-10-09T06:33:43.622574+0000","last_unstale":"2023-10-09T06:34:22.288501+0000","last_undegraded":"2023-10-09T06:34:22.288501+0000","last_fullsized":"2023-10-09T06:34:22.288501+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8221,"log_dups_size":0,"ondisk_log_size":8221,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5313,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":196673536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":771,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.172667+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:34:05.172667+0000","last_peered":"2023-10-09T06:34:05.172667+0000","last_clean":"2023-10-09T06:34:05.172667+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:34:05.172667+0000","last_undegraded":"2023-10-09T06:34:05.172667+0000","last_fullsized":"2023-10-09T06:34:05.172667+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":115,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.167379+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:34:05.167379+0000","last_peered":"2023-10-09T06:34:05.167379+0000","last_clean":"2023-10-09T06:34:05.167379+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:34:05.167379+0000","last_undegraded":"2023-10-09T06:34:05.167379+0000","last_fullsized":"2023-10-09T06:34:05.167379+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.168860+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:34:05.168860+0000","last_peered":"2023-10-09T06:34:05.168860+0000","last_clean":"2023-10-09T06:34:05.168860+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:34:05.168860+0000","last_undegraded":"2023-10-09T06:34:05.168860+0000","last_fullsized":"2023-10-09T06:34:05.168860+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"416'8210","reported_seq":20482,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:25.857218+0000","last_change":"2023-10-09T06:34:05.191255+0000","last_active":"2023-10-09T06:34:25.857218+0000","last_peered":"2023-10-09T06:34:25.857218+0000","last_clean":"2023-10-09T06:34:25.857218+0000","last_became_active":"2023-10-09T06:34:05.174880+0000","last_became_peered":"2023-10-09T06:34:05.174880+0000","last_unstale":"2023-10-09T06:34:25.857218+0000","last_undegraded":"2023-10-09T06:34:25.857218+0000","last_fullsized":"2023-10-09T06:34:25.857218+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8210,"log_dups_size":0,"ondisk_log_size":8210,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60653568,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5322,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3046,"num_bytes_recovered":199622656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":535,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.173525+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:34:05.173525+0000","last_peered":"2023-10-09T06:34:05.173525+0000","last_clean":"2023-10-09T06:34:05.173525+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:34:05.173525+0000","last_undegraded":"2023-10-09T06:34:05.173525+0000","last_fullsized":"2023-10-09T06:34:05.173525+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":411,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-10-09T06:34:13.594243+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:34:13.594243+0000","last_peered":"2023-10-09T06:34:13.594243+0000","last_clean":"2023-10-09T06:34:13.594243+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:34:13.594243+0000","last_undegraded":"2023-10-09T06:34:13.594243+0000","last_fullsized":"2023-10-09T06:34:13.594243+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":375,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-10-09T06:34:05.177659+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:34:05.177659+0000","last_peered":"2023-10-09T06:34:05.177659+0000","last_clean":"2023-10-09T06:34:05.177659+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:34:05.177659+0000","last_undegraded":"2023-10-09T06:34:05.177659+0000","last_fullsized":"2023-10-09T06:34:05.177659+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":1903984664,"num_objects":29054,"num_object_clones":0,"num_object_copies":87162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3600,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":29054,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165188,"num_write_kb":6175969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9314,"num_bytes_recovered":602144768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6721974272,"data_stored":6721962056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":198704,"ondisk_log_size":198704,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234445,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1995188,"kb_used_data":1094956,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":91749964,"statfs":{"total":95995035648,"available":93951963136,"internally_reserved":0,"allocated":1121234944,"data_stored":1120197217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":5.7720000000000002,"15min":6.1849999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":25.225000000000001,"5min":196.28899999999999,"15min":219.41900000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.097,"5min":5.9139999999999997,"15min":6.3200000000000003},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":25.248000000000001,"5min":196.333,"15min":219.399},"last":0.33300000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":4.516,"15min":4.5620000000000003},"min":{"1min":0.33000000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":8.0540000000000003,"5min":107.87,"15min":215.61699999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.8999999999999999,"5min":3.9540000000000002,"15min":4.1829999999999998},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.23300000000000001},"max":{"1min":7.8220000000000001,"5min":107.904,"15min":216.22200000000001},"last":0.28599999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":3.7879999999999998,"15min":6.718},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27700000000000002},"max":{"1min":5.9809999999999999,"5min":198.94200000000001,"15min":216.34299999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.895,"5min":3.948,"15min":6.8929999999999998},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":5.8239999999999998,"5min":198.92099999999999,"15min":216.25200000000001},"last":0.31900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":9.1150000000000002,"5min":6.282,"15min":4.5720000000000001},"min":{"1min":0.437,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":97.441999999999993,"5min":187.34200000000001,"15min":215.922},"last":0.377},{"interface":"front","average":{"1min":9.0139999999999993,"5min":6.375,"15min":4.641},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":97.162000000000006,"5min":187.08199999999999,"15min":215.94900000000001},"last":3.6709999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.2310000000000001,"15min":2.504},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":1.821,"5min":9.6880000000000006,"15min":218.446},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.111,"15min":2.3599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":2.0619999999999998,"5min":9.7699999999999996,"15min":216.172},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.226,"15min":2.472},"min":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":3.1920000000000002,"5min":8.8170000000000002,"15min":216.37200000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.373,"15min":2.4009999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.28499999999999998},"max":{"1min":3.4860000000000002,"5min":7.8739999999999997,"15min":218.15899999999999},"last":1.0229999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:33:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.464,"15min":2.3039999999999998},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":1.756,"5min":15.162000000000001,"15min":218.19999999999999},"last":0.63},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.0740000000000001,"15min":2.1030000000000002},"min":{"1min":0.36099999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":2.1200000000000001,"5min":8.2919999999999998,"15min":218.05500000000001},"last":0.55500000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496081,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1902296,"kb_used_data":999184,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91842856,"statfs":{"total":95995035648,"available":94047084544,"internally_reserved":0,"allocated":1023164416,"data_stored":1022154059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":3.577,"15min":3.7759999999999998},"min":{"1min":0.22500000000000001,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":47.363999999999997,"5min":75.061999999999998,"15min":110.926},"last":2.4060000000000001},{"interface":"front","average":{"1min":4.7809999999999997,"5min":3.73,"15min":4.0439999999999996},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":47.381,"5min":75.021000000000001,"15min":110.911},"last":2.4159999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":4.4669999999999996,"15min":5.242},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":41.912999999999997,"5min":87.147999999999996,"15min":292.92200000000003},"last":2.8999999999999999},{"interface":"front","average":{"1min":4.0049999999999999,"5min":4.5730000000000004,"15min":5.1159999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":41.927,"5min":87.191000000000003,"15min":292.904},"last":2.9159999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.8220000000000001,"15min":3.1840000000000002},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":47.484000000000002,"5min":74.748999999999995,"15min":75.191999999999993},"last":0.34599999999999997},{"interface":"front","average":{"1min":4.5880000000000001,"5min":3.4820000000000002,"15min":2.9079999999999999},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":47.494999999999997,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":5.3849999999999998,"15min":3.9159999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.761000000000003,"5min":91.174000000000007,"15min":110.358},"last":2.2389999999999999},{"interface":"front","average":{"1min":4.1799999999999997,"5min":5.3019999999999996,"15min":3.9209999999999998},"min":{"1min":0.38200000000000001,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":47.606000000000002,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.47599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.365,"15min":1.278},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.2690000000000001,"5min":12.666,"15min":17.643999999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.1000000000000001,"15min":1.1259999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.223},"max":{"1min":6.7859999999999996,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.58699999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.2769999999999999,"15min":1.377},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29199999999999998},"max":{"1min":5.7539999999999996,"5min":12.747999999999999,"15min":22.379999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5620000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":6.8140000000000001,"5min":16.536999999999999,"15min":24.401},"last":0.53800000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":1.3180000000000001,"15min":2.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":7.4029999999999996,"5min":19.295999999999999,"15min":122.937},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.7270000000000001,"15min":2.6800000000000002},"min":{"1min":0.378,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.4729999999999999,"5min":19.584,"15min":122.92100000000001},"last":0.57199999999999995}]}]},{"osd":1,"up_from":110,"seq":472446402818,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446624,"kb_used_data":698712,"kb_used_omap":4,"kb_used_meta":747899,"kb_avail":92298528,"statfs":{"total":95995035648,"available":94513692672,"internally_reserved":0,"allocated":715481088,"data_stored":713586737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":765849486},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5800000000000001,"5min":5.7990000000000004,"15min":6.1040000000000001},"min":{"1min":0.221,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":26.59,"5min":143.465,"15min":269.79199999999997},"last":1.0169999999999999},{"interface":"front","average":{"1min":3.8290000000000002,"5min":6.0309999999999997,"15min":6.5880000000000001},"min":{"1min":0.22700000000000001,"5min":0.21199999999999999,"15min":0.17899999999999999},"max":{"1min":28.992999999999999,"5min":143.66399999999999,"15min":271.20999999999998},"last":8.827}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":3.9830000000000001,"5min":3.7130000000000001,"15min":3.6230000000000002},"min":{"1min":0.186,"5min":0.184,"15min":0.184},"max":{"1min":22.614999999999998,"5min":138.452,"15min":139.75399999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.6629999999999998,"5min":4.202,"15min":4.3220000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":24.372,"5min":138.47499999999999,"15min":140.29900000000001},"last":1.7509999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:33:25 2023","interfaces":[{"interface":"back","average":{"1min":5.4560000000000004,"5min":3.2639999999999998,"15min":3.8450000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":27.064,"5min":27.064,"15min":153.18199999999999},"last":1.9179999999999999},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.1299999999999999,"15min":2.7250000000000001},"min":{"1min":0.23899999999999999,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":26.626999999999999,"5min":26.626999999999999,"15min":152.52000000000001},"last":1.0569999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":2.5630000000000002,"15min":3.7360000000000002},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.9930000000000003,"5min":38.835999999999999,"15min":199.43799999999999},"last":1.948},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.1840000000000002,"15min":3.3879999999999999},"min":{"1min":0.43099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.5,"5min":38.819000000000003,"15min":199.32900000000001},"last":2.5899999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.536,"15min":2.2330000000000001},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.406000000000001,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.5189999999999999,"5min":2.766,"15min":2.6869999999999998},"min":{"1min":0.48799999999999999,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":4.6150000000000002,"5min":52.664999999999999,"15min":77.388000000000005},"last":2.6339999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":3.9660000000000002,"15min":3.1949999999999998},"min":{"1min":0.46800000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4199999999999999,"5min":107.27200000000001,"15min":107.27200000000001},"last":2.6629999999999998},{"interface":"front","average":{"1min":2.3599999999999999,"5min":4.4320000000000004,"15min":3.4399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.9329999999999998,"5min":107.252,"15min":107.252},"last":1.9690000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":4.1920000000000002,"15min":3.6099999999999999},"min":{"1min":0.47899999999999998,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":8.0229999999999997,"5min":104.07299999999999,"15min":104.07299999999999},"last":1.9890000000000001},{"interface":"front","average":{"1min":1.159,"5min":3.3900000000000001,"15min":2.9169999999999998},"min":{"1min":0.40100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3609999999999998,"5min":104.056,"15min":104.056},"last":0.35399999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024252,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1685928,"kb_used_data":736096,"kb_used_omap":5,"kb_used_meta":949818,"kb_avail":92059224,"statfs":{"total":95995035648,"available":94268645376,"internally_reserved":0,"allocated":753762304,"data_stored":751579291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":972614266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.726,"15min":3.4729999999999999},"min":{"1min":0.19700000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":7.4909999999999997,"5min":14.977,"15min":191.446},"last":0.23200000000000001},{"interface":"front","average":{"1min":2.6429999999999998,"5min":2.2080000000000002,"15min":3.7410000000000001},"min":{"1min":0.22900000000000001,"5min":0.187,"15min":0.187},"max":{"1min":9.2599999999999998,"5min":15.129,"15min":191.393},"last":5.0389999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:33:52 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.5009999999999999,"15min":2.5350000000000001},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":9.1859999999999999,"5min":16.936,"15min":57.162999999999997},"last":6.1239999999999997},{"interface":"front","average":{"1min":2.8889999999999998,"5min":3.3210000000000002,"15min":2.9889999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.20999999999999999},"max":{"1min":8.9450000000000003,"5min":17.962,"15min":57.585000000000001},"last":1.7829999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:11 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":6.5800000000000001,"15min":5.9820000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":15.413,"5min":187.172,"15min":208.11199999999999},"last":22.670000000000002},{"interface":"front","average":{"1min":3.9609999999999999,"5min":6.2960000000000003,"15min":5.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":15.266,"5min":194.851,"15min":208.25899999999999},"last":15.471}]},{"osd":4,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.78,"15min":1.593},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":5.6059999999999999,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.2749999999999999,"5min":2.2509999999999999,"15min":1.929},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":8.2880000000000003,"5min":18.081,"15min":18.081},"last":5.117}]},{"osd":5,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.883,"15min":1.907},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":6.2809999999999997,"5min":9.7720000000000002,"15min":62.984999999999999},"last":0.309},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.492,"15min":2.3450000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":6.1529999999999996,"5min":15.747999999999999,"15min":64.983999999999995},"last":1.7569999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":3.4790000000000001,"15min":2.9540000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":6.2069999999999999,"5min":85.525999999999996,"15min":120.572},"last":1.7270000000000001},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.8370000000000002,"15min":2.4460000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.30299999999999999},"max":{"1min":5.6440000000000001,"5min":85.734999999999999,"15min":120.64400000000001},"last":0.38600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.5299999999999998,"15min":2.6579999999999999},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.32300000000000001},"max":{"1min":8.3249999999999993,"5min":18.041,"15min":122.018},"last":5.3319999999999999},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.9119999999999999,"15min":2.972},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":7.7089999999999996,"5min":26.416,"15min":122.32299999999999},"last":1.6970000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814121,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1797416,"kb_used_data":1103904,"kb_used_omap":7,"kb_used_meta":693496,"kb_avail":91947736,"statfs":{"total":95995035648,"available":94154481664,"internally_reserved":0,"allocated":1130397696,"data_stored":1128559395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":710139985},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7160000000000002,"5min":5.1029999999999998,"15min":4.9429999999999996},"min":{"1min":0.309,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":72.125,"5min":82.751000000000005,"15min":162.21100000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":5.9829999999999997,"5min":4.8460000000000001,"15min":4.4989999999999997},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":75.456000000000003,"5min":81.960999999999999,"15min":168.49700000000001},"last":1.5289999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":4.0499999999999998,"15min":4.1390000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20599999999999999},"max":{"1min":16.152999999999999,"5min":74.744,"15min":205.148},"last":6.7830000000000004},{"interface":"front","average":{"1min":2.4009999999999998,"5min":4.5,"15min":4.335},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":8.9819999999999993,"5min":75.759,"15min":205.267},"last":1.552}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2880000000000003,"5min":3.968,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":75.498999999999995,"5min":77.692999999999998,"15min":91.644000000000005},"last":3.1960000000000002},{"interface":"front","average":{"1min":6.2220000000000004,"5min":4.3140000000000001,"15min":3.9409999999999998},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":69.784000000000006,"5min":78.575999999999993,"15min":91.685000000000002},"last":2.8119999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":2.1699999999999999,"15min":2.2669999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":6.2839999999999998,"5min":44.314,"15min":90.938999999999993},"last":1.4450000000000001},{"interface":"front","average":{"1min":2.3690000000000002,"5min":2.492,"15min":2.5880000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":8.6099999999999994,"5min":43.896000000000001,"15min":90.950000000000003},"last":4.3120000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":2.0339999999999998,"15min":1.7070000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.3140000000000001,"5min":13.102,"15min":17.800000000000001},"last":1.476},{"interface":"front","average":{"1min":2.4670000000000001,"5min":2.2389999999999999,"15min":2.0089999999999999},"min":{"1min":0.39100000000000001,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":8.6829999999999998,"5min":11.285,"15min":11.285},"last":4.3840000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":4.2460000000000004,"15min":3.2999999999999998},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":8.6600000000000001,"5min":123.47799999999999,"15min":138.291},"last":4.3520000000000003},{"interface":"front","average":{"1min":3.677,"5min":4.133,"15min":3.093},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":16.012,"5min":122.517,"15min":138.01599999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5150000000000001,"5min":2.875,"15min":2.3399999999999999},"min":{"1min":0.36499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":16.064,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.46,"5min":2.5910000000000002,"15min":2.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":6.3460000000000001,"5min":47.206000000000003,"15min":55.731000000000002},"last":1.5009999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127318,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1731468,"kb_used_data":1075204,"kb_used_omap":2,"kb_used_meta":656253,"kb_avail":92013684,"statfs":{"total":95995035648,"available":94222012416,"internally_reserved":0,"allocated":1101008896,"data_stored":1099249902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":672003125},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":7.1669999999999998,"15min":5.8540000000000001},"min":{"1min":0.20000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":22.367000000000001,"5min":161.821,"15min":187.852},"last":1.0760000000000001},{"interface":"front","average":{"1min":2.3140000000000001,"5min":6.1689999999999996,"15min":5.8490000000000002},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":13.443,"5min":163.52000000000001,"15min":187.923},"last":1.349}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9209999999999998,"5min":4.4930000000000003,"15min":4.375},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":22.349,"5min":155.23099999999999,"15min":187.89099999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":4.1050000000000004,"5min":6,"15min":4.2480000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":15.066000000000001,"5min":155.24799999999999,"15min":155.24799999999999},"last":1.101}]},{"osd":2,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.7329999999999997,"15min":4.234},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":22.297000000000001,"5min":167.88499999999999,"15min":187.76400000000001},"last":1.274},{"interface":"front","average":{"1min":3.5720000000000001,"5min":5.4390000000000001,"15min":4.2380000000000004},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":13.631,"5min":168.16900000000001,"15min":187.79499999999999},"last":1.0309999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.9159999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.661,"5min":48.249000000000002,"15min":48.249000000000002},"last":0.78300000000000003},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.887,"15min":2.645},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":16.516999999999999,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.7389999999999999,"15min":2.7250000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":16.494,"5min":48.162999999999997,"15min":187.833},"last":0.437},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.1520000000000001,"15min":2.4300000000000002},"min":{"1min":0.42299999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":18.835999999999999,"5min":48.106000000000002,"15min":187.952},"last":1.298}]},{"osd":6,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":4.3339999999999996,"15min":3.3319999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.462,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.3199999999999998,"5min":3.8679999999999999,"15min":2.9900000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":22.260000000000002,"5min":130.67699999999999,"15min":187.81800000000001},"last":1.325}]},{"osd":7,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":4.5229999999999997,"15min":3.3769999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":13.695,"5min":121.613,"15min":192.20400000000001},"last":0.753},{"interface":"front","average":{"1min":3.0110000000000001,"5min":4.7270000000000003,"15min":3.4569999999999999},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.542000000000002,"5min":121.559,"15min":187.86500000000001},"last":1.0569999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019354,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553504,"kb_used_data":658520,"kb_used_omap":0,"kb_used_meta":894976,"kb_avail":92191648,"statfs":{"total":95995035648,"available":94404247552,"internally_reserved":0,"allocated":674324480,"data_stored":673306308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916455424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":11.006,"5min":11.769,"15min":5.6020000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":118.98999999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":2.1120000000000001},{"interface":"front","average":{"1min":11.007,"5min":11.597,"15min":5.8220000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":118.956,"5min":287.57100000000003,"15min":287.57100000000003},"last":2.1230000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":3.4100000000000001,"5min":4.4550000000000001,"15min":3.1779999999999999},"min":{"1min":0.30099999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":25.879000000000001,"5min":128.24299999999999,"15min":128.24299999999999},"last":1.613},{"interface":"front","average":{"1min":3.2810000000000001,"5min":4.6719999999999997,"15min":3.3700000000000001},"min":{"1min":0.41799999999999998,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":25.617999999999999,"5min":128.21600000000001,"15min":128.21600000000001},"last":1.5720000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":3.411,"15min":3.0619999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":7.7039999999999997,"5min":151.66399999999999,"15min":151.66399999999999},"last":1.587},{"interface":"front","average":{"1min":1.833,"5min":3.1469999999999998,"15min":3.1179999999999999},"min":{"1min":0.34899999999999998,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":6.8070000000000004,"5min":151.58699999999999,"15min":151.58699999999999},"last":2.9119999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":8.2629999999999999,"5min":10.750999999999999,"15min":5.8639999999999999},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":65.915000000000006,"5min":161.19300000000001,"15min":161.19300000000001},"last":1.635},{"interface":"front","average":{"1min":8.6899999999999995,"5min":10.590999999999999,"15min":5.5880000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":68.054000000000002,"5min":161.934,"15min":161.934},"last":3.02}]},{"osd":5,"last update":"Mon Oct 9 06:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.90500000000000003,"15min":1.008},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.5070000000000001,"5min":6.25,"15min":10.673},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.1120000000000001,"15min":1.0429999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":3.282,"5min":7.2249999999999996,"15min":12.552},"last":1.548}]},{"osd":6,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.4620000000000002,"15min":1.2929999999999999},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":2.5179999999999998,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.641,"15min":1.6419999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":2.9340000000000002,"5min":163.381,"15min":163.381},"last":1.294}]},{"osd":7,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.6499999999999999,"15min":1.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":2.9649999999999999,"5min":70.247,"15min":70.247},"last":0.623},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.774,"15min":1.6160000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0680000000000001,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.495}]}]},{"osd":5,"up_from":38,"seq":163208757719,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1201432,"kb_used_data":307920,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":92543720,"statfs":{"total":95995035648,"available":94764769280,"internally_reserved":0,"allocated":315310080,"data_stored":314296352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":6.1609999999999996,"15min":4.9950000000000001},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.216},"max":{"1min":8.6940000000000008,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.47,"5min":5.9009999999999998,"15min":4.8280000000000003},"min":{"1min":0.34999999999999998,"5min":0.24399999999999999,"15min":0.22},"max":{"1min":5.6539999999999999,"5min":294.30399999999997,"15min":294.30399999999997},"last":0.245}]},{"osd":1,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":3.2090000000000001,"15min":2.8700000000000001},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4160000000000004,"5min":87.795000000000002,"15min":126.71899999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.357,"5min":3.4300000000000002,"15min":3.02},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":3.2320000000000002,"5min":87.856999999999999,"15min":126.931},"last":0.61799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.625,"15min":2.5019999999999998},"min":{"1min":0.311,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.6589999999999998,"5min":64.150000000000006,"15min":81.260000000000005},"last":2.8519999999999999},{"interface":"front","average":{"1min":1.893,"5min":2.4489999999999998,"15min":2.4049999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":8.1470000000000002,"5min":64.507000000000005,"15min":81.245000000000005},"last":0.70299999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":5.6289999999999996,"15min":4.6790000000000003},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":38.582000000000001,"5min":144.56399999999999,"15min":144.56399999999999},"last":2.415},{"interface":"front","average":{"1min":3.9409999999999998,"5min":5.3700000000000001,"15min":4.5099999999999998},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":38.447000000000003,"5min":144.62799999999999,"15min":144.62799999999999},"last":0.54500000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":1.0600000000000001,"15min":1.1839999999999999},"min":{"1min":0.26100000000000001,"5min":0.25,"15min":0.25},"max":{"1min":1.7150000000000001,"5min":10.529999999999999,"15min":32.850999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.871,"15min":1.0509999999999999},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":0.93500000000000005,"5min":6.2610000000000001,"15min":32.82},"last":0.433}]},{"osd":6,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.7250000000000001,"15min":1.6200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7839999999999998,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.9810000000000001,"15min":1.7450000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0630000000000002,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.41699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.454,"15min":1.397},"min":{"1min":0.374,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.96,"5min":10.461,"15min":30.658999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.218,"15min":1.4099999999999999},"min":{"1min":0.39300000000000002,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":3.4289999999999998,"5min":6.2320000000000002,"15min":30.683},"last":0.48099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":742457344,"data_stored":742457344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":696651776,"data_stored":696647704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1103695872,"data_stored":1103691800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1081999360,"data_stored":1081999360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":669319168,"data_stored":669319168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":305856512,"data_stored":305856512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1008402432,"data_stored":1008402432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1113591808,"data_stored":1113587736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:31.424 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:34:31.455 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:31.456 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:31.462 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:31.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:31.578 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:31.614 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:31.637 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:31.897 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:31.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:31.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:32.002 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:32.083 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:32] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:34:32.097 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:32.279 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:32.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:32.371 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:34:32.411 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:32.522 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:32.627 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:32.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:32.743 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:32.790 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:32.832 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:32.877 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:32.883 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:32 smithi150 ceph-mon[194178]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 3428/76932 objects degraded (4.456%); 56/76932 objects misplaced (0.073%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:34:32.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:32.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:32.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3428/76932 objects degraded (4.456%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:32.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:32.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[212750]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 3428/76932 objects degraded (4.456%); 56/76932 objects misplaced (0.073%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3428/76932 objects degraded (4.456%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[220921]: pgmap v1279: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 3428/76932 objects degraded (4.456%); 56/76932 objects misplaced (0.073%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:34:32.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3428/76932 objects degraded (4.456%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:32.985 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:33.014 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:33.160 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:33.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:33.264 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:33.350 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:33.503 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:34:33.668 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:33.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:33.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:33.706 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:33.782 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:33.822 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:33.881 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:33.883 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:33.883 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:33.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T06:34:33.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.884 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:33 smithi150 ceph-mon[194178]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:33.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:33.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T06:34:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[220921]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:33.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/360124133' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:33.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: osdmap e417: 8 total, 8 up, 8 in 2023-10-09T06:34:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:33 smithi107 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:34:33.985 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:34.077 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:34.412 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:34.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:34.525 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.563 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:34.650 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:34.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:34.725 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[212750]: pgmap v1280: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:34.725 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.725 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[212750]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[220921]: pgmap v1280: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:34.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[220921]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:34 smithi107 ceph-mon[220921]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.755 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:34.790 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:34.861 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:34.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:34 smithi150 ceph-mon[194178]: pgmap v1280: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:34.866 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:34 smithi150 ceph-mon[194178]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.867 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:34 smithi150 ceph-mon[194178]: failed to encode map e417 with expected crc 2023-10-09T06:34:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:34.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:34.968 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:35.007 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:35.058 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:35.058 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:35.062 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:35.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:35.168 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:35.269 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:35.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:35.389 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:35.425 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:35.456 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:35.456 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:35.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:35.486 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:35.510 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:35.563 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.599 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:35.660 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:35.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:35.765 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:35.791 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.798 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:34:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/939025219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:36.008 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:36.012 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:34:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:36.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:36.116 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:36.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/939025219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:36.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/939025219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:36.154 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:36.222 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:36.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:36.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:36.328 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:36.361 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:36.424 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:36.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:36.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:36.542 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:36.572 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:34:36.572 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:36.628 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:36.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:36.734 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:36.794 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:36.898 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:36 smithi150 ceph-mon[194178]: pgmap v1282: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:36.905 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:36 smithi107 ceph-mon[212750]: pgmap v1282: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:36.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:36 smithi107 ceph-mon[220921]: pgmap v1282: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 3241/69534 objects degraded (4.661%); 56/69534 objects misplaced (0.081%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:34:37.014 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:37.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:37.136 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:37.224 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:37.343 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:37.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:37.426 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:37.444 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:37.478 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:37.659 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:37.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:37.778 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:37.887 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:37.918 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:34:37.918 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 3198/66564 objects degraded (4.804%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1690,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":417,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":8,"osd_in_since":1696832954,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":127},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":22188,"data_bytes":1452906152,"bytes_used":12262064128,"bytes_avail":755698221056,"bytes_total":767960285184,"degraded_objects":3198,"degraded_total":66564,"degraded_ratio":0.048043987741121327,"recovering_objects_per_sec":74,"recovering_bytes_per_sec":3520700,"recovering_keys_per_sec":0,"num_objects_recovered":749,"num_bytes_recovered":35258368,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":687},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":27,"modified":"2023-10-09T06:34:19.394028+0000","services":{}},"progress_events":{"3403e6b5-aeb3-47f0-97ab-e50de33524c0":{"message":"Global Recovery Event (115s)\n [===========================.] (remaining: 1s)","progress":0.98449611663818359,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:34:37.967 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3241/69534 objects degraded (4.661%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:37.968 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3241/69534 objects degraded (4.661%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:38.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3241/69534 objects degraded (4.661%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:38.479 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:34:38.479 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:34:38.479 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:34:38.479 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:34:38.479 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:34:38.556 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:38.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:38.663 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:38.741 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:38.800 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:38.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:38.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:38 smithi107 ceph-mon[212750]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 3198/66564 objects degraded (4.804%); 3.4 MiB/s, 74 objects/s recovering 2023-10-09T06:34:38.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3199348621' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:34:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:38 smithi107 ceph-mon[220921]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 3198/66564 objects degraded (4.804%); 3.4 MiB/s, 74 objects/s recovering 2023-10-09T06:34:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3199348621' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:34:38.904 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:38 smithi150 ceph-mon[194178]: pgmap v1283: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 3198/66564 objects degraded (4.804%); 3.4 MiB/s, 74 objects/s recovering 2023-10-09T06:34:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3199348621' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:34:38.933 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:38.976 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:38.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:39.054 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:34:39.082 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:39.165 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:39.433 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:34:39.712 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:39.712 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.712 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.776 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:39.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:39.883 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:34:39.930 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:39.975 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:39.985 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:39.985 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.986 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:34:39.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:39.987 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:34:39.987 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:40.006 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:34:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:40.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:34:40.112 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:40.256 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:40.421 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:40.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:40.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:40.535 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:40.558 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:40.684 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:40.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[212750]: pgmap v1284: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2986/58662 objects degraded (5.090%); 3.5 MiB/s, 76 objects/s recovering 2023-10-09T06:34:40.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:40.909 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:40.934 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:40.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:41.040 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:41.123 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:41.239 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:41.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[220921]: pgmap v1284: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2986/58662 objects degraded (5.090%); 3.5 MiB/s, 76 objects/s recovering 2023-10-09T06:34:41.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:41.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:41.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:40 smithi150 ceph-mon[194178]: pgmap v1284: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 2986/58662 objects degraded (5.090%); 3.5 MiB/s, 76 objects/s recovering 2023-10-09T06:34:41.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:41.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:41.441 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:41.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:41.550 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:41.644 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:41.959 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:34:42.149 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4192999345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.149 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4192999345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4192999345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:34:42.402 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:42] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67291 11.9 11:969e6fb0:::benchmark_data_smithi107_418363_object67290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:34:42.473 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 437.59540715100002, 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2541491790000001, 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67291 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875230+0000", 2023-10-09T06:34:42.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875284+0000", 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3637999999999997e-05 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875295+0000", 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1396000000000001e-05 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875303+0000", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1629999999999996e-06 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875356+0000", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2541999999999999e-05 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875364+0000", 2023-10-09T06:34:42.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6119999999999999e-06 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872682+0000", 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99731777899999996 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872691+0000", 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.619e-06 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.128229+0000", 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.255537956 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.129381+0000", 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011520840000000001 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67302 11.9 11:93267d49:::benchmark_data_smithi107_418363_object67301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 437.59532218999999, 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2727547110000001, 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.480 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67302 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:34:42.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875345+0000", 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9431e-05 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875351+0000", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5110000000000003e-06 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875356+0000", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875401+0000", 2023-10-09T06:34:42.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4462999999999998e-05 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875409+0000", 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3569999999999998e-06 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872710+0000", 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99730035399999994 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872716+0000", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1979999999999998e-06 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.129415+0000", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2566990149999999 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for degraded object", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.130278+0000", 2023-10-09T06:34:42.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00086301399999999995 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.147994+0000", 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017716686999999998 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.148071+0000", 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6717000000000004e-05 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.486 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67308 11.a 11.16ff13ca (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 437.59527407600001, 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 11.74595132, 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.487 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67308 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:34:42.488 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875364+0000", 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875439+0000", 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4506999999999997e-05 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875445+0000", 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2570000000000004e-06 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875448+0000", 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4589999999999998e-06 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875483+0000", 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4714e-05 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875489+0000", 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4060000000000004e-06 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871373+0000", 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99588437299999999 2023-10-09T06:34:42.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871384+0000", 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1302e-05 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109463+0000", 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 10.238078988 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109475+0000", 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2072000000000001e-05 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621303+0000", 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51182799400000001 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621316+0000", 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2662e-05 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67311 11.1 11:8e63c8cd:::benchmark_data_smithi107_418363_object67310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 437.58247488000001, 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2810258819999998, 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.494 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67311 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888162+0000", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888216+0000", 2023-10-09T06:34:42.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3372000000000003e-05 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888243+0000", 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7721000000000001e-05 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888266+0000", 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2503000000000001e-05 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888334+0000", 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8200000000000004e-05 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888343+0000", 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5529999999999993e-06 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.874624+0000", 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98628109200000003 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.874633+0000", 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6120000000000005e-06 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158060+0000", 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2834271070000001 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158218+0000", 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015740799999999999 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158398+0000", 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000179925 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166665+0000", 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082676780000000005 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166676+0000", 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0584000000000001e-05 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166718+0000", 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2141000000000002e-05 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166800+0000", 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1827000000000001e-05 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.169190+0000", 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023897020000000001 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67312 11.a 11.25d90f4a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 437.58162534000002, 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 11.732321325000001, 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67312 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.503 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889078+0000", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5513999999999996e-05 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889088+0000", 2023-10-09T06:34:42.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9129999999999998e-06 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889092+0000", 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7459999999999998e-06 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889132+0000", 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9773e-05 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889139+0000", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6850000000000001e-06 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.873695+0000", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98455533500000003 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.873705+0000", 2023-10-09T06:34:42.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0027000000000001e-05 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109514+0000", 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 10.235808864999999 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109522+0000", 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0639999999999994e-06 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621331+0000", 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51180884800000004 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621335+0000", 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0990000000000001e-06 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67318 11.a 11.c6b30b6a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.41627808099997, 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5669876509999998, 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67318 2023-10-09T06:34:42.510 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.511 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054359+0000", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054498+0000", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013886700000000001 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054512+0000", 2023-10-09T06:34:42.512 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4354e-05 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054522+0000", 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9839999999999996e-06 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109546+0000", 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0550239999999995 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.513 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109550+0000", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.569e-06 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621345+0000", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51179456199999995 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621348+0000", 2023-10-09T06:34:42.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.704e-06 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67327 11.a 11.c2bd3d8a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.311502867, 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4622259500000006, 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.515 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67327 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159135+0000", 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159191+0000", 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6713999999999999e-05 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159202+0000", 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0988000000000001e-05 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159207+0000", 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5809999999999998e-06 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.518 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109572+0000", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9503646000000003 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109575+0000", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4240000000000002e-06 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621358+0000", 2023-10-09T06:34:42.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51178295900000004 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621362+0000", 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7419999999999999e-06 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67328 11.a 11.cf83d70a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:34:42.520 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.30808706800002, 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4588306979999999, 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67328 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.521 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162550+0000", 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162624+0000", 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3619000000000001e-05 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162635+0000", 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0716999999999999e-05 2023-10-09T06:34:42.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162642+0000", 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2030000000000003e-06 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109597+0000", 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9469555560000007 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.524 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109601+0000", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9090000000000002e-06 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621376+0000", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51177449799999997 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621382+0000", 2023-10-09T06:34:42.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4010000000000002e-06 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67353 11.a 11.6050658a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.21752911800002, 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3682885060000007, 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.526 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67353 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253108+0000", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253152+0000", 2023-10-09T06:34:42.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3838999999999998e-05 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253166+0000", 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4202999999999999e-05 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253176+0000", 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4520000000000003e-06 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109622+0000", 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8564464330000003 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109625+0000", 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000002e-06 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621394+0000", 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51176914200000001 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621398+0000", 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6100000000000002e-06 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67375 11.a 11.25a7be1a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.531 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.16165410999997, 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3124281040000003, 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67375 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.532 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308983+0000", 2023-10-09T06:34:42.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309029+0000", 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6217999999999997e-05 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309045+0000", 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6220000000000001e-05 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309052+0000", 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7970000000000003e-06 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109646+0000", 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8005935120000007 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109649+0000", 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9550000000000001e-06 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621408+0000", 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51175970999999998 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621413+0000", 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2930000000000002e-06 2023-10-09T06:34:42.536 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67383 11.a 11.e2ca7dea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.15379217499998, 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3045890759999992, 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.537 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67383 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.538 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316845+0000", 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316884+0000", 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8518e-05 2023-10-09T06:34:42.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316897+0000", 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3210000000000001e-05 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316904+0000", 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0280000000000003e-06 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109669+0000", 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7927653879999994 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109672+0000", 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0910000000000001e-06 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.541 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621429+0000", 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51175660300000003 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621436+0000", 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.542 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67400 11.a 11.aa0c4bfa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.129934128, 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2807484280000008, 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67400 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.543 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340703+0000", 2023-10-09T06:34:42.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340786+0000", 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2757999999999996e-05 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340804+0000", 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7740999999999998e-05 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340812+0000", 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2530000000000004e-06 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109692+0000", 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7688797350000005 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.546 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109695+0000", 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.631e-06 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621449+0000", 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.511753817 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621453+0000", 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6569999999999999e-06 2023-10-09T06:34:42.547 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67408 11.a 11.11553a3a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.11855147400001, 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2693790020000009, 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.548 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67408 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:34:42.549 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352141+0000", 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4262999999999999e-05 2023-10-09T06:34:42.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352153+0000", 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2035000000000001e-05 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352158+0000", 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9289999999999997e-06 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109715+0000", 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.757557061 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109718+0000", 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1810000000000001e-06 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621463+0000", 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.511744529 2023-10-09T06:34:42.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621466+0000", 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5140000000000002e-06 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67417 11.a 11.b16a0e7a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:34:42.553 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.09965335499999, 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2504945739999993, 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67417 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.554 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370984+0000", 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371054+0000", 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9986000000000004e-05 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371068+0000", 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3971999999999999e-05 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371075+0000", 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3050000000000004e-06 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109738+0000", 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7386627840000006 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109742+0000", 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7689999999999998e-06 2023-10-09T06:34:42.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621476+0000", 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51173471999999998 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621480+0000", 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5659999999999998e-06 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.558 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67437 11.a 11.396f3c4a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.060550428, 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2114045979999997, 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.559 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67437 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410086+0000", 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410124+0000", 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7249999999999997e-05 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410138+0000", 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4321000000000001e-05 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410145+0000", 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5030000000000002e-06 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109761+0000", 2023-10-09T06:34:42.562 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6996166180000003 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109764+0000", 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.932e-06 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621489+0000", 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51172516300000004 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.563 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621493+0000", 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6030000000000001e-06 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67497 11.a 11.30b451da (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 434.94083576200001, 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0917029520000003, 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.564 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67497 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.565 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529802+0000", 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.566 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529851+0000", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9054999999999997e-05 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529862+0000", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529866+0000", 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1500000000000001e-06 2023-10-09T06:34:42.567 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109784+0000", 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5799179349999992 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109787+0000", 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9170000000000002e-06 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.568 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621502+0000", 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51171517700000002 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621506+0000", 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4879999999999999e-06 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.569 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67524 11.a 11.b059deda (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 434.87688495899999, 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0277651169999995, 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67524 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.570 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.571 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593752+0000", 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593814+0000", 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2033999999999996e-05 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593825+0000", 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1216999999999999e-05 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.572 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593830+0000", 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1309999999999997e-06 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109806+0000", 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5159759810000004 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.573 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109809+0000", 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1259999999999998e-06 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621515+0000", 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51170579199999999 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621519+0000", 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.692e-06 2023-10-09T06:34:42.574 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67531 11.a 11.e295152a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 434.85335098799999, 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0042557280000004, 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.575 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67531 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:34:42.576 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617286+0000", 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617336+0000", 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9450000000000003e-05 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617354+0000", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8726e-05 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617362+0000", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4410000000000004e-06 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109829+0000", 2023-10-09T06:34:42.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4924671380000003 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109833+0000", 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6780000000000002e-06 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621535+0000", 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51170254999999998 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621543+0000", 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3489999999999992e-06 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:125509 11.1 11:80899c97:::benchmark_data_smithi107_418363_object35073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 339.127270813, 2023-10-09T06:34:42.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.167635185, 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 125509 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343366+0000", 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343374+0000", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343387+0000", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2782000000000001e-05 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343394+0000", 2023-10-09T06:34:42.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2559999999999996e-06 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343702+0000", 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00030755499999999999 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343709+0000", 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4909999999999999e-06 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.584 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322829+0000", 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97911982099999995 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322840+0000", 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.128e-05 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.483725+0000", 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1608853180000001 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.483843+0000", 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117111 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2,3", 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.484379+0000", 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00053623600000000005 2023-10-09T06:34:42.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493120+0000", 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0087413509999999996 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493251+0000", 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000130691 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.508110+0000", 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014858949 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.510802+0000", 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002692046 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.510869+0000", 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7124000000000003e-05 2023-10-09T06:34:42.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.511003+0000", 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000134101 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:125522 11.1 11:8d066771:::benchmark_data_smithi107_418363_object35086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 339.092576952, 2023-10-09T06:34:42.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1353189160000001, 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 125522 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:34:42.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378060+0000", 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:34:42.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378069+0000", 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1239999999999994e-06 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378090+0000", 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1047e-05 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:34:42.592 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378098+0000", 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6480000000000003e-06 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378180+0000", 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1672e-05 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378192+0000", 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.224e-05 2023-10-09T06:34:42.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322866+0000", 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94467452600000001 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322873+0000", 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.165e-06 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488489+0000", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.165615474 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488641+0000", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015251699999999999 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2,3", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488840+0000", 2023-10-09T06:34:42.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000198171 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493124+0000", 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042848590000000002 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.497914+0000", 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047894360000000002 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.511045+0000", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013131267 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513239+0000", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021935280000000001 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513284+0000", 2023-10-09T06:34:42.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5090999999999999e-05 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513381+0000", 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6842999999999994e-05 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:34:42.598 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:34:43.264 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:43.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2745/55869 objects degraded (4.913%); 1.6 MiB/s, 70 objects/s recovering 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T06:34:43.271 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2745/55869 objects degraded (4.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2745/55869 objects degraded (4.913%); 1.6 MiB/s, 70 objects/s recovering 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.272 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2745/55869 objects degraded (4.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:43.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:43.373 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: pgmap v1285: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2745/55869 objects degraded (4.913%); 1.6 MiB/s, 70 objects/s recovering 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: osdmap e418: 8 total, 8 up, 8 in 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: failed to encode map e418 with expected crc 2023-10-09T06:34:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2745/55869 objects degraded (4.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:43.455 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:43.469 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:34:44.111 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:44.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:43 smithi107 ceph-mon[212750]: 12.1 scrub starts 2023-10-09T06:34:44.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:43 smithi107 ceph-mon[220921]: 12.1 scrub starts 2023-10-09T06:34:44.338 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:43 smithi150 ceph-mon[194178]: 12.1 scrub starts 2023-10-09T06:34:44.558 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:44.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:44.685 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:44.825 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:45.066 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:45.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:45.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:44 smithi107 ceph-mon[220921]: pgmap v1287: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 73 objects/s recovering 2023-10-09T06:34:45.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:44 smithi107 ceph-mon[212750]: pgmap v1287: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 73 objects/s recovering 2023-10-09T06:34:45.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:44 smithi150 ceph-mon[194178]: pgmap v1287: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 73 objects/s recovering 2023-10-09T06:34:45.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:45.206 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:45.210 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:45.234 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:45.256 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:45.285 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:45.322 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:45.367 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:45.408 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:45.455 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:45.472 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.472 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.472 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.472 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.473 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.473 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.473 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.473 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:34:45.909 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:45.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:45.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:45 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:34:45.972 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3204581912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:46.022 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:46.042 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:34:46.057 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:34:46.126 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:45 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:34:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3204581912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:45 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:34:46.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3204581912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:46.593 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:46.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:46.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:46.718 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:34:46.754 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:46.814 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:46.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:46.917 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:46.959 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:46.963 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:46.963 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:47.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:46 smithi150 ceph-mon[194178]: pgmap v1288: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 72 objects/s recovering 2023-10-09T06:34:47.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:46 smithi150 ceph-mon[194178]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:47.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:47.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.181 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:47.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:47.285 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:47.317 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[212750]: pgmap v1288: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 72 objects/s recovering 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[212750]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[220921]: pgmap v1288: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1017 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2518/48861 objects degraded (5.153%); 1.7 MiB/s, 72 objects/s recovering 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[220921]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:34:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:47.746 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2023-10-09T06:34:45.413521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1066178216,"num_objects":16287,"num_object_clones":0,"num_object_copies":48861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16287,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178040,"num_write_kb":6175998,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10580,"num_bytes_recovered":652225276,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208590,"ondisk_log_size":208590,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":395,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10829620,"kb_used_data":4121780,"kb_used_omap":20,"kb_used_meta":6707051,"kb_avail":739131596,"statfs":{"total":767960285184,"available":756870754304,"internally_reserved":0,"allocated":4220702720,"data_stored":4208921868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6868020878},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,0,11,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-451608576,"num_objects":-6891,"num_object_clones":0,"num_object_copies":-20673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-723,"num_objects_misplaced":-56,"num_objects_unfound":0,"num_objects_dirty":-6891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6891,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":728,"num_bytes_recovered":18087936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003115"},"pg_stats":[{"pgid":"12.15","version":"418'8612","reported_seq":8698,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.525978+0000","last_change":"2023-10-09T06:34:17.052386+0000","last_active":"2023-10-09T06:34:42.525978+0000","last_peered":"2023-10-09T06:34:42.525978+0000","last_clean":"2023-10-09T06:34:42.525978+0000","last_became_active":"2023-10-09T06:32:33.082121+0000","last_became_peered":"2023-10-09T06:32:33.082121+0000","last_unstale":"2023-10-09T06:34:42.525978+0000","last_undegraded":"2023-10-09T06:34:42.525978+0000","last_fullsized":"2023-10-09T06:34:42.525978+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33128448,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":199136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":776,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492290+0000","last_change":"2023-10-09T06:34:19.873202+0000","last_active":"2023-10-09T06:34:33.492290+0000","last_peered":"2023-10-09T06:34:33.492290+0000","last_clean":"2023-10-09T06:34:33.492290+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:34:33.492290+0000","last_undegraded":"2023-10-09T06:34:33.492290+0000","last_fullsized":"2023-10-09T06:34:33.492290+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":397,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495407+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:34:33.495407+0000","last_peered":"2023-10-09T06:34:33.495407+0000","last_clean":"2023-10-09T06:34:33.495407+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:34:33.495407+0000","last_undegraded":"2023-10-09T06:34:33.495407+0000","last_fullsized":"2023-10-09T06:34:33.495407+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":117,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.496372+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:34:33.496372+0000","last_peered":"2023-10-09T06:34:33.496372+0000","last_clean":"2023-10-09T06:34:33.496372+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:34:33.496372+0000","last_undegraded":"2023-10-09T06:34:33.496372+0000","last_fullsized":"2023-10-09T06:34:33.496372+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"418'8617","reported_seq":8691,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.540614+0000","last_change":"2023-10-09T06:32:33.003181+0000","last_active":"2023-10-09T06:34:42.540614+0000","last_peered":"2023-10-09T06:34:42.540614+0000","last_clean":"2023-10-09T06:34:42.540614+0000","last_became_active":"2023-10-09T06:32:32.998596+0000","last_became_peered":"2023-10-09T06:32:32.998596+0000","last_unstale":"2023-10-09T06:34:42.540614+0000","last_undegraded":"2023-10-09T06:34:42.540614+0000","last_fullsized":"2023-10-09T06:34:42.540614+0000","mapping_epoch":390,"log_start":"365'377","ondisk_log_start":"365'377","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8240,"log_dups_size":0,"ondisk_log_size":8240,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31653888,"num_objects":483,"num_object_clones":0,"num_object_copies":1449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5733,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":736,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493048+0000","last_change":"2023-10-09T06:34:13.788684+0000","last_active":"2023-10-09T06:34:33.493048+0000","last_peered":"2023-10-09T06:34:33.493048+0000","last_clean":"2023-10-09T06:34:33.493048+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:34:33.493048+0000","last_undegraded":"2023-10-09T06:34:33.493048+0000","last_fullsized":"2023-10-09T06:34:33.493048+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":411,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.486805+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:34:33.486805+0000","last_peered":"2023-10-09T06:34:33.486805+0000","last_clean":"2023-10-09T06:34:33.486805+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:34:33.486805+0000","last_undegraded":"2023-10-09T06:34:33.486805+0000","last_fullsized":"2023-10-09T06:34:33.486805+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":415,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.006691+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:34:24.006691+0000","last_peered":"2023-10-09T06:34:24.006691+0000","last_clean":"2023-10-09T06:34:24.006691+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:34:24.006691+0000","last_undegraded":"2023-10-09T06:34:24.006691+0000","last_fullsized":"2023-10-09T06:34:24.006691+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"418'8550","reported_seq":8626,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.563620+0000","last_change":"2023-10-09T06:32:33.092190+0000","last_active":"2023-10-09T06:34:42.563620+0000","last_peered":"2023-10-09T06:34:42.563620+0000","last_clean":"2023-10-09T06:34:42.563620+0000","last_became_active":"2023-10-09T06:32:33.084295+0000","last_became_peered":"2023-10-09T06:32:33.084295+0000","last_unstale":"2023-10-09T06:34:42.563620+0000","last_undegraded":"2023-10-09T06:34:42.563620+0000","last_fullsized":"2023-10-09T06:34:42.563620+0000","mapping_epoch":390,"log_start":"365'508","ondisk_log_start":"365'508","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8042,"log_dups_size":0,"ondisk_log_size":8042,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24346624,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5706,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":665,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495212+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:34:33.495212+0000","last_peered":"2023-10-09T06:34:33.495212+0000","last_clean":"2023-10-09T06:34:33.495212+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:34:33.495212+0000","last_undegraded":"2023-10-09T06:34:33.495212+0000","last_fullsized":"2023-10-09T06:34:33.495212+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":135,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497347+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:34:33.497347+0000","last_peered":"2023-10-09T06:34:33.497347+0000","last_clean":"2023-10-09T06:34:33.497347+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:34:33.497347+0000","last_undegraded":"2023-10-09T06:34:33.497347+0000","last_fullsized":"2023-10-09T06:34:33.497347+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":407,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501672+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:34:33.501672+0000","last_peered":"2023-10-09T06:34:33.501672+0000","last_clean":"2023-10-09T06:34:33.501672+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:34:33.501672+0000","last_undegraded":"2023-10-09T06:34:33.501672+0000","last_fullsized":"2023-10-09T06:34:33.501672+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"418'8463","reported_seq":8538,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:43.116569+0000","last_change":"2023-10-09T06:32:33.080997+0000","last_active":"2023-10-09T06:34:43.116569+0000","last_peered":"2023-10-09T06:34:43.116569+0000","last_clean":"2023-10-09T06:34:43.116569+0000","last_became_active":"2023-10-09T06:32:33.077785+0000","last_became_peered":"2023-10-09T06:32:33.077785+0000","last_unstale":"2023-10-09T06:34:43.116569+0000","last_undegraded":"2023-10-09T06:34:43.116569+0000","last_fullsized":"2023-10-09T06:34:43.116569+0000","mapping_epoch":390,"log_start":"365'676","ondisk_log_start":"365'676","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7787,"log_dups_size":0,"ondisk_log_size":7787,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38535168,"num_objects":588,"num_object_clones":0,"num_object_copies":1764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":526,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490731+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:34:33.490731+0000","last_peered":"2023-10-09T06:34:33.490731+0000","last_clean":"2023-10-09T06:34:33.490731+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:34:33.490731+0000","last_undegraded":"2023-10-09T06:34:33.490731+0000","last_fullsized":"2023-10-09T06:34:33.490731+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":90,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.002599+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:34:24.002599+0000","last_peered":"2023-10-09T06:34:24.002599+0000","last_clean":"2023-10-09T06:34:24.002599+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:34:24.002599+0000","last_undegraded":"2023-10-09T06:34:24.002599+0000","last_fullsized":"2023-10-09T06:34:24.002599+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":118,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:38.277379+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:34:38.277379+0000","last_peered":"2023-10-09T06:34:38.277379+0000","last_clean":"2023-10-09T06:34:38.277379+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:34:38.277379+0000","last_undegraded":"2023-10-09T06:34:38.277379+0000","last_fullsized":"2023-10-09T06:34:38.277379+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"418'8537","reported_seq":8620,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.481561+0000","last_change":"2023-10-09T06:34:21.023111+0000","last_active":"2023-10-09T06:34:42.481561+0000","last_peered":"2023-10-09T06:34:42.481561+0000","last_clean":"2023-10-09T06:34:42.481561+0000","last_became_active":"2023-10-09T06:32:33.074690+0000","last_became_peered":"2023-10-09T06:32:33.074690+0000","last_unstale":"2023-10-09T06:34:42.481561+0000","last_undegraded":"2023-10-09T06:34:42.481561+0000","last_fullsized":"2023-10-09T06:34:42.481561+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8537,"log_dups_size":0,"ondisk_log_size":8537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29655040,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5669,"num_write_kb":195872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":563,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491333+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:34:33.491333+0000","last_peered":"2023-10-09T06:34:33.491333+0000","last_clean":"2023-10-09T06:34:33.491333+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:34:33.491333+0000","last_undegraded":"2023-10-09T06:34:33.491333+0000","last_fullsized":"2023-10-09T06:34:33.491333+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493999+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:34:33.493999+0000","last_peered":"2023-10-09T06:34:33.493999+0000","last_clean":"2023-10-09T06:34:33.493999+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:34:33.493999+0000","last_undegraded":"2023-10-09T06:34:33.493999+0000","last_fullsized":"2023-10-09T06:34:33.493999+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:38.277428+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:34:38.277428+0000","last_peered":"2023-10-09T06:34:38.277428+0000","last_clean":"2023-10-09T06:34:38.277428+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:34:38.277428+0000","last_undegraded":"2023-10-09T06:34:38.277428+0000","last_fullsized":"2023-10-09T06:34:38.277428+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":613,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493699+0000","last_change":"2023-10-09T06:34:16.838424+0000","last_active":"2023-10-09T06:34:33.493699+0000","last_peered":"2023-10-09T06:34:33.493699+0000","last_clean":"2023-10-09T06:34:33.493699+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:34:33.493699+0000","last_undegraded":"2023-10-09T06:34:33.493699+0000","last_fullsized":"2023-10-09T06:34:33.493699+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":299,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491302+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:34:33.491302+0000","last_peered":"2023-10-09T06:34:33.491302+0000","last_clean":"2023-10-09T06:34:33.491302+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:34:33.491302+0000","last_undegraded":"2023-10-09T06:34:33.491302+0000","last_fullsized":"2023-10-09T06:34:33.491302+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":353,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.487141+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:34:33.487141+0000","last_peered":"2023-10-09T06:34:33.487141+0000","last_clean":"2023-10-09T06:34:33.487141+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:34:33.487141+0000","last_undegraded":"2023-10-09T06:34:33.487141+0000","last_fullsized":"2023-10-09T06:34:33.487141+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":524,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497456+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:34:33.497456+0000","last_peered":"2023-10-09T06:34:33.497456+0000","last_clean":"2023-10-09T06:34:33.497456+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:34:33.497456+0000","last_undegraded":"2023-10-09T06:34:33.497456+0000","last_fullsized":"2023-10-09T06:34:33.497456+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":424,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495698+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:34:33.495698+0000","last_peered":"2023-10-09T06:34:33.495698+0000","last_clean":"2023-10-09T06:34:33.495698+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:34:33.495698+0000","last_undegraded":"2023-10-09T06:34:33.495698+0000","last_fullsized":"2023-10-09T06:34:33.495698+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":388,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495606+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:34:33.495606+0000","last_peered":"2023-10-09T06:34:33.495606+0000","last_clean":"2023-10-09T06:34:33.495606+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:34:33.495606+0000","last_undegraded":"2023-10-09T06:34:33.495606+0000","last_fullsized":"2023-10-09T06:34:33.495606+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":688,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.002123+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:34:24.002123+0000","last_peered":"2023-10-09T06:34:24.002123+0000","last_clean":"2023-10-09T06:34:24.002123+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:34:24.002123+0000","last_undegraded":"2023-10-09T06:34:24.002123+0000","last_fullsized":"2023-10-09T06:34:24.002123+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":267,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491218+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:34:33.491218+0000","last_peered":"2023-10-09T06:34:33.491218+0000","last_clean":"2023-10-09T06:34:33.491218+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:34:33.491218+0000","last_undegraded":"2023-10-09T06:34:33.491218+0000","last_fullsized":"2023-10-09T06:34:33.491218+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":118,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960883+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:34:41.960883+0000","last_peered":"2023-10-09T06:34:41.960883+0000","last_clean":"2023-10-09T06:34:41.960883+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:34:41.960883+0000","last_undegraded":"2023-10-09T06:34:41.960883+0000","last_fullsized":"2023-10-09T06:34:41.960883+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":532,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492373+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:34:33.492373+0000","last_peered":"2023-10-09T06:34:33.492373+0000","last_clean":"2023-10-09T06:34:33.492373+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:34:33.492373+0000","last_undegraded":"2023-10-09T06:34:33.492373+0000","last_fullsized":"2023-10-09T06:34:33.492373+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":413,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.002396+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:34:24.002396+0000","last_peered":"2023-10-09T06:34:24.002396+0000","last_clean":"2023-10-09T06:34:24.002396+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:34:24.002396+0000","last_undegraded":"2023-10-09T06:34:24.002396+0000","last_fullsized":"2023-10-09T06:34:24.002396+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493905+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:34:33.493905+0000","last_peered":"2023-10-09T06:34:33.493905+0000","last_clean":"2023-10-09T06:34:33.493905+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:34:33.493905+0000","last_undegraded":"2023-10-09T06:34:33.493905+0000","last_fullsized":"2023-10-09T06:34:33.493905+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":743,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.498323+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:34:33.498323+0000","last_peered":"2023-10-09T06:34:33.498323+0000","last_clean":"2023-10-09T06:34:33.498323+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:34:33.498323+0000","last_undegraded":"2023-10-09T06:34:33.498323+0000","last_fullsized":"2023-10-09T06:34:33.498323+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490882+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:34:33.490882+0000","last_peered":"2023-10-09T06:34:33.490882+0000","last_clean":"2023-10-09T06:34:33.490882+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:34:33.490882+0000","last_undegraded":"2023-10-09T06:34:33.490882+0000","last_fullsized":"2023-10-09T06:34:33.490882+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":90,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.006423+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:34:24.006423+0000","last_peered":"2023-10-09T06:34:24.006423+0000","last_clean":"2023-10-09T06:34:24.006423+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:34:24.006423+0000","last_undegraded":"2023-10-09T06:34:24.006423+0000","last_fullsized":"2023-10-09T06:34:24.006423+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":532,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493498+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:34:33.493498+0000","last_peered":"2023-10-09T06:34:33.493498+0000","last_clean":"2023-10-09T06:34:33.493498+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:34:33.493498+0000","last_undegraded":"2023-10-09T06:34:33.493498+0000","last_fullsized":"2023-10-09T06:34:33.493498+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":496,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.500106+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:34:33.500106+0000","last_peered":"2023-10-09T06:34:33.500106+0000","last_clean":"2023-10-09T06:34:33.500106+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:34:33.500106+0000","last_undegraded":"2023-10-09T06:34:33.500106+0000","last_fullsized":"2023-10-09T06:34:33.500106+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":117,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.496350+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:34:33.496350+0000","last_peered":"2023-10-09T06:34:33.496350+0000","last_clean":"2023-10-09T06:34:33.496350+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:34:33.496350+0000","last_undegraded":"2023-10-09T06:34:33.496350+0000","last_fullsized":"2023-10-09T06:34:33.496350+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":545,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490829+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:34:33.490829+0000","last_peered":"2023-10-09T06:34:33.490829+0000","last_clean":"2023-10-09T06:34:33.490829+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:34:33.490829+0000","last_undegraded":"2023-10-09T06:34:33.490829+0000","last_fullsized":"2023-10-09T06:34:33.490829+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"418'8676","reported_seq":8811,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.390533+0000","last_change":"2023-10-09T06:34:08.464064+0000","last_active":"2023-10-09T06:34:42.390533+0000","last_peered":"2023-10-09T06:34:42.390533+0000","last_clean":"2023-10-09T06:34:42.390533+0000","last_became_active":"2023-10-09T06:32:33.069345+0000","last_became_peered":"2023-10-09T06:32:33.069345+0000","last_unstale":"2023-10-09T06:34:42.390533+0000","last_undegraded":"2023-10-09T06:34:42.390533+0000","last_fullsized":"2023-10-09T06:34:42.390533+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'7928","last_scrub_stamp":"2023-10-09T06:34:08.463956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:34:08.463956+0000","objects_scrubbed":1240,"log_size":8676,"log_dups_size":0,"ondisk_log_size":8676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:39.228319+0000","scrub_duration":0.275394735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32178176,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":749,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":303,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490809+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:34:33.490809+0000","last_peered":"2023-10-09T06:34:33.490809+0000","last_clean":"2023-10-09T06:34:33.490809+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:34:33.490809+0000","last_undegraded":"2023-10-09T06:34:33.490809+0000","last_fullsized":"2023-10-09T06:34:33.490809+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492621+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:34:33.492621+0000","last_peered":"2023-10-09T06:34:33.492621+0000","last_clean":"2023-10-09T06:34:33.492621+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:34:33.492621+0000","last_undegraded":"2023-10-09T06:34:33.492621+0000","last_fullsized":"2023-10-09T06:34:33.492621+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":802,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.496667+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:34:33.496667+0000","last_peered":"2023-10-09T06:34:33.496667+0000","last_clean":"2023-10-09T06:34:33.496667+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:34:33.496667+0000","last_undegraded":"2023-10-09T06:34:33.496667+0000","last_fullsized":"2023-10-09T06:34:33.496667+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"418'8667","reported_seq":8752,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.505947+0000","last_change":"2023-10-09T06:34:00.106009+0000","last_active":"2023-10-09T06:34:42.505947+0000","last_peered":"2023-10-09T06:34:42.505947+0000","last_clean":"2023-10-09T06:34:42.505947+0000","last_became_active":"2023-10-09T06:32:33.000165+0000","last_became_peered":"2023-10-09T06:32:33.000165+0000","last_unstale":"2023-10-09T06:34:42.505947+0000","last_undegraded":"2023-10-09T06:34:42.505947+0000","last_fullsized":"2023-10-09T06:34:42.505947+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34275328,"num_objects":523,"num_object_clones":0,"num_object_copies":1569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5759,"num_write_kb":201024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":265,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491388+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:34:33.491388+0000","last_peered":"2023-10-09T06:34:33.491388+0000","last_clean":"2023-10-09T06:34:33.491388+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:34:33.491388+0000","last_undegraded":"2023-10-09T06:34:33.491388+0000","last_fullsized":"2023-10-09T06:34:33.491388+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":415,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495363+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:34:33.495363+0000","last_peered":"2023-10-09T06:34:33.495363+0000","last_clean":"2023-10-09T06:34:33.495363+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:34:33.495363+0000","last_undegraded":"2023-10-09T06:34:33.495363+0000","last_fullsized":"2023-10-09T06:34:33.495363+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":719,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501575+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:34:33.501575+0000","last_peered":"2023-10-09T06:34:33.501575+0000","last_clean":"2023-10-09T06:34:33.501575+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:34:33.501575+0000","last_undegraded":"2023-10-09T06:34:33.501575+0000","last_fullsized":"2023-10-09T06:34:33.501575+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"418'8658","reported_seq":8755,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.292901+0000","last_change":"2023-10-09T06:34:21.334474+0000","last_active":"2023-10-09T06:34:42.292901+0000","last_peered":"2023-10-09T06:34:42.292901+0000","last_clean":"2023-10-09T06:34:42.292901+0000","last_became_active":"2023-10-09T06:32:32.996849+0000","last_became_peered":"2023-10-09T06:32:32.996849+0000","last_unstale":"2023-10-09T06:34:42.292901+0000","last_undegraded":"2023-10-09T06:34:42.292901+0000","last_fullsized":"2023-10-09T06:34:42.292901+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8658,"log_dups_size":0,"ondisk_log_size":8658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32276480,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5754,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":321,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491068+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:34:33.491068+0000","last_peered":"2023-10-09T06:34:33.491068+0000","last_clean":"2023-10-09T06:34:33.491068+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:34:33.491068+0000","last_undegraded":"2023-10-09T06:34:33.491068+0000","last_fullsized":"2023-10-09T06:34:33.491068+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":394,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.500953+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:34:33.500953+0000","last_peered":"2023-10-09T06:34:33.500953+0000","last_clean":"2023-10-09T06:34:33.500953+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:34:33.500953+0000","last_undegraded":"2023-10-09T06:34:33.500953+0000","last_fullsized":"2023-10-09T06:34:33.500953+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":542,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495975+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:34:33.495975+0000","last_peered":"2023-10-09T06:34:33.495975+0000","last_clean":"2023-10-09T06:34:33.495975+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:34:33.495975+0000","last_undegraded":"2023-10-09T06:34:33.495975+0000","last_fullsized":"2023-10-09T06:34:33.495975+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"417'8606","reported_seq":8682,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:40.831938+0000","last_change":"2023-10-09T06:32:32.995971+0000","last_active":"2023-10-09T06:34:40.831938+0000","last_peered":"2023-10-09T06:34:40.831938+0000","last_clean":"2023-10-09T06:34:40.831938+0000","last_became_active":"2023-10-09T06:32:32.993707+0000","last_became_peered":"2023-10-09T06:32:32.993707+0000","last_unstale":"2023-10-09T06:34:40.831938+0000","last_undegraded":"2023-10-09T06:34:40.831938+0000","last_fullsized":"2023-10-09T06:34:40.831938+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8606,"log_dups_size":0,"ondisk_log_size":8606,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36339712,"num_objects":554,"num_object_clones":0,"num_object_copies":1662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":200608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493048+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:34:33.493048+0000","last_peered":"2023-10-09T06:34:33.493048+0000","last_clean":"2023-10-09T06:34:33.493048+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:34:33.493048+0000","last_undegraded":"2023-10-09T06:34:33.493048+0000","last_fullsized":"2023-10-09T06:34:33.493048+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":377,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.003066+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:34:24.003066+0000","last_peered":"2023-10-09T06:34:24.003066+0000","last_clean":"2023-10-09T06:34:24.003066+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:34:24.003066+0000","last_undegraded":"2023-10-09T06:34:24.003066+0000","last_fullsized":"2023-10-09T06:34:24.003066+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"418'8575","reported_seq":8658,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.488630+0000","last_change":"2023-10-09T06:34:11.068769+0000","last_active":"2023-10-09T06:34:42.488630+0000","last_peered":"2023-10-09T06:34:42.488630+0000","last_clean":"2023-10-09T06:34:42.488630+0000","last_became_active":"2023-10-09T06:32:32.995923+0000","last_became_peered":"2023-10-09T06:32:32.995923+0000","last_unstale":"2023-10-09T06:34:42.488630+0000","last_undegraded":"2023-10-09T06:34:42.488630+0000","last_fullsized":"2023-10-09T06:34:42.488630+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8575,"log_dups_size":0,"ondisk_log_size":8575,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31653888,"num_objects":483,"num_object_clones":0,"num_object_copies":1449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5703,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":770,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490736+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:34:33.490736+0000","last_peered":"2023-10-09T06:34:33.490736+0000","last_clean":"2023-10-09T06:34:33.490736+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:34:33.490736+0000","last_undegraded":"2023-10-09T06:34:33.490736+0000","last_fullsized":"2023-10-09T06:34:33.490736+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":144,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492025+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:34:33.492025+0000","last_peered":"2023-10-09T06:34:33.492025+0000","last_clean":"2023-10-09T06:34:33.492025+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:34:33.492025+0000","last_undegraded":"2023-10-09T06:34:33.492025+0000","last_fullsized":"2023-10-09T06:34:33.492025+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":398,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495611+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:34:33.495611+0000","last_peered":"2023-10-09T06:34:33.495611+0000","last_clean":"2023-10-09T06:34:33.495611+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:34:33.495611+0000","last_undegraded":"2023-10-09T06:34:33.495611+0000","last_fullsized":"2023-10-09T06:34:33.495611+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"418'8643","reported_seq":8726,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.515380+0000","last_change":"2023-10-09T06:34:14.119582+0000","last_active":"2023-10-09T06:34:42.515380+0000","last_peered":"2023-10-09T06:34:42.515380+0000","last_clean":"2023-10-09T06:34:42.515380+0000","last_became_active":"2023-10-09T06:32:33.044765+0000","last_became_peered":"2023-10-09T06:32:33.044765+0000","last_unstale":"2023-10-09T06:34:42.515380+0000","last_undegraded":"2023-10-09T06:34:42.515380+0000","last_fullsized":"2023-10-09T06:34:42.515380+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8643,"log_dups_size":0,"ondisk_log_size":8643,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32407552,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5748,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":750,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492448+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:34:33.492448+0000","last_peered":"2023-10-09T06:34:33.492448+0000","last_clean":"2023-10-09T06:34:33.492448+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:34:33.492448+0000","last_undegraded":"2023-10-09T06:34:33.492448+0000","last_fullsized":"2023-10-09T06:34:33.492448+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":375,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501067+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:34:33.501067+0000","last_peered":"2023-10-09T06:34:33.501067+0000","last_clean":"2023-10-09T06:34:33.501067+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:34:33.501067+0000","last_undegraded":"2023-10-09T06:34:33.501067+0000","last_fullsized":"2023-10-09T06:34:33.501067+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":395,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495923+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:34:33.495923+0000","last_peered":"2023-10-09T06:34:33.495923+0000","last_clean":"2023-10-09T06:34:33.495923+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:34:33.495923+0000","last_undegraded":"2023-10-09T06:34:33.495923+0000","last_fullsized":"2023-10-09T06:34:33.495923+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":752,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.498400+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:34:33.498400+0000","last_peered":"2023-10-09T06:34:33.498400+0000","last_clean":"2023-10-09T06:34:33.498400+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:34:33.498400+0000","last_undegraded":"2023-10-09T06:34:33.498400+0000","last_fullsized":"2023-10-09T06:34:33.498400+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"418'8626","reported_seq":8701,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.493693+0000","last_change":"2023-10-09T06:32:32.976279+0000","last_active":"2023-10-09T06:34:42.493693+0000","last_peered":"2023-10-09T06:34:42.493693+0000","last_clean":"2023-10-09T06:34:42.493693+0000","last_became_active":"2023-10-09T06:32:32.973171+0000","last_became_peered":"2023-10-09T06:32:32.973171+0000","last_unstale":"2023-10-09T06:34:42.493693+0000","last_undegraded":"2023-10-09T06:34:42.493693+0000","last_fullsized":"2023-10-09T06:34:42.493693+0000","mapping_epoch":361,"log_start":"365'344","ondisk_log_start":"365'344","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8282,"log_dups_size":0,"ondisk_log_size":8282,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161216,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5742,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":399,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495275+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:34:33.495275+0000","last_peered":"2023-10-09T06:34:33.495275+0000","last_clean":"2023-10-09T06:34:33.495275+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:34:33.495275+0000","last_undegraded":"2023-10-09T06:34:33.495275+0000","last_fullsized":"2023-10-09T06:34:33.495275+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":378,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495845+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:34:33.495845+0000","last_peered":"2023-10-09T06:34:33.495845+0000","last_clean":"2023-10-09T06:34:33.495845+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:34:33.495845+0000","last_undegraded":"2023-10-09T06:34:33.495845+0000","last_fullsized":"2023-10-09T06:34:33.495845+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"418'8408","reported_seq":8483,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.553373+0000","last_change":"2023-10-09T06:32:33.079508+0000","last_active":"2023-10-09T06:34:42.553373+0000","last_peered":"2023-10-09T06:34:42.553373+0000","last_clean":"2023-10-09T06:34:42.553373+0000","last_became_active":"2023-10-09T06:32:33.075123+0000","last_became_peered":"2023-10-09T06:32:33.075123+0000","last_unstale":"2023-10-09T06:34:42.553373+0000","last_undegraded":"2023-10-09T06:34:42.553373+0000","last_fullsized":"2023-10-09T06:34:42.553373+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8408,"log_dups_size":0,"ondisk_log_size":8408,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38764544,"num_objects":591,"num_object_clones":0,"num_object_copies":1773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":728,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493638+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:34:33.493638+0000","last_peered":"2023-10-09T06:34:33.493638+0000","last_clean":"2023-10-09T06:34:33.493638+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:34:33.493638+0000","last_undegraded":"2023-10-09T06:34:33.493638+0000","last_fullsized":"2023-10-09T06:34:33.493638+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497586+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:34:33.497586+0000","last_peered":"2023-10-09T06:34:33.497586+0000","last_clean":"2023-10-09T06:34:33.497586+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:34:33.497586+0000","last_undegraded":"2023-10-09T06:34:33.497586+0000","last_fullsized":"2023-10-09T06:34:33.497586+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":371,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.499503+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:34:33.499503+0000","last_peered":"2023-10-09T06:34:33.499503+0000","last_clean":"2023-10-09T06:34:33.499503+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:34:33.499503+0000","last_undegraded":"2023-10-09T06:34:33.499503+0000","last_fullsized":"2023-10-09T06:34:33.499503+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"417'11381","reported_seq":11447,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:40.845158+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:34:40.845158+0000","last_peered":"2023-10-09T06:34:40.845158+0000","last_clean":"2023-10-09T06:34:40.845158+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:34:40.845158+0000","last_undegraded":"2023-10-09T06:34:40.845158+0000","last_fullsized":"2023-10-09T06:34:40.845158+0000","mapping_epoch":361,"log_start":"382'4800","ondisk_log_start":"382'4800","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6581,"log_dups_size":0,"ondisk_log_size":6581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70582272,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11381,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":710,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501714+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:34:33.501714+0000","last_peered":"2023-10-09T06:34:33.501714+0000","last_clean":"2023-10-09T06:34:33.501714+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:34:33.501714+0000","last_undegraded":"2023-10-09T06:34:33.501714+0000","last_fullsized":"2023-10-09T06:34:33.501714+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":423,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497394+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:34:33.497394+0000","last_peered":"2023-10-09T06:34:33.497394+0000","last_clean":"2023-10-09T06:34:33.497394+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:34:33.497394+0000","last_undegraded":"2023-10-09T06:34:33.497394+0000","last_fullsized":"2023-10-09T06:34:33.497394+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493824+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:34:33.493824+0000","last_peered":"2023-10-09T06:34:33.493824+0000","last_clean":"2023-10-09T06:34:33.493824+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:34:33.493824+0000","last_undegraded":"2023-10-09T06:34:33.493824+0000","last_fullsized":"2023-10-09T06:34:33.493824+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":357,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.487014+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:34:33.487014+0000","last_peered":"2023-10-09T06:34:33.487014+0000","last_clean":"2023-10-09T06:34:33.487014+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:34:33.487014+0000","last_undegraded":"2023-10-09T06:34:33.487014+0000","last_fullsized":"2023-10-09T06:34:33.487014+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"417'11408","reported_seq":11474,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:40.932711+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:34:40.932711+0000","last_peered":"2023-10-09T06:34:40.932711+0000","last_clean":"2023-10-09T06:34:40.932711+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:34:40.932711+0000","last_undegraded":"2023-10-09T06:34:40.932711+0000","last_fullsized":"2023-10-09T06:34:40.932711+0000","mapping_epoch":361,"log_start":"382'4800","ondisk_log_start":"382'4800","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70778880,"num_objects":1080,"num_object_clones":0,"num_object_copies":3240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11408,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":764,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493258+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:34:33.493258+0000","last_peered":"2023-10-09T06:34:33.493258+0000","last_clean":"2023-10-09T06:34:33.493258+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:34:33.493258+0000","last_undegraded":"2023-10-09T06:34:33.493258+0000","last_fullsized":"2023-10-09T06:34:33.493258+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493779+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:34:33.493779+0000","last_peered":"2023-10-09T06:34:33.493779+0000","last_clean":"2023-10-09T06:34:33.493779+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:34:33.493779+0000","last_undegraded":"2023-10-09T06:34:33.493779+0000","last_fullsized":"2023-10-09T06:34:33.493779+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":399,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495528+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:34:33.495528+0000","last_peered":"2023-10-09T06:34:33.495528+0000","last_clean":"2023-10-09T06:34:33.495528+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:34:33.495528+0000","last_undegraded":"2023-10-09T06:34:33.495528+0000","last_fullsized":"2023-10-09T06:34:33.495528+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":387,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497324+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:34:33.497324+0000","last_peered":"2023-10-09T06:34:33.497324+0000","last_clean":"2023-10-09T06:34:33.497324+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:34:33.497324+0000","last_undegraded":"2023-10-09T06:34:33.497324+0000","last_fullsized":"2023-10-09T06:34:33.497324+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"418'11524","reported_seq":11588,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.529279+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:34:42.529279+0000","last_peered":"2023-10-09T06:34:42.529279+0000","last_clean":"2023-10-09T06:34:42.529279+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:34:42.529279+0000","last_undegraded":"2023-10-09T06:34:42.529279+0000","last_fullsized":"2023-10-09T06:34:42.529279+0000","mapping_epoch":361,"log_start":"388'5700","ondisk_log_start":"388'5700","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11524,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":760,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491183+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:34:33.491183+0000","last_peered":"2023-10-09T06:34:33.491183+0000","last_clean":"2023-10-09T06:34:33.491183+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:34:33.491183+0000","last_undegraded":"2023-10-09T06:34:33.491183+0000","last_fullsized":"2023-10-09T06:34:33.491183+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":408,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.492974+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:34:33.492974+0000","last_peered":"2023-10-09T06:34:33.492974+0000","last_clean":"2023-10-09T06:34:33.492974+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:34:33.492974+0000","last_undegraded":"2023-10-09T06:34:33.492974+0000","last_fullsized":"2023-10-09T06:34:33.492974+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":395,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.486946+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:34:33.486946+0000","last_peered":"2023-10-09T06:34:33.486946+0000","last_clean":"2023-10-09T06:34:33.486946+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:34:33.486946+0000","last_undegraded":"2023-10-09T06:34:33.486946+0000","last_fullsized":"2023-10-09T06:34:33.486946+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":418,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.963895+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:34:41.963895+0000","last_peered":"2023-10-09T06:34:41.963895+0000","last_clean":"2023-10-09T06:34:41.963895+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:34:41.963895+0000","last_undegraded":"2023-10-09T06:34:41.963895+0000","last_fullsized":"2023-10-09T06:34:41.963895+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"418'11657","reported_seq":11723,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.372445+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:34:42.372445+0000","last_peered":"2023-10-09T06:34:42.372445+0000","last_clean":"2023-10-09T06:34:42.372445+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:34:42.372445+0000","last_undegraded":"2023-10-09T06:34:42.372445+0000","last_fullsized":"2023-10-09T06:34:42.372445+0000","mapping_epoch":361,"log_start":"392'6249","ondisk_log_start":"392'6249","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5408,"log_dups_size":0,"ondisk_log_size":5408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652440,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11657,"num_write_kb":403585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":105,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493197+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:34:33.493197+0000","last_peered":"2023-10-09T06:34:33.493197+0000","last_clean":"2023-10-09T06:34:33.493197+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:34:33.493197+0000","last_undegraded":"2023-10-09T06:34:33.493197+0000","last_fullsized":"2023-10-09T06:34:33.493197+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":741,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490549+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:34:33.490549+0000","last_peered":"2023-10-09T06:34:33.490549+0000","last_clean":"2023-10-09T06:34:33.490549+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:34:33.490549+0000","last_undegraded":"2023-10-09T06:34:33.490549+0000","last_fullsized":"2023-10-09T06:34:33.490549+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497979+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:34:33.497979+0000","last_peered":"2023-10-09T06:34:33.497979+0000","last_clean":"2023-10-09T06:34:33.497979+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:34:33.497979+0000","last_undegraded":"2023-10-09T06:34:33.497979+0000","last_fullsized":"2023-10-09T06:34:33.497979+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":380,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501381+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:34:33.501381+0000","last_peered":"2023-10-09T06:34:33.501381+0000","last_clean":"2023-10-09T06:34:33.501381+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:34:33.501381+0000","last_undegraded":"2023-10-09T06:34:33.501381+0000","last_fullsized":"2023-10-09T06:34:33.501381+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"417'11531","reported_seq":11597,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:40.967423+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:34:40.967423+0000","last_peered":"2023-10-09T06:34:40.967423+0000","last_clean":"2023-10-09T06:34:40.967423+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:34:40.967423+0000","last_undegraded":"2023-10-09T06:34:40.967423+0000","last_fullsized":"2023-10-09T06:34:40.967423+0000","mapping_epoch":361,"log_start":"382'5000","ondisk_log_start":"382'5000","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73728000,"num_objects":1125,"num_object_clones":0,"num_object_copies":3375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11531,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":726,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.498472+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:34:33.498472+0000","last_peered":"2023-10-09T06:34:33.498472+0000","last_clean":"2023-10-09T06:34:33.498472+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:34:33.498472+0000","last_undegraded":"2023-10-09T06:34:33.498472+0000","last_fullsized":"2023-10-09T06:34:33.498472+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":281,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.490391+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:34:33.490391+0000","last_peered":"2023-10-09T06:34:33.490391+0000","last_clean":"2023-10-09T06:34:33.490391+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:34:33.490391+0000","last_undegraded":"2023-10-09T06:34:33.490391+0000","last_fullsized":"2023-10-09T06:34:33.490391+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":90,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.002347+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:34:24.002347+0000","last_peered":"2023-10-09T06:34:24.002347+0000","last_clean":"2023-10-09T06:34:24.002347+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:34:24.002347+0000","last_undegraded":"2023-10-09T06:34:24.002347+0000","last_fullsized":"2023-10-09T06:34:24.002347+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":401,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.006856+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:34:24.006856+0000","last_peered":"2023-10-09T06:34:24.006856+0000","last_clean":"2023-10-09T06:34:24.006856+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:34:24.006856+0000","last_undegraded":"2023-10-09T06:34:24.006856+0000","last_fullsized":"2023-10-09T06:34:24.006856+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"418'11343","reported_seq":11410,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:43.254935+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:34:43.254935+0000","last_peered":"2023-10-09T06:34:43.254935+0000","last_clean":"2023-10-09T06:34:43.254935+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:34:43.254935+0000","last_undegraded":"2023-10-09T06:34:43.254935+0000","last_fullsized":"2023-10-09T06:34:43.254935+0000","mapping_epoch":361,"log_start":"393'6109","ondisk_log_start":"393'6109","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59047936,"num_objects":901,"num_object_clones":0,"num_object_copies":2703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11343,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":740,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501305+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:34:33.501305+0000","last_peered":"2023-10-09T06:34:33.501305+0000","last_clean":"2023-10-09T06:34:33.501305+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:34:33.501305+0000","last_undegraded":"2023-10-09T06:34:33.501305+0000","last_fullsized":"2023-10-09T06:34:33.501305+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":281,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491205+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:34:33.491205+0000","last_peered":"2023-10-09T06:34:33.491205+0000","last_clean":"2023-10-09T06:34:33.491205+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:34:33.491205+0000","last_undegraded":"2023-10-09T06:34:33.491205+0000","last_fullsized":"2023-10-09T06:34:33.491205+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":413,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497236+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:34:33.497236+0000","last_peered":"2023-10-09T06:34:33.497236+0000","last_clean":"2023-10-09T06:34:33.497236+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:34:33.497236+0000","last_undegraded":"2023-10-09T06:34:33.497236+0000","last_fullsized":"2023-10-09T06:34:33.497236+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":399,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.003281+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:34:24.003281+0000","last_peered":"2023-10-09T06:34:24.003281+0000","last_clean":"2023-10-09T06:34:24.003281+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:34:24.003281+0000","last_undegraded":"2023-10-09T06:34:24.003281+0000","last_fullsized":"2023-10-09T06:34:24.003281+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"418'8628","reported_seq":8705,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:43.255277+0000","last_change":"2023-10-09T06:32:47.459790+0000","last_active":"2023-10-09T06:34:43.255277+0000","last_peered":"2023-10-09T06:34:43.255277+0000","last_clean":"2023-10-09T06:34:43.255277+0000","last_became_active":"2023-10-09T06:32:33.070761+0000","last_became_peered":"2023-10-09T06:32:33.070761+0000","last_unstale":"2023-10-09T06:34:43.255277+0000","last_undegraded":"2023-10-09T06:34:43.255277+0000","last_fullsized":"2023-10-09T06:34:43.255277+0000","mapping_epoch":361,"log_start":"365'1287","ondisk_log_start":"365'1287","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7341,"log_dups_size":0,"ondisk_log_size":7341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27459584,"num_objects":419,"num_object_clones":0,"num_object_copies":1257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5763,"num_write_kb":197824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":741,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495800+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:34:33.495800+0000","last_peered":"2023-10-09T06:34:33.495800+0000","last_clean":"2023-10-09T06:34:33.495800+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:34:33.495800+0000","last_undegraded":"2023-10-09T06:34:33.495800+0000","last_fullsized":"2023-10-09T06:34:33.495800+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":148,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.496487+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:34:33.496487+0000","last_peered":"2023-10-09T06:34:33.496487+0000","last_clean":"2023-10-09T06:34:33.496487+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:34:33.496487+0000","last_undegraded":"2023-10-09T06:34:33.496487+0000","last_fullsized":"2023-10-09T06:34:33.496487+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":381,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.962715+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:34:41.962715+0000","last_peered":"2023-10-09T06:34:41.962715+0000","last_clean":"2023-10-09T06:34:41.962715+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:34:41.962715+0000","last_undegraded":"2023-10-09T06:34:41.962715+0000","last_fullsized":"2023-10-09T06:34:41.962715+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":136,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497506+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:34:33.497506+0000","last_peered":"2023-10-09T06:34:33.497506+0000","last_clean":"2023-10-09T06:34:33.497506+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:34:33.497506+0000","last_undegraded":"2023-10-09T06:34:33.497506+0000","last_fullsized":"2023-10-09T06:34:33.497506+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"418'8552","reported_seq":8636,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.528903+0000","last_change":"2023-10-09T06:34:05.172567+0000","last_active":"2023-10-09T06:34:42.528903+0000","last_peered":"2023-10-09T06:34:42.528903+0000","last_clean":"2023-10-09T06:34:42.528903+0000","last_became_active":"2023-10-09T06:32:33.044852+0000","last_became_peered":"2023-10-09T06:32:33.044852+0000","last_unstale":"2023-10-09T06:34:42.528903+0000","last_undegraded":"2023-10-09T06:34:42.528903+0000","last_fullsized":"2023-10-09T06:34:42.528903+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8552,"log_dups_size":0,"ondisk_log_size":8552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32866304,"num_objects":501,"num_object_clones":0,"num_object_copies":1503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5683,"num_write_kb":197920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":770,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497096+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:34:33.497096+0000","last_peered":"2023-10-09T06:34:33.497096+0000","last_clean":"2023-10-09T06:34:33.497096+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:34:33.497096+0000","last_undegraded":"2023-10-09T06:34:33.497096+0000","last_fullsized":"2023-10-09T06:34:33.497096+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":402,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.501813+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:34:33.501813+0000","last_peered":"2023-10-09T06:34:33.501813+0000","last_clean":"2023-10-09T06:34:33.501813+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:34:33.501813+0000","last_undegraded":"2023-10-09T06:34:33.501813+0000","last_fullsized":"2023-10-09T06:34:33.501813+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":382,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.006564+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:34:24.006564+0000","last_peered":"2023-10-09T06:34:24.006564+0000","last_clean":"2023-10-09T06:34:24.006564+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:34:24.006564+0000","last_undegraded":"2023-10-09T06:34:24.006564+0000","last_fullsized":"2023-10-09T06:34:24.006564+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":381,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495329+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:34:33.495329+0000","last_peered":"2023-10-09T06:34:33.495329+0000","last_clean":"2023-10-09T06:34:33.495329+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:34:33.495329+0000","last_undegraded":"2023-10-09T06:34:33.495329+0000","last_fullsized":"2023-10-09T06:34:33.495329+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"418'8570","reported_seq":8658,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.497417+0000","last_change":"2023-10-09T06:34:16.084491+0000","last_active":"2023-10-09T06:34:42.497417+0000","last_peered":"2023-10-09T06:34:42.497417+0000","last_clean":"2023-10-09T06:34:42.497417+0000","last_became_active":"2023-10-09T06:32:33.005074+0000","last_became_peered":"2023-10-09T06:32:33.005074+0000","last_unstale":"2023-10-09T06:34:42.497417+0000","last_undegraded":"2023-10-09T06:34:42.497417+0000","last_fullsized":"2023-10-09T06:34:42.497417+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8570,"log_dups_size":0,"ondisk_log_size":8570,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":697,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.940352+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:34:41.940352+0000","last_peered":"2023-10-09T06:34:41.940352+0000","last_clean":"2023-10-09T06:34:41.940352+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:34:41.940352+0000","last_undegraded":"2023-10-09T06:34:41.940352+0000","last_fullsized":"2023-10-09T06:34:41.940352+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":383,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.003207+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:34:24.003207+0000","last_peered":"2023-10-09T06:34:24.003207+0000","last_clean":"2023-10-09T06:34:24.003207+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:34:24.003207+0000","last_undegraded":"2023-10-09T06:34:24.003207+0000","last_fullsized":"2023-10-09T06:34:24.003207+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":406,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497227+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:34:33.497227+0000","last_peered":"2023-10-09T06:34:33.497227+0000","last_clean":"2023-10-09T06:34:33.497227+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:34:33.497227+0000","last_undegraded":"2023-10-09T06:34:33.497227+0000","last_fullsized":"2023-10-09T06:34:33.497227+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.13","version":"417'8574","reported_seq":12309,"reported_epoch":417,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:40.968708+0000","last_change":"2023-10-09T06:34:03.204624+0000","last_active":"2023-10-09T06:34:40.968708+0000","last_peered":"2023-10-09T06:34:40.968708+0000","last_clean":"2023-10-09T06:32:51.037365+0000","last_became_active":"2023-10-09T06:32:53.531678+0000","last_became_peered":"2023-10-09T06:32:53.531678+0000","last_unstale":"2023-10-09T06:34:40.968708+0000","last_undegraded":"2023-10-09T06:32:53.443414+0000","last_fullsized":"2023-10-09T06:32:53.427850+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8574,"log_dups_size":0,"ondisk_log_size":8574,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36470784,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1294,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5683,"num_write_kb":199648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1827,"num_bytes_recovered":90112000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,7","objects":-737}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":627,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.499721+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:34:33.499721+0000","last_peered":"2023-10-09T06:34:33.499721+0000","last_clean":"2023-10-09T06:34:33.499721+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:34:33.499721+0000","last_undegraded":"2023-10-09T06:34:33.499721+0000","last_fullsized":"2023-10-09T06:34:33.499721+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":109,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493428+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:34:33.493428+0000","last_peered":"2023-10-09T06:34:33.493428+0000","last_clean":"2023-10-09T06:34:33.493428+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:34:33.493428+0000","last_undegraded":"2023-10-09T06:34:33.493428+0000","last_fullsized":"2023-10-09T06:34:33.493428+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":425,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495500+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:34:33.495500+0000","last_peered":"2023-10-09T06:34:33.495500+0000","last_clean":"2023-10-09T06:34:33.495500+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:34:33.495500+0000","last_undegraded":"2023-10-09T06:34:33.495500+0000","last_fullsized":"2023-10-09T06:34:33.495500+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"418'8694","reported_seq":16339,"reported_epoch":418,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:43.269404+0000","last_change":"2023-10-09T06:32:52.162166+0000","last_active":"2023-10-09T06:34:43.269404+0000","last_peered":"2023-10-09T06:34:43.269404+0000","last_clean":"2023-10-09T06:32:48.657566+0000","last_became_active":"2023-10-09T06:32:52.155917+0000","last_became_peered":"2023-10-09T06:32:52.155917+0000","last_unstale":"2023-10-09T06:34:43.269404+0000","last_undegraded":"2023-10-09T06:32:52.150896+0000","last_fullsized":"2023-10-09T06:32:52.051991+0000","mapping_epoch":400,"log_start":"365'1263","ondisk_log_start":"365'1263","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7431,"log_dups_size":0,"ondisk_log_size":7431,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30310400,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1224,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5791,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2521,"num_bytes_recovered":165216256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"1,3","objects":-6},{"shards":"3,4,7","objects":469}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":109,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493658+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:34:33.493658+0000","last_peered":"2023-10-09T06:34:33.493658+0000","last_clean":"2023-10-09T06:34:33.493658+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:34:33.493658+0000","last_undegraded":"2023-10-09T06:34:33.493658+0000","last_fullsized":"2023-10-09T06:34:33.493658+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":435,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495647+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:34:33.495647+0000","last_peered":"2023-10-09T06:34:33.495647+0000","last_clean":"2023-10-09T06:34:33.495647+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:34:33.495647+0000","last_undegraded":"2023-10-09T06:34:33.495647+0000","last_fullsized":"2023-10-09T06:34:33.495647+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"418'8640","reported_seq":18120,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.359288+0000","last_change":"2023-10-09T06:34:18.854131+0000","last_active":"2023-10-09T06:34:42.359288+0000","last_peered":"2023-10-09T06:34:42.359288+0000","last_clean":"2023-10-09T06:34:42.359288+0000","last_became_active":"2023-10-09T06:33:43.622574+0000","last_became_peered":"2023-10-09T06:33:43.622574+0000","last_unstale":"2023-10-09T06:34:42.359288+0000","last_undegraded":"2023-10-09T06:34:42.359288+0000","last_fullsized":"2023-10-09T06:34:42.359288+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8640,"log_dups_size":0,"ondisk_log_size":8640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33816576,"num_objects":516,"num_object_clones":0,"num_object_copies":1548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5732,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":196673536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":774,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.491229+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:34:33.491229+0000","last_peered":"2023-10-09T06:34:33.491229+0000","last_clean":"2023-10-09T06:34:33.491229+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:34:33.491229+0000","last_undegraded":"2023-10-09T06:34:33.491229+0000","last_fullsized":"2023-10-09T06:34:33.491229+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":118,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.499928+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:34:33.499928+0000","last_peered":"2023-10-09T06:34:33.499928+0000","last_clean":"2023-10-09T06:34:33.499928+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:34:33.499928+0000","last_undegraded":"2023-10-09T06:34:33.499928+0000","last_fullsized":"2023-10-09T06:34:33.499928+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.493231+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:34:33.493231+0000","last_peered":"2023-10-09T06:34:33.493231+0000","last_clean":"2023-10-09T06:34:33.493231+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:34:33.493231+0000","last_undegraded":"2023-10-09T06:34:33.493231+0000","last_fullsized":"2023-10-09T06:34:33.493231+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"417'8517","reported_seq":20790,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:40.891735+0000","last_change":"2023-10-09T06:34:05.191255+0000","last_active":"2023-10-09T06:34:40.891735+0000","last_peered":"2023-10-09T06:34:40.891735+0000","last_clean":"2023-10-09T06:34:40.891735+0000","last_became_active":"2023-10-09T06:34:05.174880+0000","last_became_peered":"2023-10-09T06:34:05.174880+0000","last_unstale":"2023-10-09T06:34:40.891735+0000","last_undegraded":"2023-10-09T06:34:40.891735+0000","last_fullsized":"2023-10-09T06:34:40.891735+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8517,"log_dups_size":0,"ondisk_log_size":8517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40534016,"num_objects":619,"num_object_clones":0,"num_object_copies":1857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3046,"num_bytes_recovered":199622656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":538,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.497714+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:34:33.497714+0000","last_peered":"2023-10-09T06:34:33.497714+0000","last_clean":"2023-10-09T06:34:33.497714+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:34:33.497714+0000","last_undegraded":"2023-10-09T06:34:33.497714+0000","last_fullsized":"2023-10-09T06:34:33.497714+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":412,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-10-09T06:34:24.002601+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:34:24.002601+0000","last_peered":"2023-10-09T06:34:24.002601+0000","last_clean":"2023-10-09T06:34:24.002601+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:34:24.002601+0000","last_undegraded":"2023-10-09T06:34:24.002601+0000","last_fullsized":"2023-10-09T06:34:24.002601+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":378,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-10-09T06:34:33.495122+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:34:33.495122+0000","last_peered":"2023-10-09T06:34:33.495122+0000","last_clean":"2023-10-09T06:34:33.495122+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:34:33.495122+0000","last_undegraded":"2023-10-09T06:34:33.495122+0000","last_fullsized":"2023-10-09T06:34:33.495122+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":1065713688,"num_objects":16263,"num_object_clones":0,"num_object_copies":48789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16263,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177979,"num_write_kb":6175969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10524,"num_bytes_recovered":651821056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4138283008,"data_stored":4138270792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203544,"ondisk_log_size":203544,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234449,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1598860,"kb_used_data":694404,"kb_used_omap":0,"kb_used_meta":904320,"kb_avail":92146292,"statfs":{"total":95995035648,"available":94357803008,"internally_reserved":0,"allocated":711069696,"data_stored":710031960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":926023680},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":5.7720000000000002,"15min":6.1849999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":25.225000000000001,"5min":196.28899999999999,"15min":219.41900000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.097,"5min":5.9139999999999997,"15min":6.3200000000000003},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":25.248000000000001,"5min":196.333,"15min":219.399},"last":3.1160000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":4.516,"15min":4.5620000000000003},"min":{"1min":0.33000000000000002,"5min":0.254,"15min":0.20599999999999999},"max":{"1min":8.0540000000000003,"5min":107.87,"15min":215.61699999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.8999999999999999,"5min":3.9540000000000002,"15min":4.1829999999999998},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.23300000000000001},"max":{"1min":7.8220000000000001,"5min":107.904,"15min":216.22200000000001},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":3.7879999999999998,"15min":6.718},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27700000000000002},"max":{"1min":5.9809999999999999,"5min":198.94200000000001,"15min":216.34299999999999},"last":4.3280000000000003},{"interface":"front","average":{"1min":1.895,"5min":3.948,"15min":6.8929999999999998},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":5.8239999999999998,"5min":198.92099999999999,"15min":216.25200000000001},"last":0.64900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":6.0700000000000003,"15min":4.6120000000000001},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":4.9809999999999999,"5min":187.34200000000001,"15min":215.922},"last":3.242},{"interface":"front","average":{"1min":1.929,"5min":6.0670000000000002,"15min":4.6959999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.9189999999999996,"5min":187.08199999999999,"15min":215.94900000000001},"last":3.1549999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.79600000000000004,"15min":2.4729999999999999},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":0.56999999999999995,"5min":6.1689999999999996,"15min":218.446},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.74299999999999999,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":1.7190000000000001,"5min":4.891,"15min":216.172},"last":3.1829999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.86699999999999999,"15min":2.4620000000000002},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":4.2309999999999999,"5min":8.8170000000000002,"15min":216.37200000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.087,"15min":2.3889999999999998},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":2.1000000000000001,"5min":6.3019999999999996,"15min":218.15899999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.123,"15min":2.2909999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":2.6560000000000001,"5min":15.162000000000001,"15min":218.19999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.89000000000000001,"15min":2.0979999999999999},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":1.7589999999999999,"5min":8.2919999999999998,"15min":218.05500000000001},"last":3.21}]}]},{"osd":6,"up_from":46,"seq":197568496085,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494320,"kb_used_data":587112,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92250832,"statfs":{"total":95995035648,"available":94464851968,"internally_reserved":0,"allocated":601202688,"data_stored":600192322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":3.577,"15min":3.7759999999999998},"min":{"1min":0.22500000000000001,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":47.363999999999997,"5min":75.061999999999998,"15min":110.926},"last":0.29999999999999999},{"interface":"front","average":{"1min":4.7809999999999997,"5min":3.73,"15min":4.0439999999999996},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":47.381,"5min":75.021000000000001,"15min":110.911},"last":4.8819999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:33:48 2023","interfaces":[{"interface":"back","average":{"1min":4.0339999999999998,"5min":4.4669999999999996,"15min":5.242},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":41.912999999999997,"5min":87.147999999999996,"15min":292.92200000000003},"last":2.7109999999999999},{"interface":"front","average":{"1min":4.0049999999999999,"5min":4.5730000000000004,"15min":5.1159999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":41.927,"5min":87.191000000000003,"15min":292.904},"last":2.5470000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.8220000000000001,"15min":3.1840000000000002},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":47.484000000000002,"5min":74.748999999999995,"15min":75.191999999999993},"last":2.7519999999999998},{"interface":"front","average":{"1min":4.5880000000000001,"5min":3.4820000000000002,"15min":2.9079999999999999},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":47.494999999999997,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":5.3849999999999998,"15min":3.9159999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.761000000000003,"5min":91.174000000000007,"15min":110.358},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.1799999999999997,"5min":5.3019999999999996,"15min":3.9209999999999998},"min":{"1min":0.38200000000000001,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":47.606000000000002,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.365,"15min":1.278},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.2690000000000001,"5min":12.666,"15min":17.643999999999998},"last":2.7869999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.1000000000000001,"15min":1.1259999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.223},"max":{"1min":6.7859999999999996,"5min":8.6059999999999999,"15min":10.563000000000001},"last":2.8889999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.2769999999999999,"15min":1.377},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29199999999999998},"max":{"1min":5.7539999999999996,"5min":12.747999999999999,"15min":22.379999999999999},"last":3.907},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5620000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":6.8140000000000001,"5min":16.536999999999999,"15min":24.401},"last":2.508}]},{"osd":7,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.204,"15min":2.528},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":19.295999999999999,"15min":122.937},"last":0.5},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.4690000000000001,"15min":2.673},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4049999999999998,"5min":19.584,"15min":122.92100000000001},"last":2.6589999999999998}]}]},{"osd":1,"up_from":110,"seq":472446402822,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185476,"kb_used_data":393020,"kb_used_omap":4,"kb_used_meta":792187,"kb_avail":92559676,"statfs":{"total":95995035648,"available":94781108224,"internally_reserved":0,"allocated":402452480,"data_stored":400546912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":811200398},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":6.2249999999999996,"15min":6.3029999999999999},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":19.699999999999999,"5min":143.465,"15min":269.79199999999997},"last":6.4500000000000002},{"interface":"front","average":{"1min":4.1660000000000004,"5min":6.2460000000000004,"15min":6.7720000000000002},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.17899999999999999},"max":{"1min":11.663,"5min":143.66399999999999,"15min":271.20999999999998},"last":7.9349999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:33:42 2023","interfaces":[{"interface":"back","average":{"1min":3.9830000000000001,"5min":3.7130000000000001,"15min":3.6230000000000002},"min":{"1min":0.186,"5min":0.184,"15min":0.184},"max":{"1min":22.614999999999998,"5min":138.452,"15min":139.75399999999999},"last":4.774},{"interface":"front","average":{"1min":3.6629999999999998,"5min":4.202,"15min":4.3220000000000001},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":24.372,"5min":138.47499999999999,"15min":140.29900000000001},"last":7.8899999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":3.2890000000000001,"15min":3.9620000000000002},"min":{"1min":0.28499999999999998,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":8.9420000000000002,"5min":27.064,"15min":153.18199999999999},"last":2.5819999999999999},{"interface":"front","average":{"1min":2.891,"5min":2.1920000000000002,"15min":2.819},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":13.129,"5min":26.626999999999999,"15min":152.52000000000001},"last":6.569}]},{"osd":4,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6739999999999999,"15min":3.7789999999999999},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.6159999999999997,"5min":38.835999999999999,"15min":199.43799999999999},"last":2.613},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.895,"15min":3.4140000000000001},"min":{"1min":0.36599999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":8.8089999999999993,"5min":38.819000000000003,"15min":199.32900000000001},"last":1.792}]},{"osd":5,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.5750000000000002,"15min":2.339},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.163,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.5619999999999998,"15min":2.7040000000000002},"min":{"1min":0.40999999999999998,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":8.7759999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":1.7470000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":3.6949999999999998,"15min":3.1890000000000001},"min":{"1min":0.39900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.9250000000000007,"5min":107.27200000000001,"15min":107.27200000000001},"last":1.8380000000000001},{"interface":"front","average":{"1min":2.52,"5min":4.452,"15min":3.4700000000000002},"min":{"1min":0.379,"5min":0.316,"15min":0.316},"max":{"1min":7.6950000000000003,"5min":107.252,"15min":107.252},"last":2.6709999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":4.2190000000000003,"15min":3.6389999999999998},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":7.6529999999999996,"5min":104.07299999999999,"15min":104.07299999999999},"last":2.6389999999999998},{"interface":"front","average":{"1min":2.7309999999999999,"5min":3.4750000000000001,"15min":3.0110000000000001},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.191000000000001,"5min":104.056,"15min":104.056},"last":0.39000000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024255,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411124,"kb_used_data":465068,"kb_used_omap":5,"kb_used_meta":945722,"kb_avail":92334028,"statfs":{"total":95995035648,"available":94550044672,"internally_reserved":0,"allocated":476229632,"data_stored":474042880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":968419962},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,6],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.9930000000000001,"15min":3.4260000000000002},"min":{"1min":0.19900000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":33.091999999999999,"5min":33.091999999999999,"15min":191.446},"last":0.45800000000000002},{"interface":"front","average":{"1min":4.25,"5min":2.6120000000000001,"15min":3.7109999999999999},"min":{"1min":0.23799999999999999,"5min":0.187,"15min":0.187},"max":{"1min":24.838999999999999,"5min":24.838999999999999,"15min":191.393},"last":0.80600000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:33:52 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.5009999999999999,"15min":2.5350000000000001},"min":{"1min":0.23300000000000001,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":9.1859999999999999,"5min":16.936,"15min":57.162999999999997},"last":2.145},{"interface":"front","average":{"1min":2.8889999999999998,"5min":3.3210000000000002,"15min":2.9889999999999999},"min":{"1min":0.22600000000000001,"5min":0.223,"15min":0.20999999999999999},"max":{"1min":8.9450000000000003,"5min":17.962,"15min":57.585000000000001},"last":3.1419999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:11 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":6.5800000000000001,"15min":5.9820000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":15.413,"5min":187.172,"15min":208.11199999999999},"last":3.194},{"interface":"front","average":{"1min":3.9609999999999999,"5min":6.2960000000000003,"15min":5.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":15.266,"5min":194.851,"15min":208.25899999999999},"last":2.5059999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.78,"15min":1.593},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":5.6059999999999999,"5min":9.7159999999999993,"15min":11.162000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.2749999999999999,"5min":2.2509999999999999,"15min":1.929},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":8.2880000000000003,"5min":18.081,"15min":18.081},"last":0.84999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.883,"15min":1.907},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":6.2809999999999997,"5min":9.7720000000000002,"15min":62.984999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.492,"15min":2.3450000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":6.1529999999999996,"5min":15.747999999999999,"15min":64.983999999999995},"last":1.27}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":3.4790000000000001,"15min":2.9540000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":6.2069999999999999,"5min":85.525999999999996,"15min":120.572},"last":1.306},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.8370000000000002,"15min":2.4460000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.30299999999999999},"max":{"1min":5.6440000000000001,"5min":85.734999999999999,"15min":120.64400000000001},"last":0.42999999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.5299999999999998,"15min":2.6579999999999999},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.32300000000000001},"max":{"1min":8.3249999999999993,"5min":18.041,"15min":122.018},"last":0.88800000000000001},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.9119999999999999,"15min":2.972},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":7.7089999999999996,"5min":26.416,"15min":122.32299999999999},"last":1.234}]}]},{"osd":2,"up_from":128,"seq":549755814124,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386036,"kb_used_data":660844,"kb_used_omap":7,"kb_used_meta":725176,"kb_avail":92359116,"statfs":{"total":95995035648,"available":94575734784,"internally_reserved":0,"allocated":676704256,"data_stored":674862216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":742580305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":7.7160000000000002,"5min":5.1029999999999998,"15min":4.9429999999999996},"min":{"1min":0.309,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":72.125,"5min":82.751000000000005,"15min":162.21100000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":5.9829999999999997,"5min":4.8460000000000001,"15min":4.4989999999999997},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":75.456000000000003,"5min":81.960999999999999,"15min":168.49700000000001},"last":0.36499999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":4.0499999999999998,"15min":4.1390000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20599999999999999},"max":{"1min":16.152999999999999,"5min":74.744,"15min":205.148},"last":140.452},{"interface":"front","average":{"1min":2.4009999999999998,"5min":4.5,"15min":4.335},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":8.9819999999999993,"5min":75.759,"15min":205.267},"last":139.75299999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2880000000000003,"5min":3.968,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":75.498999999999995,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.27800000000000002},{"interface":"front","average":{"1min":6.2220000000000004,"5min":4.3140000000000001,"15min":3.9409999999999998},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":69.784000000000006,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.32400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":2.1699999999999999,"15min":2.2669999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.29099999999999998},"max":{"1min":6.2839999999999998,"5min":44.314,"15min":90.938999999999993},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.3690000000000002,"5min":2.492,"15min":2.5880000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":8.6099999999999994,"5min":43.896000000000001,"15min":90.950000000000003},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":2.0339999999999998,"15min":1.7070000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.3140000000000001,"5min":13.102,"15min":17.800000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.4670000000000001,"5min":2.2389999999999999,"15min":2.0089999999999999},"min":{"1min":0.39100000000000001,"5min":0.33800000000000002,"15min":0.32600000000000001},"max":{"1min":8.6829999999999998,"5min":11.285,"15min":11.285},"last":0.38}]},{"osd":6,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":4.2460000000000004,"15min":3.2999999999999998},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":8.6600000000000001,"5min":123.47799999999999,"15min":138.291},"last":14.648},{"interface":"front","average":{"1min":3.677,"5min":4.133,"15min":3.093},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":16.012,"5min":122.517,"15min":138.01599999999999},"last":14.522}]},{"osd":7,"last update":"Mon Oct 9 06:33:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5150000000000001,"5min":2.875,"15min":2.3399999999999999},"min":{"1min":0.36499999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":16.064,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.46,"5min":2.5910000000000002,"15min":2.1970000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":6.3460000000000001,"5min":47.206000000000003,"15min":55.731000000000002},"last":0.40300000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127322,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1328108,"kb_used_data":667748,"kb_used_omap":2,"kb_used_meta":660349,"kb_avail":92417044,"statfs":{"total":95995035648,"available":94635053056,"internally_reserved":0,"allocated":683773952,"data_stored":682007477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":676197429},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":7.1669999999999998,"15min":5.8540000000000001},"min":{"1min":0.20000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":22.367000000000001,"5min":161.821,"15min":187.852},"last":5.6260000000000003},{"interface":"front","average":{"1min":2.3140000000000001,"5min":6.1689999999999996,"15min":5.8490000000000002},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":13.443,"5min":163.52000000000001,"15min":187.923},"last":7.468}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9209999999999998,"5min":4.4930000000000003,"15min":4.375},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":22.349,"5min":155.23099999999999,"15min":187.89099999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":4.1050000000000004,"5min":6,"15min":4.2480000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":15.066000000000001,"5min":155.24799999999999,"15min":155.24799999999999},"last":5.8330000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.7329999999999997,"15min":4.234},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":22.297000000000001,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.439},{"interface":"front","average":{"1min":3.5720000000000001,"5min":5.4390000000000001,"15min":4.2380000000000004},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":13.631,"5min":168.16900000000001,"15min":187.79499999999999},"last":6.1870000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.9159999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.661,"5min":48.249000000000002,"15min":48.249000000000002},"last":5.8099999999999996},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.887,"15min":2.645},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":16.516999999999999,"5min":48.031999999999996,"15min":187.67400000000001},"last":1.3200000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.7389999999999999,"15min":2.7250000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":16.494,"5min":48.162999999999997,"15min":187.833},"last":1.3879999999999999},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.1520000000000001,"15min":2.4300000000000002},"min":{"1min":0.42299999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":18.835999999999999,"5min":48.106000000000002,"15min":187.952},"last":0.46999999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":4.3339999999999996,"15min":3.3319999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.462,"5min":130.39599999999999,"15min":187.78200000000001},"last":1.355},{"interface":"front","average":{"1min":2.3199999999999998,"5min":3.8679999999999999,"15min":2.9900000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":22.260000000000002,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":4.5229999999999997,"15min":3.3769999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":13.695,"5min":121.613,"15min":192.20400000000001},"last":5.7699999999999996},{"interface":"front","average":{"1min":3.0110000000000001,"5min":4.7270000000000003,"15min":3.4569999999999999},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.542000000000002,"5min":121.559,"15min":187.86500000000001},"last":1.4119999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019358,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367480,"kb_used_data":480688,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":92377672,"statfs":{"total":95995035648,"available":94594736128,"internally_reserved":0,"allocated":492224512,"data_stored":491206331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":11.006,"5min":11.769,"15min":5.6020000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":118.98999999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.93400000000000005},{"interface":"front","average":{"1min":11.007,"5min":11.597,"15min":5.8220000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":118.956,"5min":287.57100000000003,"15min":287.57100000000003},"last":1.296}]},{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":10.643000000000001,"5min":5.5620000000000003,"15min":3.4590000000000001},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":134.37100000000001,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.375},{"interface":"front","average":{"1min":10.766,"5min":5.7549999999999999,"15min":3.669},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":134.21700000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":1.329}]},{"osd":2,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":5.3170000000000002,"15min":3.7749999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":148.36099999999999,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.497},{"interface":"front","average":{"1min":12.159000000000001,"5min":5.2990000000000004,"15min":3.7559999999999998},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":145.755,"5min":151.58699999999999,"15min":151.58699999999999},"last":3.286}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":8.2629999999999999,"5min":10.750999999999999,"15min":5.8639999999999999},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":65.915000000000006,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.432},{"interface":"front","average":{"1min":8.6899999999999995,"5min":10.590999999999999,"15min":5.5880000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":68.054000000000002,"5min":161.934,"15min":161.934},"last":0.46000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.70699999999999996,"15min":0.98999999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80600000000000005,"5min":6.0199999999999996,"15min":10.673},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.82799999999999996,"15min":1.0029999999999999},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.548,"5min":5.7000000000000002,"15min":12.552},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.4620000000000002,"15min":1.2929999999999999},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":2.5179999999999998,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.641,"15min":1.6419999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":2.9340000000000002,"5min":163.381,"15min":163.381},"last":0.45100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.6499999999999999,"15min":1.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":2.9649999999999999,"5min":70.247,"15min":70.247},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.774,"15min":1.6160000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0680000000000001,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.51400000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757723,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1058216,"kb_used_data":172896,"kb_used_omap":0,"kb_used_meta":885312,"kb_avail":92686936,"statfs":{"total":95995035648,"available":94911422464,"internally_reserved":0,"allocated":177045504,"data_stored":176031770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906559488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":6.1609999999999996,"15min":4.9950000000000001},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.216},"max":{"1min":8.6940000000000008,"5min":294.89699999999999,"15min":294.89699999999999},"last":7.0199999999999996},{"interface":"front","average":{"1min":1.47,"5min":5.9009999999999998,"15min":4.8280000000000003},"min":{"1min":0.34999999999999998,"5min":0.24399999999999999,"15min":0.22},"max":{"1min":5.6539999999999999,"5min":294.30399999999997,"15min":294.30399999999997},"last":4.1950000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":3.2090000000000001,"15min":2.8700000000000001},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.4160000000000004,"5min":87.795000000000002,"15min":126.71899999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.357,"5min":3.4300000000000002,"15min":3.02},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":3.2320000000000002,"5min":87.856999999999999,"15min":126.931},"last":3.105}]},{"osd":2,"last update":"Mon Oct 9 06:33:45 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.625,"15min":2.5019999999999998},"min":{"1min":0.311,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.6589999999999998,"5min":64.150000000000006,"15min":81.260000000000005},"last":4.5149999999999997},{"interface":"front","average":{"1min":1.893,"5min":2.4489999999999998,"15min":2.4049999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":8.1470000000000002,"5min":64.507000000000005,"15min":81.245000000000005},"last":0.36899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":5.6289999999999996,"15min":4.6790000000000003},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":38.582000000000001,"5min":144.56399999999999,"15min":144.56399999999999},"last":12.109},{"interface":"front","average":{"1min":3.9409999999999998,"5min":5.3700000000000001,"15min":4.5099999999999998},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":38.447000000000003,"5min":144.62799999999999,"15min":144.62799999999999},"last":1.343}]},{"osd":4,"last update":"Mon Oct 9 06:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.74199999999999999,"15min":1.1859999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.8700000000000001,"5min":4.9119999999999999,"15min":32.850999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.78900000000000003,"15min":1.044},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":2.8919999999999999,"5min":6.2610000000000001,"15min":32.82},"last":0.46800000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.7250000000000001,"15min":1.6200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7839999999999998,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.9810000000000001,"15min":1.7450000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0630000000000002,"5min":55.040999999999997,"15min":55.040999999999997},"last":3.0659999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.147,"15min":1.397},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1829999999999998,"5min":7.4290000000000003,"15min":30.658999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.022,"15min":1.399},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":2.3540000000000001,"5min":6.2320000000000002,"15min":30.683},"last":0.53900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":466747392,"data_stored":466747392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":387190784,"data_stored":387186712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":669126656,"data_stored":669122584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":662568960,"data_stored":662568960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":487653376,"data_stored":487653376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":167641088,"data_stored":167641088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":591396864,"data_stored":591396864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":705957888,"data_stored":705953816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:47.769 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:47.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:47.850 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:34:47.873 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:47.964 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:47.975 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:48.101 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:48.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:47 smithi150 ceph-mon[194178]: 12.9 scrub starts 2023-10-09T06:34:48.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2518/48861 objects degraded (5.153%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:48.209 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:48.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:47 smithi107 ceph-mon[212750]: 12.9 scrub starts 2023-10-09T06:34:48.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2518/48861 objects degraded (5.153%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:48.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:47 smithi107 ceph-mon[220921]: 12.9 scrub starts 2023-10-09T06:34:48.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2518/48861 objects degraded (5.153%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:48.291 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:48.300 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:48.462 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:48.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:48.567 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:48.668 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:48 smithi150 ceph-mon[194178]: pgmap v1289: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 953 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2208/45786 objects degraded (4.822%); 1.4 MiB/s, 77 objects/s recovering 2023-10-09T06:34:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:48 smithi107 ceph-mon[212750]: pgmap v1289: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 953 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2208/45786 objects degraded (4.822%); 1.4 MiB/s, 77 objects/s recovering 2023-10-09T06:34:49.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:48 smithi107 ceph-mon[220921]: pgmap v1289: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 953 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2208/45786 objects degraded (4.822%); 1.4 MiB/s, 77 objects/s recovering 2023-10-09T06:34:49.440 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:34:50.008 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:50.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:50.118 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:50 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:34:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:50 smithi150 ceph-mon[194178]: 12.4 scrub starts 2023-10-09T06:34:50.191 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:50.192 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:50.232 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:50.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:50 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:34:50.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:50 smithi107 ceph-mon[212750]: 12.4 scrub starts 2023-10-09T06:34:50.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:50 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:34:50.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:50 smithi107 ceph-mon[220921]: 12.4 scrub starts 2023-10-09T06:34:50.623 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:50.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:50.732 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:50.756 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:34:50.777 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:50.803 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:51.009 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:51.013 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:34:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:51.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:51.115 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:51.153 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:51.159 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:51 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:51.159 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:51 smithi107 ceph-mon[212750]: pgmap v1290: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 793 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2004/38121 objects degraded (5.257%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:34:51.167 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:51 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:51.167 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:51 smithi107 ceph-mon[220921]: pgmap v1290: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 793 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2004/38121 objects degraded (5.257%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:34:51.211 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:51.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:34:51.323 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:51.389 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:51 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:51 smithi150 ceph-mon[194178]: pgmap v1290: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 793 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2004/38121 objects degraded (5.257%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:34:51.492 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:34:51.590 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:51.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:51.693 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:51.778 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:52.067 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:34:52.204 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:52.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:52.319 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:52.348 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:52.351 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:52.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:52.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:34:52.401 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:52.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:34:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:34:52] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2023-10-09T06:34:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:52.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:52.505 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:52.531 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:52.588 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:52.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:52.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:52.719 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:53.060 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 729 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 1622/35067 objects degraded (4.625%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: osdmap e419: 8 total, 8 up, 8 in 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1622/35067 objects degraded (4.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 729 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 1622/35067 objects degraded (4.625%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: osdmap e419: 8 total, 8 up, 8 in 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:53 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1622/35067 objects degraded (4.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: pgmap v1291: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 729 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 1622/35067 objects degraded (4.625%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3103794962' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: osdmap e419: 8 total, 8 up, 8 in 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: failed to encode map e419 with expected crc 2023-10-09T06:34:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:53 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1622/35067 objects degraded (4.625%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:53.522 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:53.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:53.549 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:34:53.549 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:34:53.632 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:53.725 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:54.149 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:54.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:54.243 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:54.257 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:54.350 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:54.376 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:34:54.556 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:54.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:54.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.680 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.717 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: pgmap v1293: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:54.718 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.718 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.718 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:34:54.718 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:34:54.718 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:34:54.776 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:54.801 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:34:54.863 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:34:54.912 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:34:54.959 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:34:54.996 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:55.001 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:34:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:55.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:55.046 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:34:55.089 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:34:55.107 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:55.143 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:34:55.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.145 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.146 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.146 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.146 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.146 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:34:55.182 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:55.229 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:55.295 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:55.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:55.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:55.437 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:55.482 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:55.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:55.587 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:55.623 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:55.666 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:55.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:55.773 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:55.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:55 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:55.833 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1830933750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:55.840 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:55 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:55.840 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1830933750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:55.841 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:55 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1830933750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:34:56.274 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:34:56.319 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:56.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:56.427 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:56.525 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:56.562 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:56.565 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[212750]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:56.565 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[212750]: 12.5 scrub starts 2023-10-09T06:34:56.566 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:56.566 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[220921]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:56.566 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:56 smithi107 ceph-mon[220921]: 12.5 scrub starts 2023-10-09T06:34:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:56 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:56 smithi150 ceph-mon[194178]: pgmap v1294: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 575 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 1412/27660 objects degraded (5.105%); 1.3 MiB/s, 89 objects/s recovering 2023-10-09T06:34:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:56 smithi150 ceph-mon[194178]: 12.5 scrub starts 2023-10-09T06:34:56.904 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:56.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:57.019 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:57.127 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:57.398 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:57.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:57.507 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:57.541 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:57.586 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1412/27660 objects degraded (5.105%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:57.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1412/27660 objects degraded (5.105%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:57.604 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:57.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:57.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:57.713 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:34:57.782 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:57.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1412/27660 objects degraded (5.105%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:34:57.914 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:34:57.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:57.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:34:58.028 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:58.044 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:34:58.047 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:34:58.055 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:58.098 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:58.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:58.201 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:34:58.227 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:58.268 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:34:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:58.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:34:58.371 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:58.414 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:58.482 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:58.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:58.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:58.590 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:34:58.595 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:58 smithi107 ceph-mon[212750]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 516 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 1098/24822 objects degraded (4.423%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:58.595 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:58 smithi107 ceph-mon[212750]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:58.675 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:58.842 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:34:58.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:58.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:34:58.850 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:58 smithi107 ceph-mon[220921]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 516 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 1098/24822 objects degraded (4.423%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:58.850 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:58 smithi107 ceph-mon[220921]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:58 smithi150 ceph-mon[194178]: pgmap v1295: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 516 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 1098/24822 objects degraded (4.423%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:34:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:58 smithi150 ceph-mon[194178]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:34:58.951 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:34:59.047 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:59.077 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:34:59.362 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:34:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:59.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:34:59.471 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:34:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:59.508 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:34:59.523 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1295,"stamp":"2023-10-09T06:34:57.416851+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":541038248,"num_objects":8274,"num_object_clones":0,"num_object_copies":24822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1098,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8274,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186053,"num_write_kb":6175998,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11793,"num_bytes_recovered":665791228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208340,"ondisk_log_size":208340,"up":387,"acting":385,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":394,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9187352,"kb_used_data":2484248,"kb_used_omap":20,"kb_used_meta":6702955,"kb_avail":740773864,"statfs":{"total":767960285184,"available":758552436736,"internally_reserved":0,"allocated":2543869952,"data_stored":2532070390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6863826574},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-457965568,"num_objects":-6988,"num_object_clones":0,"num_object_copies":-20964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1110,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6988,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":903,"num_bytes_recovered":13565952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002855"},"pg_stats":[{"pgid":"12.15","version":"419'8820","reported_seq":8907,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.226753+0000","last_change":"2023-10-09T06:34:17.052386+0000","last_active":"2023-10-09T06:34:52.226753+0000","last_peered":"2023-10-09T06:34:52.226753+0000","last_clean":"2023-10-09T06:34:52.226753+0000","last_became_active":"2023-10-09T06:32:33.082121+0000","last_became_peered":"2023-10-09T06:32:33.082121+0000","last_unstale":"2023-10-09T06:34:52.226753+0000","last_undegraded":"2023-10-09T06:34:52.226753+0000","last_fullsized":"2023-10-09T06:34:52.226753+0000","mapping_epoch":390,"log_start":"365'257","ondisk_log_start":"365'257","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8563,"log_dups_size":0,"ondisk_log_size":8563,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19496960,"num_objects":298,"num_object_clones":0,"num_object_copies":894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5926,"num_write_kb":199136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":780,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:44.780222+0000","last_change":"2023-10-09T06:34:44.780190+0000","last_active":"2023-10-09T06:34:44.780222+0000","last_peered":"2023-10-09T06:34:44.780222+0000","last_clean":"2023-10-09T06:34:44.780222+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:34:44.780222+0000","last_undegraded":"2023-10-09T06:34:44.780222+0000","last_fullsized":"2023-10-09T06:34:44.780222+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":398,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930806+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:34:41.930806+0000","last_peered":"2023-10-09T06:34:41.930806+0000","last_clean":"2023-10-09T06:34:41.930806+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:34:41.930806+0000","last_undegraded":"2023-10-09T06:34:41.930806+0000","last_fullsized":"2023-10-09T06:34:41.930806+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":118,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960619+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:34:41.960619+0000","last_peered":"2023-10-09T06:34:41.960619+0000","last_clean":"2023-10-09T06:34:41.960619+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:34:41.960619+0000","last_undegraded":"2023-10-09T06:34:41.960619+0000","last_fullsized":"2023-10-09T06:34:41.960619+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"419'8829","reported_seq":8905,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.405522+0000","last_change":"2023-10-09T06:32:33.003181+0000","last_active":"2023-10-09T06:34:52.405522+0000","last_peered":"2023-10-09T06:34:52.405522+0000","last_clean":"2023-10-09T06:34:52.405522+0000","last_became_active":"2023-10-09T06:32:32.998596+0000","last_became_peered":"2023-10-09T06:32:32.998596+0000","last_unstale":"2023-10-09T06:34:52.405522+0000","last_undegraded":"2023-10-09T06:34:52.405522+0000","last_fullsized":"2023-10-09T06:34:52.405522+0000","mapping_epoch":390,"log_start":"365'804","ondisk_log_start":"365'804","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8025,"log_dups_size":0,"ondisk_log_size":8025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17760256,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5945,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":740,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:48.810253+0000","last_change":"2023-10-09T06:34:48.810215+0000","last_active":"2023-10-09T06:34:48.810253+0000","last_peered":"2023-10-09T06:34:48.810253+0000","last_clean":"2023-10-09T06:34:48.810253+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:34:48.810253+0000","last_undegraded":"2023-10-09T06:34:48.810253+0000","last_fullsized":"2023-10-09T06:34:48.810253+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:30.014465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:30.014465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":413,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:47.654670+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:34:47.654670+0000","last_peered":"2023-10-09T06:34:47.654670+0000","last_clean":"2023-10-09T06:34:47.654670+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:34:47.654670+0000","last_undegraded":"2023-10-09T06:34:47.654670+0000","last_fullsized":"2023-10-09T06:34:47.654670+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":417,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931454+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:34:41.931454+0000","last_peered":"2023-10-09T06:34:41.931454+0000","last_clean":"2023-10-09T06:34:41.931454+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:34:41.931454+0000","last_undegraded":"2023-10-09T06:34:41.931454+0000","last_fullsized":"2023-10-09T06:34:41.931454+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"419'8761","reported_seq":8838,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.556975+0000","last_change":"2023-10-09T06:32:33.092190+0000","last_active":"2023-10-09T06:34:52.556975+0000","last_peered":"2023-10-09T06:34:52.556975+0000","last_clean":"2023-10-09T06:34:52.556975+0000","last_became_active":"2023-10-09T06:32:33.084295+0000","last_became_peered":"2023-10-09T06:32:33.084295+0000","last_unstale":"2023-10-09T06:34:52.556975+0000","last_undegraded":"2023-10-09T06:34:52.556975+0000","last_fullsized":"2023-10-09T06:34:52.556975+0000","mapping_epoch":390,"log_start":"365'891","ondisk_log_start":"365'891","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7870,"log_dups_size":0,"ondisk_log_size":7870,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10518528,"num_objects":161,"num_object_clones":0,"num_object_copies":483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5917,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":666,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930672+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:34:41.930672+0000","last_peered":"2023-10-09T06:34:41.930672+0000","last_clean":"2023-10-09T06:34:41.930672+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:34:41.930672+0000","last_undegraded":"2023-10-09T06:34:41.930672+0000","last_fullsized":"2023-10-09T06:34:41.930672+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":136,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.940312+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:34:41.940312+0000","last_peered":"2023-10-09T06:34:41.940312+0000","last_clean":"2023-10-09T06:34:41.940312+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:34:41.940312+0000","last_undegraded":"2023-10-09T06:34:41.940312+0000","last_fullsized":"2023-10-09T06:34:41.940312+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":408,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948877+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:34:41.948877+0000","last_peered":"2023-10-09T06:34:41.948877+0000","last_clean":"2023-10-09T06:34:41.948877+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:34:41.948877+0000","last_undegraded":"2023-10-09T06:34:41.948877+0000","last_fullsized":"2023-10-09T06:34:41.948877+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"419'8693","reported_seq":8769,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:53.202766+0000","last_change":"2023-10-09T06:32:33.080997+0000","last_active":"2023-10-09T06:34:53.202766+0000","last_peered":"2023-10-09T06:34:53.202766+0000","last_clean":"2023-10-09T06:34:53.202766+0000","last_became_active":"2023-10-09T06:32:33.077785+0000","last_became_peered":"2023-10-09T06:32:33.077785+0000","last_unstale":"2023-10-09T06:34:53.202766+0000","last_undegraded":"2023-10-09T06:34:53.202766+0000","last_fullsized":"2023-10-09T06:34:53.202766+0000","mapping_epoch":390,"log_start":"365'1205","ondisk_log_start":"365'1205","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7488,"log_dups_size":0,"ondisk_log_size":7488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23461888,"num_objects":358,"num_object_clones":0,"num_object_copies":1074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":527,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945747+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:34:41.945747+0000","last_peered":"2023-10-09T06:34:41.945747+0000","last_clean":"2023-10-09T06:34:41.945747+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:34:41.945747+0000","last_undegraded":"2023-10-09T06:34:41.945747+0000","last_fullsized":"2023-10-09T06:34:41.945747+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":92,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931411+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:34:41.931411+0000","last_peered":"2023-10-09T06:34:41.931411+0000","last_clean":"2023-10-09T06:34:41.931411+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:34:41.931411+0000","last_undegraded":"2023-10-09T06:34:41.931411+0000","last_fullsized":"2023-10-09T06:34:41.931411+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":119,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.962809+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:34:41.962809+0000","last_peered":"2023-10-09T06:34:41.962809+0000","last_clean":"2023-10-09T06:34:41.962809+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:34:41.962809+0000","last_undegraded":"2023-10-09T06:34:41.962809+0000","last_fullsized":"2023-10-09T06:34:41.962809+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"419'8741","reported_seq":8825,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.310828+0000","last_change":"2023-10-09T06:34:21.023111+0000","last_active":"2023-10-09T06:34:52.310828+0000","last_peered":"2023-10-09T06:34:52.310828+0000","last_clean":"2023-10-09T06:34:52.310828+0000","last_became_active":"2023-10-09T06:32:33.074690+0000","last_became_peered":"2023-10-09T06:32:33.074690+0000","last_unstale":"2023-10-09T06:34:52.310828+0000","last_undegraded":"2023-10-09T06:34:52.310828+0000","last_fullsized":"2023-10-09T06:34:52.310828+0000","mapping_epoch":390,"log_start":"365'159","ondisk_log_start":"365'159","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8582,"log_dups_size":0,"ondisk_log_size":8582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16285696,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":195872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":564,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945807+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:34:41.945807+0000","last_peered":"2023-10-09T06:34:41.945807+0000","last_clean":"2023-10-09T06:34:41.945807+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:34:41.945807+0000","last_undegraded":"2023-10-09T06:34:41.945807+0000","last_fullsized":"2023-10-09T06:34:41.945807+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934923+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:34:41.934923+0000","last_peered":"2023-10-09T06:34:41.934923+0000","last_clean":"2023-10-09T06:34:41.934923+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:34:41.934923+0000","last_undegraded":"2023-10-09T06:34:41.934923+0000","last_fullsized":"2023-10-09T06:34:41.934923+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960878+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:34:41.960878+0000","last_peered":"2023-10-09T06:34:41.960878+0000","last_clean":"2023-10-09T06:34:41.960878+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:34:41.960878+0000","last_undegraded":"2023-10-09T06:34:41.960878+0000","last_fullsized":"2023-10-09T06:34:41.960878+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":617,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:51.813321+0000","last_change":"2023-10-09T06:34:51.813303+0000","last_active":"2023-10-09T06:34:51.813321+0000","last_peered":"2023-10-09T06:34:51.813321+0000","last_clean":"2023-10-09T06:34:51.813321+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:34:51.813321+0000","last_undegraded":"2023-10-09T06:34:51.813321+0000","last_fullsized":"2023-10-09T06:34:51.813321+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":300,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946394+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:34:41.946394+0000","last_peered":"2023-10-09T06:34:41.946394+0000","last_clean":"2023-10-09T06:34:41.946394+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:34:41.946394+0000","last_undegraded":"2023-10-09T06:34:41.946394+0000","last_fullsized":"2023-10-09T06:34:41.946394+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":354,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960453+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:34:41.960453+0000","last_peered":"2023-10-09T06:34:41.960453+0000","last_clean":"2023-10-09T06:34:41.960453+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:34:41.960453+0000","last_undegraded":"2023-10-09T06:34:41.960453+0000","last_fullsized":"2023-10-09T06:34:41.960453+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":526,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:47.654793+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:34:47.654793+0000","last_peered":"2023-10-09T06:34:47.654793+0000","last_clean":"2023-10-09T06:34:47.654793+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:34:47.654793+0000","last_undegraded":"2023-10-09T06:34:47.654793+0000","last_fullsized":"2023-10-09T06:34:47.654793+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":425,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931049+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:34:41.931049+0000","last_peered":"2023-10-09T06:34:41.931049+0000","last_clean":"2023-10-09T06:34:41.931049+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:34:41.931049+0000","last_undegraded":"2023-10-09T06:34:41.931049+0000","last_fullsized":"2023-10-09T06:34:41.931049+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":389,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931240+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:34:41.931240+0000","last_peered":"2023-10-09T06:34:41.931240+0000","last_clean":"2023-10-09T06:34:41.931240+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:34:41.931240+0000","last_undegraded":"2023-10-09T06:34:41.931240+0000","last_fullsized":"2023-10-09T06:34:41.931240+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":690,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930761+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:34:41.930761+0000","last_peered":"2023-10-09T06:34:41.930761+0000","last_clean":"2023-10-09T06:34:41.930761+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:34:41.930761+0000","last_undegraded":"2023-10-09T06:34:41.930761+0000","last_fullsized":"2023-10-09T06:34:41.930761+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":268,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945646+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:34:41.945646+0000","last_peered":"2023-10-09T06:34:41.945646+0000","last_clean":"2023-10-09T06:34:41.945646+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:34:41.945646+0000","last_undegraded":"2023-10-09T06:34:41.945646+0000","last_fullsized":"2023-10-09T06:34:41.945646+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":119,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:43.280925+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:34:43.280925+0000","last_peered":"2023-10-09T06:34:43.280925+0000","last_clean":"2023-10-09T06:34:43.280925+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:34:43.280925+0000","last_undegraded":"2023-10-09T06:34:43.280925+0000","last_fullsized":"2023-10-09T06:34:43.280925+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":533,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.933974+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:34:41.933974+0000","last_peered":"2023-10-09T06:34:41.933974+0000","last_clean":"2023-10-09T06:34:41.933974+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:34:41.933974+0000","last_undegraded":"2023-10-09T06:34:41.933974+0000","last_fullsized":"2023-10-09T06:34:41.933974+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":415,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961571+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:34:41.961571+0000","last_peered":"2023-10-09T06:34:41.961571+0000","last_clean":"2023-10-09T06:34:41.961571+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:34:41.961571+0000","last_undegraded":"2023-10-09T06:34:41.961571+0000","last_fullsized":"2023-10-09T06:34:41.961571+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934825+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:34:41.934825+0000","last_peered":"2023-10-09T06:34:41.934825+0000","last_clean":"2023-10-09T06:34:41.934825+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:34:41.934825+0000","last_undegraded":"2023-10-09T06:34:41.934825+0000","last_fullsized":"2023-10-09T06:34:41.934825+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":744,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961476+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:34:41.961476+0000","last_peered":"2023-10-09T06:34:41.961476+0000","last_clean":"2023-10-09T06:34:41.961476+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:34:41.961476+0000","last_undegraded":"2023-10-09T06:34:41.961476+0000","last_fullsized":"2023-10-09T06:34:41.961476+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":262,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946700+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:34:41.946700+0000","last_peered":"2023-10-09T06:34:41.946700+0000","last_clean":"2023-10-09T06:34:41.946700+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:34:41.946700+0000","last_undegraded":"2023-10-09T06:34:41.946700+0000","last_fullsized":"2023-10-09T06:34:41.946700+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931155+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:34:41.931155+0000","last_peered":"2023-10-09T06:34:41.931155+0000","last_clean":"2023-10-09T06:34:41.931155+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:34:41.931155+0000","last_undegraded":"2023-10-09T06:34:41.931155+0000","last_fullsized":"2023-10-09T06:34:41.931155+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":533,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934868+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:34:41.934868+0000","last_peered":"2023-10-09T06:34:41.934868+0000","last_clean":"2023-10-09T06:34:41.934868+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:34:41.934868+0000","last_undegraded":"2023-10-09T06:34:41.934868+0000","last_fullsized":"2023-10-09T06:34:41.934868+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":497,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.947886+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:34:41.947886+0000","last_peered":"2023-10-09T06:34:41.947886+0000","last_clean":"2023-10-09T06:34:41.947886+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:34:41.947886+0000","last_undegraded":"2023-10-09T06:34:41.947886+0000","last_fullsized":"2023-10-09T06:34:41.947886+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":119,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.061685+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:34:52.061685+0000","last_peered":"2023-10-09T06:34:52.061685+0000","last_clean":"2023-10-09T06:34:52.061685+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:34:52.061685+0000","last_undegraded":"2023-10-09T06:34:52.061685+0000","last_fullsized":"2023-10-09T06:34:52.061685+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":546,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945940+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:34:41.945940+0000","last_peered":"2023-10-09T06:34:41.945940+0000","last_clean":"2023-10-09T06:34:41.945940+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:34:41.945940+0000","last_undegraded":"2023-10-09T06:34:41.945940+0000","last_fullsized":"2023-10-09T06:34:41.945940+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"419'8896","reported_seq":9032,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.259132+0000","last_change":"2023-10-09T06:34:08.464064+0000","last_active":"2023-10-09T06:34:52.259132+0000","last_peered":"2023-10-09T06:34:52.259132+0000","last_clean":"2023-10-09T06:34:52.259132+0000","last_became_active":"2023-10-09T06:32:33.069345+0000","last_became_peered":"2023-10-09T06:32:33.069345+0000","last_unstale":"2023-10-09T06:34:52.259132+0000","last_undegraded":"2023-10-09T06:34:52.259132+0000","last_fullsized":"2023-10-09T06:34:52.259132+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'7928","last_scrub_stamp":"2023-10-09T06:34:08.463956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:34:08.463956+0000","objects_scrubbed":1240,"log_size":8896,"log_dups_size":0,"ondisk_log_size":8896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:39.228319+0000","scrub_duration":0.275394735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17760256,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":969,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":304,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946273+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:34:41.946273+0000","last_peered":"2023-10-09T06:34:41.946273+0000","last_clean":"2023-10-09T06:34:41.946273+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:34:41.946273+0000","last_undegraded":"2023-10-09T06:34:41.946273+0000","last_fullsized":"2023-10-09T06:34:41.946273+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934068+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:34:41.934068+0000","last_peered":"2023-10-09T06:34:41.934068+0000","last_clean":"2023-10-09T06:34:41.934068+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:34:41.934068+0000","last_undegraded":"2023-10-09T06:34:41.934068+0000","last_fullsized":"2023-10-09T06:34:41.934068+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":803,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960323+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:34:41.960323+0000","last_peered":"2023-10-09T06:34:41.960323+0000","last_clean":"2023-10-09T06:34:41.960323+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:34:41.960323+0000","last_undegraded":"2023-10-09T06:34:41.960323+0000","last_fullsized":"2023-10-09T06:34:41.960323+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"419'8892","reported_seq":8981,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.349074+0000","last_change":"2023-10-09T06:34:43.375542+0000","last_active":"2023-10-09T06:34:52.349074+0000","last_peered":"2023-10-09T06:34:52.349074+0000","last_clean":"2023-10-09T06:34:52.349074+0000","last_became_active":"2023-10-09T06:32:33.000165+0000","last_became_peered":"2023-10-09T06:32:33.000165+0000","last_unstale":"2023-10-09T06:34:52.349074+0000","last_undegraded":"2023-10-09T06:34:52.349074+0000","last_fullsized":"2023-10-09T06:34:52.349074+0000","mapping_epoch":361,"log_start":"365'417","ondisk_log_start":"365'417","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8475,"log_dups_size":0,"ondisk_log_size":8475,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19529728,"num_objects":298,"num_object_clones":0,"num_object_copies":894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":201024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":266,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946165+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:34:41.946165+0000","last_peered":"2023-10-09T06:34:41.946165+0000","last_clean":"2023-10-09T06:34:41.946165+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:34:41.946165+0000","last_undegraded":"2023-10-09T06:34:41.946165+0000","last_fullsized":"2023-10-09T06:34:41.946165+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":416,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930067+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:34:41.930067+0000","last_peered":"2023-10-09T06:34:41.930067+0000","last_clean":"2023-10-09T06:34:41.930067+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:34:41.930067+0000","last_undegraded":"2023-10-09T06:34:41.930067+0000","last_fullsized":"2023-10-09T06:34:41.930067+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":720,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948769+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:34:41.948769+0000","last_peered":"2023-10-09T06:34:41.948769+0000","last_clean":"2023-10-09T06:34:41.948769+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:34:41.948769+0000","last_undegraded":"2023-10-09T06:34:41.948769+0000","last_fullsized":"2023-10-09T06:34:41.948769+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"419'8874","reported_seq":8978,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.337647+0000","last_change":"2023-10-09T06:34:51.360117+0000","last_active":"2023-10-09T06:34:52.337647+0000","last_peered":"2023-10-09T06:34:52.337647+0000","last_clean":"2023-10-09T06:34:52.337647+0000","last_became_active":"2023-10-09T06:32:32.996849+0000","last_became_peered":"2023-10-09T06:32:32.996849+0000","last_unstale":"2023-10-09T06:34:52.337647+0000","last_undegraded":"2023-10-09T06:34:52.337647+0000","last_fullsized":"2023-10-09T06:34:52.337647+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8874,"log_dups_size":0,"ondisk_log_size":8874,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18120704,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5970,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":322,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946176+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:34:41.946176+0000","last_peered":"2023-10-09T06:34:41.946176+0000","last_clean":"2023-10-09T06:34:41.946176+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:34:41.946176+0000","last_undegraded":"2023-10-09T06:34:41.946176+0000","last_fullsized":"2023-10-09T06:34:41.946176+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":395,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948373+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:34:41.948373+0000","last_peered":"2023-10-09T06:34:41.948373+0000","last_clean":"2023-10-09T06:34:41.948373+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:34:41.948373+0000","last_undegraded":"2023-10-09T06:34:41.948373+0000","last_fullsized":"2023-10-09T06:34:41.948373+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":543,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931705+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:34:41.931705+0000","last_peered":"2023-10-09T06:34:41.931705+0000","last_clean":"2023-10-09T06:34:41.931705+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:34:41.931705+0000","last_undegraded":"2023-10-09T06:34:41.931705+0000","last_fullsized":"2023-10-09T06:34:41.931705+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"419'8930","reported_seq":9008,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:55.927057+0000","last_change":"2023-10-09T06:32:32.995971+0000","last_active":"2023-10-09T06:34:55.927057+0000","last_peered":"2023-10-09T06:34:55.927057+0000","last_clean":"2023-10-09T06:34:55.927057+0000","last_became_active":"2023-10-09T06:32:32.993707+0000","last_became_peered":"2023-10-09T06:32:32.993707+0000","last_unstale":"2023-10-09T06:34:55.927057+0000","last_undegraded":"2023-10-09T06:34:55.927057+0000","last_fullsized":"2023-10-09T06:34:55.927057+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8930,"log_dups_size":0,"ondisk_log_size":8930,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15106048,"num_objects":230,"num_object_clones":0,"num_object_copies":690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6038,"num_write_kb":200608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.935014+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:34:41.935014+0000","last_peered":"2023-10-09T06:34:41.935014+0000","last_clean":"2023-10-09T06:34:41.935014+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:34:41.935014+0000","last_undegraded":"2023-10-09T06:34:41.935014+0000","last_fullsized":"2023-10-09T06:34:41.935014+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":379,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.962146+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:34:41.962146+0000","last_peered":"2023-10-09T06:34:41.962146+0000","last_clean":"2023-10-09T06:34:41.962146+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:34:41.962146+0000","last_undegraded":"2023-10-09T06:34:41.962146+0000","last_fullsized":"2023-10-09T06:34:41.962146+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"419'8787","reported_seq":8874,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.369713+0000","last_change":"2023-10-09T06:34:49.477325+0000","last_active":"2023-10-09T06:34:52.369713+0000","last_peered":"2023-10-09T06:34:52.369713+0000","last_clean":"2023-10-09T06:34:52.369713+0000","last_became_active":"2023-10-09T06:32:32.995923+0000","last_became_peered":"2023-10-09T06:32:32.995923+0000","last_unstale":"2023-10-09T06:34:52.369713+0000","last_undegraded":"2023-10-09T06:34:52.369713+0000","last_fullsized":"2023-10-09T06:34:52.369713+0000","mapping_epoch":361,"log_start":"365'297","ondisk_log_start":"365'297","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8490,"log_dups_size":0,"ondisk_log_size":8490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17760256,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5915,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":771,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946649+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:34:41.946649+0000","last_peered":"2023-10-09T06:34:41.946649+0000","last_clean":"2023-10-09T06:34:41.946649+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:34:41.946649+0000","last_undegraded":"2023-10-09T06:34:41.946649+0000","last_fullsized":"2023-10-09T06:34:41.946649+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":145,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.964007+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:34:41.964007+0000","last_peered":"2023-10-09T06:34:41.964007+0000","last_clean":"2023-10-09T06:34:41.964007+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:34:41.964007+0000","last_undegraded":"2023-10-09T06:34:41.964007+0000","last_fullsized":"2023-10-09T06:34:41.964007+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":399,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930951+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:34:41.930951+0000","last_peered":"2023-10-09T06:34:41.930951+0000","last_clean":"2023-10-09T06:34:41.930951+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:34:41.930951+0000","last_undegraded":"2023-10-09T06:34:41.930951+0000","last_fullsized":"2023-10-09T06:34:41.930951+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"419'8865","reported_seq":8949,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.291599+0000","last_change":"2023-10-09T06:34:14.119582+0000","last_active":"2023-10-09T06:34:52.291599+0000","last_peered":"2023-10-09T06:34:52.291599+0000","last_clean":"2023-10-09T06:34:52.291599+0000","last_became_active":"2023-10-09T06:32:33.044765+0000","last_became_peered":"2023-10-09T06:32:33.044765+0000","last_unstale":"2023-10-09T06:34:52.291599+0000","last_undegraded":"2023-10-09T06:34:52.291599+0000","last_fullsized":"2023-10-09T06:34:52.291599+0000","mapping_epoch":361,"log_start":"365'448","ondisk_log_start":"365'448","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17858560,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5970,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":751,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934046+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:34:41.934046+0000","last_peered":"2023-10-09T06:34:41.934046+0000","last_clean":"2023-10-09T06:34:41.934046+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:34:41.934046+0000","last_undegraded":"2023-10-09T06:34:41.934046+0000","last_fullsized":"2023-10-09T06:34:41.934046+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":376,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948244+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:34:41.948244+0000","last_peered":"2023-10-09T06:34:41.948244+0000","last_clean":"2023-10-09T06:34:41.948244+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:34:41.948244+0000","last_undegraded":"2023-10-09T06:34:41.948244+0000","last_fullsized":"2023-10-09T06:34:41.948244+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":396,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931347+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:34:41.931347+0000","last_peered":"2023-10-09T06:34:41.931347+0000","last_clean":"2023-10-09T06:34:41.931347+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:34:41.931347+0000","last_undegraded":"2023-10-09T06:34:41.931347+0000","last_fullsized":"2023-10-09T06:34:41.931347+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":753,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.959977+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:34:41.959977+0000","last_peered":"2023-10-09T06:34:41.959977+0000","last_clean":"2023-10-09T06:34:41.959977+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:34:41.959977+0000","last_undegraded":"2023-10-09T06:34:41.959977+0000","last_fullsized":"2023-10-09T06:34:41.959977+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"419'8844","reported_seq":8921,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.372235+0000","last_change":"2023-10-09T06:32:32.976279+0000","last_active":"2023-10-09T06:34:52.372235+0000","last_peered":"2023-10-09T06:34:52.372235+0000","last_clean":"2023-10-09T06:34:52.372235+0000","last_became_active":"2023-10-09T06:32:32.973171+0000","last_became_peered":"2023-10-09T06:32:32.973171+0000","last_unstale":"2023-10-09T06:34:52.372235+0000","last_undegraded":"2023-10-09T06:34:52.372235+0000","last_fullsized":"2023-10-09T06:34:52.372235+0000","mapping_epoch":361,"log_start":"365'776","ondisk_log_start":"365'776","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8068,"log_dups_size":0,"ondisk_log_size":8068,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5960,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":400,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.929872+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:34:41.929872+0000","last_peered":"2023-10-09T06:34:41.929872+0000","last_clean":"2023-10-09T06:34:41.929872+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:34:41.929872+0000","last_undegraded":"2023-10-09T06:34:41.929872+0000","last_fullsized":"2023-10-09T06:34:41.929872+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":379,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931983+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:34:41.931983+0000","last_peered":"2023-10-09T06:34:41.931983+0000","last_clean":"2023-10-09T06:34:41.931983+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:34:41.931983+0000","last_undegraded":"2023-10-09T06:34:41.931983+0000","last_fullsized":"2023-10-09T06:34:41.931983+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"419'8628","reported_seq":8706,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.375534+0000","last_change":"2023-10-09T06:32:33.079508+0000","last_active":"2023-10-09T06:34:52.375534+0000","last_peered":"2023-10-09T06:34:52.375534+0000","last_clean":"2023-10-09T06:34:52.375534+0000","last_became_active":"2023-10-09T06:32:33.075123+0000","last_became_peered":"2023-10-09T06:32:33.075123+0000","last_unstale":"2023-10-09T06:34:52.375534+0000","last_undegraded":"2023-10-09T06:34:52.375534+0000","last_fullsized":"2023-10-09T06:34:52.375534+0000","mapping_epoch":361,"log_start":"365'233","ondisk_log_start":"365'233","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8395,"log_dups_size":0,"ondisk_log_size":8395,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24346624,"num_objects":371,"num_object_clones":0,"num_object_copies":1113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5783,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":729,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.935004+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:34:41.935004+0000","last_peered":"2023-10-09T06:34:41.935004+0000","last_clean":"2023-10-09T06:34:41.935004+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:34:41.935004+0000","last_undegraded":"2023-10-09T06:34:41.935004+0000","last_fullsized":"2023-10-09T06:34:41.935004+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.959766+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:34:41.959766+0000","last_peered":"2023-10-09T06:34:41.959766+0000","last_clean":"2023-10-09T06:34:41.959766+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:34:41.959766+0000","last_undegraded":"2023-10-09T06:34:41.959766+0000","last_fullsized":"2023-10-09T06:34:41.959766+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":372,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.963823+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:34:41.963823+0000","last_peered":"2023-10-09T06:34:41.963823+0000","last_clean":"2023-10-09T06:34:41.963823+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:34:41.963823+0000","last_undegraded":"2023-10-09T06:34:41.963823+0000","last_fullsized":"2023-10-09T06:34:41.963823+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"419'12042","reported_seq":12110,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:55.962034+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:34:55.962034+0000","last_peered":"2023-10-09T06:34:55.962034+0000","last_clean":"2023-10-09T06:34:55.962034+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:34:55.962034+0000","last_undegraded":"2023-10-09T06:34:55.962034+0000","last_fullsized":"2023-10-09T06:34:55.962034+0000","mapping_epoch":361,"log_start":"387'5500","ondisk_log_start":"387'5500","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12042,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":711,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948937+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:34:41.948937+0000","last_peered":"2023-10-09T06:34:41.948937+0000","last_clean":"2023-10-09T06:34:41.948937+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:34:41.948937+0000","last_undegraded":"2023-10-09T06:34:41.948937+0000","last_fullsized":"2023-10-09T06:34:41.948937+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":424,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.965139+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:34:41.965139+0000","last_peered":"2023-10-09T06:34:41.965139+0000","last_clean":"2023-10-09T06:34:41.965139+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:34:41.965139+0000","last_undegraded":"2023-10-09T06:34:41.965139+0000","last_fullsized":"2023-10-09T06:34:41.965139+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934715+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:34:41.934715+0000","last_peered":"2023-10-09T06:34:41.934715+0000","last_clean":"2023-10-09T06:34:41.934715+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:34:41.934715+0000","last_undegraded":"2023-10-09T06:34:41.934715+0000","last_fullsized":"2023-10-09T06:34:41.934715+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":358,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960162+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:34:41.960162+0000","last_peered":"2023-10-09T06:34:41.960162+0000","last_clean":"2023-10-09T06:34:41.960162+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:34:41.960162+0000","last_undegraded":"2023-10-09T06:34:41.960162+0000","last_fullsized":"2023-10-09T06:34:41.960162+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"419'12050","reported_seq":12118,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:55.998422+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:34:55.998422+0000","last_peered":"2023-10-09T06:34:55.998422+0000","last_clean":"2023-10-09T06:34:55.998422+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:34:55.998422+0000","last_undegraded":"2023-10-09T06:34:55.998422+0000","last_fullsized":"2023-10-09T06:34:55.998422+0000","mapping_epoch":361,"log_start":"387'5500","ondisk_log_start":"387'5500","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28704768,"num_objects":438,"num_object_clones":0,"num_object_copies":1314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12050,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":765,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.937134+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:34:41.937134+0000","last_peered":"2023-10-09T06:34:41.937134+0000","last_clean":"2023-10-09T06:34:41.937134+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:34:41.937134+0000","last_undegraded":"2023-10-09T06:34:41.937134+0000","last_fullsized":"2023-10-09T06:34:41.937134+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.935139+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:34:41.935139+0000","last_peered":"2023-10-09T06:34:41.935139+0000","last_clean":"2023-10-09T06:34:41.935139+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:34:41.935139+0000","last_undegraded":"2023-10-09T06:34:41.935139+0000","last_fullsized":"2023-10-09T06:34:41.935139+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":400,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931168+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:34:41.931168+0000","last_peered":"2023-10-09T06:34:41.931168+0000","last_clean":"2023-10-09T06:34:41.931168+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:34:41.931168+0000","last_undegraded":"2023-10-09T06:34:41.931168+0000","last_fullsized":"2023-10-09T06:34:41.931168+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":388,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.964578+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:34:41.964578+0000","last_peered":"2023-10-09T06:34:41.964578+0000","last_clean":"2023-10-09T06:34:41.964578+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:34:41.964578+0000","last_undegraded":"2023-10-09T06:34:41.964578+0000","last_fullsized":"2023-10-09T06:34:41.964578+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"419'11963","reported_seq":12029,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.347082+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:34:52.347082+0000","last_peered":"2023-10-09T06:34:52.347082+0000","last_clean":"2023-10-09T06:34:52.347082+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:34:52.347082+0000","last_undegraded":"2023-10-09T06:34:52.347082+0000","last_fullsized":"2023-10-09T06:34:52.347082+0000","mapping_epoch":361,"log_start":"392'6100","ondisk_log_start":"392'6100","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5863,"log_dups_size":0,"ondisk_log_size":5863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11963,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":761,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945597+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:34:41.945597+0000","last_peered":"2023-10-09T06:34:41.945597+0000","last_clean":"2023-10-09T06:34:41.945597+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:34:41.945597+0000","last_undegraded":"2023-10-09T06:34:41.945597+0000","last_fullsized":"2023-10-09T06:34:41.945597+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":409,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960106+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:34:41.960106+0000","last_peered":"2023-10-09T06:34:41.960106+0000","last_clean":"2023-10-09T06:34:41.960106+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:34:41.960106+0000","last_undegraded":"2023-10-09T06:34:41.960106+0000","last_fullsized":"2023-10-09T06:34:41.960106+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":396,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.959832+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:34:41.959832+0000","last_peered":"2023-10-09T06:34:41.959832+0000","last_clean":"2023-10-09T06:34:41.959832+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:34:41.959832+0000","last_undegraded":"2023-10-09T06:34:41.959832+0000","last_fullsized":"2023-10-09T06:34:41.959832+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":419,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.652302+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:34:42.652302+0000","last_peered":"2023-10-09T06:34:42.652302+0000","last_clean":"2023-10-09T06:34:42.652302+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:34:42.652302+0000","last_undegraded":"2023-10-09T06:34:42.652302+0000","last_fullsized":"2023-10-09T06:34:42.652302+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"419'12075","reported_seq":12142,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.388753+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:34:52.388753+0000","last_peered":"2023-10-09T06:34:52.388753+0000","last_clean":"2023-10-09T06:34:52.388753+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:34:52.388753+0000","last_undegraded":"2023-10-09T06:34:52.388753+0000","last_fullsized":"2023-10-09T06:34:52.388753+0000","mapping_epoch":361,"log_start":"399'6649","ondisk_log_start":"399'6649","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5426,"log_dups_size":0,"ondisk_log_size":5426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35258392,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12075,"num_write_kb":403585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":106,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934347+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:34:41.934347+0000","last_peered":"2023-10-09T06:34:41.934347+0000","last_clean":"2023-10-09T06:34:41.934347+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:34:41.934347+0000","last_undegraded":"2023-10-09T06:34:41.934347+0000","last_fullsized":"2023-10-09T06:34:41.934347+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":742,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.944700+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:34:41.944700+0000","last_peered":"2023-10-09T06:34:41.944700+0000","last_clean":"2023-10-09T06:34:41.944700+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:34:41.944700+0000","last_undegraded":"2023-10-09T06:34:41.944700+0000","last_fullsized":"2023-10-09T06:34:41.944700+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961185+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:34:41.961185+0000","last_peered":"2023-10-09T06:34:41.961185+0000","last_clean":"2023-10-09T06:34:41.961185+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:34:41.961185+0000","last_undegraded":"2023-10-09T06:34:41.961185+0000","last_fullsized":"2023-10-09T06:34:41.961185+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":381,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948954+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:34:41.948954+0000","last_peered":"2023-10-09T06:34:41.948954+0000","last_clean":"2023-10-09T06:34:41.948954+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:34:41.948954+0000","last_undegraded":"2023-10-09T06:34:41.948954+0000","last_fullsized":"2023-10-09T06:34:41.948954+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"419'12207","reported_seq":12275,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:55.974882+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:34:55.974882+0000","last_peered":"2023-10-09T06:34:55.974882+0000","last_clean":"2023-10-09T06:34:55.974882+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:34:55.974882+0000","last_undegraded":"2023-10-09T06:34:55.974882+0000","last_fullsized":"2023-10-09T06:34:55.974882+0000","mapping_epoch":361,"log_start":"387'5600","ondisk_log_start":"387'5600","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12207,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":727,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960062+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:34:41.960062+0000","last_peered":"2023-10-09T06:34:41.960062+0000","last_clean":"2023-10-09T06:34:41.960062+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:34:41.960062+0000","last_undegraded":"2023-10-09T06:34:41.960062+0000","last_fullsized":"2023-10-09T06:34:41.960062+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":282,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.944564+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:34:41.944564+0000","last_peered":"2023-10-09T06:34:41.944564+0000","last_clean":"2023-10-09T06:34:41.944564+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:34:41.944564+0000","last_undegraded":"2023-10-09T06:34:41.944564+0000","last_fullsized":"2023-10-09T06:34:41.944564+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930949+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:34:41.930949+0000","last_peered":"2023-10-09T06:34:41.930949+0000","last_clean":"2023-10-09T06:34:41.930949+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:34:41.930949+0000","last_undegraded":"2023-10-09T06:34:41.930949+0000","last_fullsized":"2023-10-09T06:34:41.930949+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":403,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961680+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:34:41.961680+0000","last_peered":"2023-10-09T06:34:41.961680+0000","last_clean":"2023-10-09T06:34:41.961680+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:34:41.961680+0000","last_undegraded":"2023-10-09T06:34:41.961680+0000","last_fullsized":"2023-10-09T06:34:41.961680+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"419'11755","reported_seq":11822,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:53.294703+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:34:53.294703+0000","last_peered":"2023-10-09T06:34:53.294703+0000","last_clean":"2023-10-09T06:34:53.294703+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:34:53.294703+0000","last_undegraded":"2023-10-09T06:34:53.294703+0000","last_fullsized":"2023-10-09T06:34:53.294703+0000","mapping_epoch":361,"log_start":"403'6509","ondisk_log_start":"403'6509","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5246,"log_dups_size":0,"ondisk_log_size":5246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32047104,"num_objects":489,"num_object_clones":0,"num_object_copies":1467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11755,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":741,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.948867+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:34:41.948867+0000","last_peered":"2023-10-09T06:34:41.948867+0000","last_clean":"2023-10-09T06:34:41.948867+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:34:41.948867+0000","last_undegraded":"2023-10-09T06:34:41.948867+0000","last_fullsized":"2023-10-09T06:34:41.948867+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":282,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.946310+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:34:41.946310+0000","last_peered":"2023-10-09T06:34:41.946310+0000","last_clean":"2023-10-09T06:34:41.946310+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:34:41.946310+0000","last_undegraded":"2023-10-09T06:34:41.946310+0000","last_fullsized":"2023-10-09T06:34:41.946310+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":414,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.964506+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:34:41.964506+0000","last_peered":"2023-10-09T06:34:41.964506+0000","last_clean":"2023-10-09T06:34:41.964506+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:34:41.964506+0000","last_undegraded":"2023-10-09T06:34:41.964506+0000","last_fullsized":"2023-10-09T06:34:41.964506+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":401,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.962377+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:34:41.962377+0000","last_peered":"2023-10-09T06:34:41.962377+0000","last_clean":"2023-10-09T06:34:41.962377+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:34:41.962377+0000","last_undegraded":"2023-10-09T06:34:41.962377+0000","last_fullsized":"2023-10-09T06:34:41.962377+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"419'8880","reported_seq":8958,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:53.252517+0000","last_change":"2023-10-09T06:32:47.459790+0000","last_active":"2023-10-09T06:34:53.252517+0000","last_peered":"2023-10-09T06:34:53.252517+0000","last_clean":"2023-10-09T06:34:53.252517+0000","last_became_active":"2023-10-09T06:32:33.070761+0000","last_became_peered":"2023-10-09T06:32:33.070761+0000","last_unstale":"2023-10-09T06:34:53.252517+0000","last_undegraded":"2023-10-09T06:34:53.252517+0000","last_fullsized":"2023-10-09T06:34:53.252517+0000","mapping_epoch":361,"log_start":"366'1790","ondisk_log_start":"366'1790","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7090,"log_dups_size":0,"ondisk_log_size":7090,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10944512,"num_objects":167,"num_object_clones":0,"num_object_copies":501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6015,"num_write_kb":197824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":742,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931219+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:34:41.931219+0000","last_peered":"2023-10-09T06:34:41.931219+0000","last_clean":"2023-10-09T06:34:41.931219+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:34:41.931219+0000","last_undegraded":"2023-10-09T06:34:41.931219+0000","last_fullsized":"2023-10-09T06:34:41.931219+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":149,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960957+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:34:41.960957+0000","last_peered":"2023-10-09T06:34:41.960957+0000","last_clean":"2023-10-09T06:34:41.960957+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:34:41.960957+0000","last_undegraded":"2023-10-09T06:34:41.960957+0000","last_fullsized":"2023-10-09T06:34:41.960957+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":382,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:42.652257+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:34:42.652257+0000","last_peered":"2023-10-09T06:34:42.652257+0000","last_clean":"2023-10-09T06:34:42.652257+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:34:42.652257+0000","last_undegraded":"2023-10-09T06:34:42.652257+0000","last_fullsized":"2023-10-09T06:34:42.652257+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":137,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.959673+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:34:41.959673+0000","last_peered":"2023-10-09T06:34:41.959673+0000","last_clean":"2023-10-09T06:34:41.959673+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:34:41.959673+0000","last_undegraded":"2023-10-09T06:34:41.959673+0000","last_fullsized":"2023-10-09T06:34:41.959673+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"419'8790","reported_seq":8878,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.305183+0000","last_change":"2023-10-09T06:34:47.445399+0000","last_active":"2023-10-09T06:34:52.305183+0000","last_peered":"2023-10-09T06:34:52.305183+0000","last_clean":"2023-10-09T06:34:52.305183+0000","last_became_active":"2023-10-09T06:32:33.044852+0000","last_became_peered":"2023-10-09T06:32:33.044852+0000","last_unstale":"2023-10-09T06:34:52.305183+0000","last_undegraded":"2023-10-09T06:34:52.305183+0000","last_fullsized":"2023-10-09T06:34:52.305183+0000","mapping_epoch":361,"log_start":"365'355","ondisk_log_start":"365'355","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8435,"log_dups_size":0,"ondisk_log_size":8435,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17268736,"num_objects":263,"num_object_clones":0,"num_object_copies":789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5921,"num_write_kb":197920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":771,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960846+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:34:41.960846+0000","last_peered":"2023-10-09T06:34:41.960846+0000","last_clean":"2023-10-09T06:34:41.960846+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:34:41.960846+0000","last_undegraded":"2023-10-09T06:34:41.960846+0000","last_fullsized":"2023-10-09T06:34:41.960846+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":403,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.949025+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:34:41.949025+0000","last_peered":"2023-10-09T06:34:41.949025+0000","last_clean":"2023-10-09T06:34:41.949025+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:34:41.949025+0000","last_undegraded":"2023-10-09T06:34:41.949025+0000","last_fullsized":"2023-10-09T06:34:41.949025+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":384,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931309+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:34:41.931309+0000","last_peered":"2023-10-09T06:34:41.931309+0000","last_clean":"2023-10-09T06:34:41.931309+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:34:41.931309+0000","last_undegraded":"2023-10-09T06:34:41.931309+0000","last_fullsized":"2023-10-09T06:34:41.931309+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":382,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.929996+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:34:41.929996+0000","last_peered":"2023-10-09T06:34:41.929996+0000","last_clean":"2023-10-09T06:34:41.929996+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:34:41.929996+0000","last_undegraded":"2023-10-09T06:34:41.929996+0000","last_fullsized":"2023-10-09T06:34:41.929996+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"419'8772","reported_seq":8861,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.090927+0000","last_change":"2023-10-09T06:34:16.084491+0000","last_active":"2023-10-09T06:34:52.090927+0000","last_peered":"2023-10-09T06:34:52.090927+0000","last_clean":"2023-10-09T06:34:52.090927+0000","last_became_active":"2023-10-09T06:32:33.005074+0000","last_became_peered":"2023-10-09T06:32:33.005074+0000","last_unstale":"2023-10-09T06:34:52.090927+0000","last_undegraded":"2023-10-09T06:34:52.090927+0000","last_fullsized":"2023-10-09T06:34:52.090927+0000","mapping_epoch":390,"log_start":"365'239","ondisk_log_start":"365'239","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8533,"log_dups_size":0,"ondisk_log_size":8533,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19398656,"num_objects":296,"num_object_clones":0,"num_object_copies":888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5900,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":698,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:43.280827+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:34:43.280827+0000","last_peered":"2023-10-09T06:34:43.280827+0000","last_clean":"2023-10-09T06:34:43.280827+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:34:43.280827+0000","last_undegraded":"2023-10-09T06:34:43.280827+0000","last_fullsized":"2023-10-09T06:34:43.280827+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":385,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.962302+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:34:41.962302+0000","last_peered":"2023-10-09T06:34:41.962302+0000","last_clean":"2023-10-09T06:34:41.962302+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:34:41.962302+0000","last_undegraded":"2023-10-09T06:34:41.962302+0000","last_fullsized":"2023-10-09T06:34:41.962302+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":407,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.964430+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:34:41.964430+0000","last_peered":"2023-10-09T06:34:41.964430+0000","last_clean":"2023-10-09T06:34:41.964430+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:34:41.964430+0000","last_undegraded":"2023-10-09T06:34:41.964430+0000","last_fullsized":"2023-10-09T06:34:41.964430+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.13","version":"419'8920","reported_seq":14670,"reported_epoch":419,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:55.995179+0000","last_change":"2023-10-09T06:34:03.204624+0000","last_active":"2023-10-09T06:34:55.995179+0000","last_peered":"2023-10-09T06:34:55.995179+0000","last_clean":"2023-10-09T06:32:51.037365+0000","last_became_active":"2023-10-09T06:32:53.531678+0000","last_became_peered":"2023-10-09T06:32:53.531678+0000","last_unstale":"2023-10-09T06:34:55.995179+0000","last_undegraded":"2023-10-09T06:32:53.443414+0000","last_fullsized":"2023-10-09T06:32:53.427850+0000","mapping_epoch":402,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8920,"log_dups_size":0,"ondisk_log_size":8920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13795328,"num_objects":211,"num_object_clones":0,"num_object_copies":633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6029,"num_write_kb":199648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2833,"num_bytes_recovered":90112000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,7","objects":-77}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":628,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961661+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:34:41.961661+0000","last_peered":"2023-10-09T06:34:41.961661+0000","last_clean":"2023-10-09T06:34:41.961661+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:34:41.961661+0000","last_undegraded":"2023-10-09T06:34:41.961661+0000","last_fullsized":"2023-10-09T06:34:41.961661+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":110,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.935098+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:34:41.935098+0000","last_peered":"2023-10-09T06:34:41.935098+0000","last_clean":"2023-10-09T06:34:41.935098+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:34:41.935098+0000","last_undegraded":"2023-10-09T06:34:41.935098+0000","last_fullsized":"2023-10-09T06:34:41.935098+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":426,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931123+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:34:41.931123+0000","last_peered":"2023-10-09T06:34:41.931123+0000","last_clean":"2023-10-09T06:34:41.931123+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:34:41.931123+0000","last_undegraded":"2023-10-09T06:34:41.931123+0000","last_fullsized":"2023-10-09T06:34:41.931123+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"419'8901","reported_seq":17167,"reported_epoch":419,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:34:53.279206+0000","last_change":"2023-10-09T06:32:52.162166+0000","last_active":"2023-10-09T06:34:53.279206+0000","last_peered":"2023-10-09T06:34:53.279206+0000","last_clean":"2023-10-09T06:32:48.657566+0000","last_became_active":"2023-10-09T06:32:52.155917+0000","last_became_peered":"2023-10-09T06:32:52.155917+0000","last_unstale":"2023-10-09T06:34:53.279206+0000","last_undegraded":"2023-10-09T06:32:52.150896+0000","last_fullsized":"2023-10-09T06:32:52.051991+0000","mapping_epoch":400,"log_start":"366'1647","ondisk_log_start":"366'1647","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7254,"log_dups_size":0,"ondisk_log_size":7254,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16744448,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":810,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5998,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2728,"num_bytes_recovered":178782208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"1,3","objects":-6},{"shards":"3,4,7","objects":262}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":110,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.934573+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:34:41.934573+0000","last_peered":"2023-10-09T06:34:41.934573+0000","last_clean":"2023-10-09T06:34:41.934573+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:34:41.934573+0000","last_undegraded":"2023-10-09T06:34:41.934573+0000","last_fullsized":"2023-10-09T06:34:41.934573+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":436,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.930860+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:34:41.930860+0000","last_peered":"2023-10-09T06:34:41.930860+0000","last_clean":"2023-10-09T06:34:41.930860+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:34:41.930860+0000","last_undegraded":"2023-10-09T06:34:41.930860+0000","last_fullsized":"2023-10-09T06:34:41.930860+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"419'8898","reported_seq":18379,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:52.311094+0000","last_change":"2023-10-09T06:34:18.854131+0000","last_active":"2023-10-09T06:34:52.311094+0000","last_peered":"2023-10-09T06:34:52.311094+0000","last_clean":"2023-10-09T06:34:52.311094+0000","last_became_active":"2023-10-09T06:33:43.622574+0000","last_became_peered":"2023-10-09T06:33:43.622574+0000","last_unstale":"2023-10-09T06:34:52.311094+0000","last_undegraded":"2023-10-09T06:34:52.311094+0000","last_fullsized":"2023-10-09T06:34:52.311094+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16908288,"num_objects":258,"num_object_clones":0,"num_object_copies":774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":196673536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":775,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.945693+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:34:41.945693+0000","last_peered":"2023-10-09T06:34:41.945693+0000","last_clean":"2023-10-09T06:34:41.945693+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:34:41.945693+0000","last_undegraded":"2023-10-09T06:34:41.945693+0000","last_fullsized":"2023-10-09T06:34:41.945693+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":119,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.961858+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:34:41.961858+0000","last_peered":"2023-10-09T06:34:41.961858+0000","last_clean":"2023-10-09T06:34:41.961858+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:34:41.961858+0000","last_undegraded":"2023-10-09T06:34:41.961858+0000","last_fullsized":"2023-10-09T06:34:41.961858+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.960758+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:34:41.960758+0000","last_peered":"2023-10-09T06:34:41.960758+0000","last_clean":"2023-10-09T06:34:41.960758+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:34:41.960758+0000","last_undegraded":"2023-10-09T06:34:41.960758+0000","last_fullsized":"2023-10-09T06:34:41.960758+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"419'8857","reported_seq":21132,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-10-09T06:34:55.888657+0000","last_change":"2023-10-09T06:34:05.191255+0000","last_active":"2023-10-09T06:34:55.888657+0000","last_peered":"2023-10-09T06:34:55.888657+0000","last_clean":"2023-10-09T06:34:55.888657+0000","last_became_active":"2023-10-09T06:34:05.174880+0000","last_became_peered":"2023-10-09T06:34:05.174880+0000","last_unstale":"2023-10-09T06:34:55.888657+0000","last_undegraded":"2023-10-09T06:34:55.888657+0000","last_fullsized":"2023-10-09T06:34:55.888657+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18251776,"num_objects":279,"num_object_clones":0,"num_object_copies":837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5969,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3046,"num_bytes_recovered":199622656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":540,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:47.654766+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:34:47.654766+0000","last_peered":"2023-10-09T06:34:47.654766+0000","last_clean":"2023-10-09T06:34:47.654766+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:34:47.654766+0000","last_undegraded":"2023-10-09T06:34:47.654766+0000","last_fullsized":"2023-10-09T06:34:47.654766+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":414,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.931336+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:34:41.931336+0000","last_peered":"2023-10-09T06:34:41.931336+0000","last_clean":"2023-10-09T06:34:41.931336+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:34:41.931336+0000","last_undegraded":"2023-10-09T06:34:41.931336+0000","last_fullsized":"2023-10-09T06:34:41.931336+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":379,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-10-09T06:34:41.929619+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:34:41.929619+0000","last_peered":"2023-10-09T06:34:41.929619+0000","last_clean":"2023-10-09T06:34:41.929619+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:34:41.929619+0000","last_undegraded":"2023-10-09T06:34:41.929619+0000","last_fullsized":"2023-10-09T06:34:41.929619+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":540573720,"num_objects":8250,"num_object_clones":0,"num_object_copies":24750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1098,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8250,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":185992,"num_write_kb":6175969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11737,"num_bytes_recovered":665387008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2461806592,"data_stored":2461794376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203294,"ondisk_log_size":203294,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234451,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389076,"kb_used_data":488844,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92356076,"statfs":{"total":95995035648,"available":94572621824,"internally_reserved":0,"allocated":500576256,"data_stored":499538517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":5.7720000000000002,"15min":6.1849999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":25.225000000000001,"5min":196.28899999999999,"15min":219.41900000000001},"last":1.8480000000000001},{"interface":"front","average":{"1min":3.097,"5min":5.9139999999999997,"15min":6.3200000000000003},"min":{"1min":0.27700000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":25.248000000000001,"5min":196.333,"15min":219.399},"last":2.2709999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":3.8900000000000001,"15min":4.3540000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":6.3920000000000003,"5min":107.87,"15min":215.61699999999999},"last":2.1789999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":3.3660000000000001,"15min":4.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":8.0830000000000002,"5min":107.904,"15min":216.22200000000001},"last":1.5209999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":4.2050000000000001,"15min":6.9450000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":39.747,"5min":198.94200000000001,"15min":216.34299999999999},"last":4.0250000000000004},{"interface":"front","average":{"1min":4.4509999999999996,"5min":4.2679999999999998,"15min":7.0730000000000004},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":39.658999999999999,"5min":198.92099999999999,"15min":216.25200000000001},"last":3.8900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":6.0700000000000003,"15min":4.6120000000000001},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":4.9809999999999999,"5min":187.34200000000001,"15min":215.922},"last":2.306},{"interface":"front","average":{"1min":1.929,"5min":6.0670000000000002,"15min":4.6959999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.9189999999999996,"5min":187.08199999999999,"15min":215.94900000000001},"last":3}]},{"osd":4,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.79600000000000004,"15min":2.4729999999999999},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":0.56999999999999995,"5min":6.1689999999999996,"15min":218.446},"last":1.587},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.74299999999999999,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":1.7190000000000001,"5min":4.891,"15min":216.172},"last":2.331}]},{"osd":5,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.86699999999999999,"15min":2.4620000000000002},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":4.2309999999999999,"5min":8.8170000000000002,"15min":216.37200000000001},"last":2.1299999999999999},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.087,"15min":2.3889999999999998},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":2.1000000000000001,"5min":6.3019999999999996,"15min":218.15899999999999},"last":1.5569999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.123,"15min":2.2909999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":2.6560000000000001,"5min":15.162000000000001,"15min":218.19999999999999},"last":2.4369999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.89000000000000001,"15min":2.0979999999999999},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":1.7589999999999999,"5min":8.2919999999999998,"15min":218.05500000000001},"last":2.3580000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496087,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1283768,"kb_used_data":380656,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":92461384,"statfs":{"total":95995035648,"available":94680457216,"internally_reserved":0,"allocated":389791744,"data_stored":388781375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":3.577,"15min":3.7759999999999998},"min":{"1min":0.22500000000000001,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":47.363999999999997,"5min":75.061999999999998,"15min":110.926},"last":3.3279999999999998},{"interface":"front","average":{"1min":4.7809999999999997,"5min":3.73,"15min":4.0439999999999996},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":47.381,"5min":75.021000000000001,"15min":110.911},"last":1.1040000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":3.3300000000000001,"15min":5.0759999999999996},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.7949999999999999,"5min":82.159000000000006,"15min":292.92200000000003},"last":5.6980000000000004},{"interface":"front","average":{"1min":1.853,"5min":3.5569999999999999,"15min":4.9569999999999999},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":9.2560000000000002,"5min":82.174000000000007,"15min":292.904},"last":5.7190000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.8220000000000001,"15min":3.1840000000000002},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.25700000000000001},"max":{"1min":47.484000000000002,"5min":74.748999999999995,"15min":75.191999999999993},"last":2.0870000000000002},{"interface":"front","average":{"1min":4.5880000000000001,"5min":3.4820000000000002,"15min":2.9079999999999999},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":47.494999999999997,"5min":74.733999999999995,"15min":75.120000000000005},"last":3.4209999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":5.3849999999999998,"15min":3.9159999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.761000000000003,"5min":91.174000000000007,"15min":110.358},"last":3.3580000000000001},{"interface":"front","average":{"1min":4.1799999999999997,"5min":5.3019999999999996,"15min":3.9209999999999998},"min":{"1min":0.38200000000000001,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":47.606000000000002,"5min":91.438999999999993,"15min":110.43899999999999},"last":3.298}]},{"osd":4,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.365,"15min":1.278},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.2690000000000001,"5min":12.666,"15min":17.643999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.1000000000000001,"15min":1.1259999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.223},"max":{"1min":6.7859999999999996,"5min":8.6059999999999999,"15min":10.563000000000001},"last":0.437}]},{"osd":5,"last update":"Mon Oct 9 06:33:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.2769999999999999,"15min":1.377},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29199999999999998},"max":{"1min":5.7539999999999996,"5min":12.747999999999999,"15min":22.379999999999999},"last":3.653},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.5620000000000001,"15min":1.383},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":6.8140000000000001,"5min":16.536999999999999,"15min":24.401},"last":0.58399999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.204,"15min":2.528},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":19.295999999999999,"15min":122.937},"last":3.7599999999999998},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.4690000000000001,"15min":2.673},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4049999999999998,"5min":19.584,"15min":122.92100000000001},"last":0.61299999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402824,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1008336,"kb_used_data":220232,"kb_used_omap":4,"kb_used_meta":788091,"kb_avail":92736816,"statfs":{"total":95995035648,"available":94962499584,"internally_reserved":0,"allocated":225517568,"data_stored":223608261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":807006094},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":6.2249999999999996,"15min":6.3029999999999999},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":19.699999999999999,"5min":143.465,"15min":269.79199999999997},"last":0.46100000000000002},{"interface":"front","average":{"1min":4.1660000000000004,"5min":6.2460000000000004,"15min":6.7720000000000002},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.17899999999999999},"max":{"1min":11.663,"5min":143.66399999999999,"15min":271.20999999999998},"last":2.1970000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:45 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":4.0869999999999997,"15min":3.6629999999999998},"min":{"1min":0.23400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":13.217000000000001,"5min":138.452,"15min":139.75399999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.3069999999999999,"5min":4.21,"15min":4.4119999999999999},"min":{"1min":0.254,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.244,"5min":138.47499999999999,"15min":140.29900000000001},"last":2.7930000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":3.2890000000000001,"15min":3.9620000000000002},"min":{"1min":0.28499999999999998,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":8.9420000000000002,"5min":27.064,"15min":153.18199999999999},"last":2.5129999999999999},{"interface":"front","average":{"1min":2.891,"5min":2.1920000000000002,"15min":2.819},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":13.129,"5min":26.626999999999999,"15min":152.52000000000001},"last":0.51500000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6739999999999999,"15min":3.7789999999999999},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.6159999999999997,"5min":38.835999999999999,"15min":199.43799999999999},"last":2.3069999999999999},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.895,"15min":3.4140000000000001},"min":{"1min":0.36599999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":8.8089999999999993,"5min":38.819000000000003,"15min":199.32900000000001},"last":2.2850000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.5750000000000002,"15min":2.339},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.163,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.5619999999999998,"15min":2.7040000000000002},"min":{"1min":0.40999999999999998,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":8.7759999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":2.2509999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":3.6949999999999998,"15min":3.1890000000000001},"min":{"1min":0.39900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.9250000000000007,"5min":107.27200000000001,"15min":107.27200000000001},"last":2.3450000000000002},{"interface":"front","average":{"1min":2.52,"5min":4.452,"15min":3.4700000000000002},"min":{"1min":0.379,"5min":0.316,"15min":0.316},"max":{"1min":7.6950000000000003,"5min":107.252,"15min":107.252},"last":2.3929999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":4.2190000000000003,"15min":3.6389999999999998},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":7.6529999999999996,"5min":104.07299999999999,"15min":104.07299999999999},"last":2.359},{"interface":"front","average":{"1min":2.7309999999999999,"5min":3.4750000000000001,"15min":3.0110000000000001},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.191000000000001,"5min":104.056,"15min":104.056},"last":0.56100000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024258,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1133644,"kb_used_data":192004,"kb_used_omap":5,"kb_used_meta":941626,"kb_avail":92611508,"statfs":{"total":95995035648,"available":94834184192,"internally_reserved":0,"allocated":196612096,"data_stored":194417866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":964225658},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.9930000000000001,"15min":3.4260000000000002},"min":{"1min":0.19900000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":33.091999999999999,"5min":33.091999999999999,"15min":191.446},"last":3.774},{"interface":"front","average":{"1min":4.25,"5min":2.6120000000000001,"15min":3.7109999999999999},"min":{"1min":0.23799999999999999,"5min":0.187,"15min":0.187},"max":{"1min":24.838999999999999,"5min":24.838999999999999,"15min":191.393},"last":7.4370000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:34:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4199999999999999,"5min":2.8119999999999998,"15min":2.6429999999999998},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":12.532,"5min":16.936,"15min":57.162999999999997},"last":10.929},{"interface":"front","average":{"1min":4.133,"5min":3.524,"15min":3.121},"min":{"1min":0.22800000000000001,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":13.579000000000001,"5min":15.794,"15min":57.585000000000001},"last":3.8159999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:11 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":6.5800000000000001,"15min":5.9820000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":15.413,"5min":187.172,"15min":208.11199999999999},"last":3.879},{"interface":"front","average":{"1min":3.9609999999999999,"5min":6.2960000000000003,"15min":5.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":15.266,"5min":194.851,"15min":208.25899999999999},"last":8.3849999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.78,"15min":1.593},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":5.6059999999999999,"5min":9.7159999999999993,"15min":11.162000000000001},"last":3.6840000000000002},{"interface":"front","average":{"1min":2.2749999999999999,"5min":2.2509999999999999,"15min":1.929},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.29499999999999998},"max":{"1min":8.2880000000000003,"5min":18.081,"15min":18.081},"last":7.5469999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.883,"15min":1.907},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":6.2809999999999997,"5min":9.7720000000000002,"15min":62.984999999999999},"last":3.7440000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.492,"15min":2.3450000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":6.1529999999999996,"5min":15.747999999999999,"15min":64.983999999999995},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":3.4790000000000001,"15min":2.9540000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":6.2069999999999999,"5min":85.525999999999996,"15min":120.572},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.4890000000000001,"5min":2.8370000000000002,"15min":2.4460000000000002},"min":{"1min":0.35499999999999998,"5min":0.33700000000000002,"15min":0.30299999999999999},"max":{"1min":5.6440000000000001,"5min":85.734999999999999,"15min":120.64400000000001},"last":3.7130000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:34:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.5299999999999998,"15min":2.6579999999999999},"min":{"1min":0.45600000000000002,"5min":0.376,"15min":0.32300000000000001},"max":{"1min":8.3249999999999993,"5min":18.041,"15min":122.018},"last":7.516},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.9119999999999999,"15min":2.972},"min":{"1min":0.39400000000000002,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":7.7089999999999996,"5min":26.416,"15min":122.32299999999999},"last":0.51400000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814127,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":988680,"kb_used_data":259328,"kb_used_omap":7,"kb_used_meta":729272,"kb_avail":92756472,"statfs":{"total":95995035648,"available":94982627328,"internally_reserved":0,"allocated":265551872,"data_stored":263706090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":746774609},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":15.65,"5min":7.032,"15min":5.5129999999999999},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":156.94499999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":8.1940000000000008},{"interface":"front","average":{"1min":15.272,"5min":6.734,"15min":5.0049999999999999},"min":{"1min":0.315,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":156.922,"5min":156.922,"15min":168.49700000000001},"last":2.137}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":14.833,"5min":6.202,"15min":4.7000000000000002},"min":{"1min":0.41499999999999998,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":140.452,"5min":140.452,"15min":205.148},"last":8.2319999999999993},{"interface":"front","average":{"1min":14.69,"5min":5.9409999999999998,"15min":4.8739999999999997},"min":{"1min":0.35599999999999998,"5min":0.183,"15min":0.183},"max":{"1min":139.75299999999999,"5min":139.75299999999999,"15min":205.267},"last":1.6299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2880000000000003,"5min":3.968,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":75.498999999999995,"5min":77.692999999999998,"15min":91.644000000000005},"last":2.0960000000000001},{"interface":"front","average":{"1min":6.2220000000000004,"5min":4.3140000000000001,"15min":3.9409999999999998},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":69.784000000000006,"5min":78.575999999999993,"15min":91.685000000000002},"last":1.0249999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.0249999999999999,"15min":2.27},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":3.423,"5min":44.314,"15min":90.938999999999993},"last":1.0509999999999999},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.4329999999999998,"15min":2.6019999999999999},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":4.6950000000000003,"5min":43.896000000000001,"15min":90.950000000000003},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.794,"15min":1.675},"min":{"1min":0.379,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7250000000000001,"5min":13.102,"15min":17.800000000000001},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.958,"5min":2.0009999999999999,"15min":1.9630000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.6779999999999999,"5min":11.285,"15min":11.285},"last":0.73999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":4.524,"15min":3.4060000000000001},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":20.445,"5min":123.47799999999999,"15min":138.291},"last":0.76700000000000002},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.3840000000000003,"15min":3.202},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.148,"5min":122.517,"15min":138.01599999999999},"last":8}]},{"osd":7,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":2.7839999999999998,"15min":2.3820000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":16.574999999999999,"5min":47.389000000000003,"15min":55.851999999999997},"last":8.0519999999999996},{"interface":"front","average":{"1min":1.444,"5min":2.3570000000000002,"15min":2.1549999999999998},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":3.3330000000000002,"5min":47.206000000000003,"15min":55.731000000000002},"last":2.367}]}]},{"osd":3,"up_from":149,"seq":639950127324,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111928,"kb_used_data":455664,"kb_used_omap":2,"kb_used_meta":656253,"kb_avail":92633224,"statfs":{"total":95995035648,"available":94856421376,"internally_reserved":0,"allocated":466599936,"data_stored":464829722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":672003125},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":7.1669999999999998,"15min":5.8540000000000001},"min":{"1min":0.20000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":22.367000000000001,"5min":161.821,"15min":187.852},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.3140000000000001,"5min":6.1689999999999996,"15min":5.8490000000000002},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":13.443,"5min":163.52000000000001,"15min":187.923},"last":2.3759999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9209999999999998,"5min":4.4930000000000003,"15min":4.375},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":22.349,"5min":155.23099999999999,"15min":187.89099999999999},"last":2.4249999999999998},{"interface":"front","average":{"1min":4.1050000000000004,"5min":6,"15min":4.2480000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":15.066000000000001,"5min":155.24799999999999,"15min":155.24799999999999},"last":7.601}]},{"osd":2,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.7329999999999997,"15min":4.234},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":22.297000000000001,"5min":167.88499999999999,"15min":187.76400000000001},"last":2.4470000000000001},{"interface":"front","average":{"1min":3.5720000000000001,"5min":5.4390000000000001,"15min":4.2380000000000004},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":13.631,"5min":168.16900000000001,"15min":187.79499999999999},"last":8.3179999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.9159999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.661,"5min":48.249000000000002,"15min":48.249000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.887,"15min":2.645},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":16.516999999999999,"5min":48.031999999999996,"15min":187.67400000000001},"last":2.516}]},{"osd":5,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.7389999999999999,"15min":2.7250000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":16.494,"5min":48.162999999999997,"15min":187.833},"last":2.6120000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.1520000000000001,"15min":2.4300000000000002},"min":{"1min":0.42299999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":18.835999999999999,"5min":48.106000000000002,"15min":187.952},"last":2.504}]},{"osd":6,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":4.3339999999999996,"15min":3.3319999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.462,"5min":130.39599999999999,"15min":187.78200000000001},"last":2.5590000000000002},{"interface":"front","average":{"1min":2.3199999999999998,"5min":3.8679999999999999,"15min":2.9900000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":22.260000000000002,"5min":130.67699999999999,"15min":187.81800000000001},"last":2.73}]},{"osd":7,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":4.5229999999999997,"15min":3.3769999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":13.695,"5min":121.613,"15min":192.20400000000001},"last":7.5519999999999996},{"interface":"front","average":{"1min":3.0110000000000001,"5min":4.7270000000000003,"15min":3.4569999999999999},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.542000000000002,"5min":121.559,"15min":187.86500000000001},"last":2.5870000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019360,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1277952,"kb_used_data":387064,"kb_used_omap":0,"kb_used_meta":890880,"kb_avail":92467200,"statfs":{"total":95995035648,"available":94686412800,"internally_reserved":0,"allocated":396353536,"data_stored":395335352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912261120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":11.006,"5min":11.769,"15min":5.6020000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":118.98999999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":11.007,"5min":11.597,"15min":5.8220000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":118.956,"5min":287.57100000000003,"15min":287.57100000000003},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":10.643000000000001,"5min":5.5620000000000003,"15min":3.4590000000000001},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":134.37100000000001,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.373},{"interface":"front","average":{"1min":10.766,"5min":5.7549999999999999,"15min":3.669},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":134.21700000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.28499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":5.3170000000000002,"15min":3.7749999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":148.36099999999999,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":12.159000000000001,"5min":5.2990000000000004,"15min":3.7559999999999998},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":145.755,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.35199999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":8.2629999999999999,"5min":10.750999999999999,"15min":5.8639999999999999},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":65.915000000000006,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":8.6899999999999995,"5min":10.590999999999999,"15min":5.5880000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":68.054000000000002,"5min":161.934,"15min":161.934},"last":0.40000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.70699999999999996,"15min":0.98999999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80600000000000005,"5min":6.0199999999999996,"15min":10.673},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.82799999999999996,"15min":1.0029999999999999},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.548,"5min":5.7000000000000002,"15min":12.552},"last":0.29699999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.4620000000000002,"15min":1.2929999999999999},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":2.5179999999999998,"5min":163.36699999999999,"15min":163.36699999999999},"last":13.369},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.641,"15min":1.6419999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":2.9340000000000002,"5min":163.381,"15min":163.381},"last":13.157999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.6499999999999999,"15min":1.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":2.9649999999999999,"5min":70.247,"15min":70.247},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.774,"15min":1.6160000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0680000000000001,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.41899999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757725,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993968,"kb_used_data":100456,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":92751184,"statfs":{"total":95995035648,"available":94977212416,"internally_reserved":0,"allocated":102866944,"data_stored":101853207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":6.1609999999999996,"15min":4.9950000000000001},"min":{"1min":0.29399999999999998,"5min":0.223,"15min":0.216},"max":{"1min":8.6940000000000008,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.47,"5min":5.9009999999999998,"15min":4.8280000000000003},"min":{"1min":0.34999999999999998,"5min":0.24399999999999999,"15min":0.22},"max":{"1min":5.6539999999999999,"5min":294.30399999999997,"15min":294.30399999999997},"last":3.823}]},{"osd":1,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":7.4349999999999996,"5min":4.0570000000000004,"15min":3.274},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":119.024,"5min":119.024,"15min":126.71899999999999},"last":2.827},{"interface":"front","average":{"1min":7.5579999999999998,"5min":4.2779999999999996,"15min":3.4369999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":119.133,"5min":119.133,"15min":126.931},"last":4.0119999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.4609999999999999,"15min":2.4969999999999999},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.5149999999999997,"5min":64.150000000000006,"15min":81.260000000000005},"last":4.0419999999999998},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.2290000000000001,"15min":2.4289999999999998},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":4.8550000000000004,"5min":64.507000000000005,"15min":81.245000000000005},"last":3.9740000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":5.6289999999999996,"15min":4.6790000000000003},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":38.582000000000001,"5min":144.56399999999999,"15min":144.56399999999999},"last":2.8919999999999999},{"interface":"front","average":{"1min":3.9409999999999998,"5min":5.3700000000000001,"15min":4.5099999999999998},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":38.447000000000003,"5min":144.62799999999999,"15min":144.62799999999999},"last":3.891}]},{"osd":4,"last update":"Mon Oct 9 06:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.74199999999999999,"15min":1.1859999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.8700000000000001,"5min":4.9119999999999999,"15min":32.850999999999999},"last":2.8610000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.78900000000000003,"15min":1.044},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":2.8919999999999999,"5min":6.2610000000000001,"15min":32.82},"last":3.863}]},{"osd":6,"last update":"Mon Oct 9 06:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.7250000000000001,"15min":1.6200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7839999999999998,"5min":57.505000000000003,"15min":57.505000000000003},"last":3.9460000000000002},{"interface":"front","average":{"1min":0.63,"5min":1.9810000000000001,"15min":1.7450000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0630000000000002,"5min":55.040999999999997,"15min":55.040999999999997},"last":4.173}]},{"osd":7,"last update":"Mon Oct 9 06:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.147,"15min":1.397},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1829999999999998,"5min":7.4290000000000003,"15min":30.658999999999999},"last":2.9729999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.022,"15min":1.399},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":2.3540000000000001,"5min":6.2320000000000002,"15min":30.683},"last":3.9169999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":183304192,"data_stored":183304192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":217518080,"data_stored":217514008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":250744832,"data_stored":250740760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":447086592,"data_stored":447086592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":391839744,"data_stored":391839744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":92733440,"data_stored":92733440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":382861312,"data_stored":382861312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":495718400,"data_stored":495714328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:34:59.531 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:34:59.554 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:34:59.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:59.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:34:59.659 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:34:59.689 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:59.747 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:34:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:59.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:34:59.852 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:34:59.880 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:59 smithi150 ceph-mon[194178]: 12.11 scrub starts 2023-10-09T06:34:59.880 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:34:59 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:34:59.884 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:34:59.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:59 smithi107 ceph-mon[212750]: 12.11 scrub starts 2023-10-09T06:34:59.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:34:59 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:34:59.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:59 smithi107 ceph-mon[220921]: 12.11 scrub starts 2023-10-09T06:34:59.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:34:59 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:34:59.932 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:34:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:34:59.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:00.011 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:195413 12.13 12:cd6f93d7:::benchmark_data_smithi107_479504_object95802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:00.003546+0000", 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0073957420000000003, 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074276109999999998, 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:35:00.012 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 195413 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003546+0000", 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003546+0000", 2023-10-09T06:35:00.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003545+0000", 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003549+0000", 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0740000000000003e-06 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003557+0000", 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8890000000000002e-06 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003563+0000", 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8270000000000003e-06 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:00.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003647+0000", 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4519999999999997e-05 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003794+0000", 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014684600000000001 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.004403+0000", 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00060835499999999999 2023-10-09T06:35:00.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.010278+0000", 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058756959999999997 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.010426+0000", 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000148073 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:00.017 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:195415 12.10 12:0931d344:::benchmark_data_smithi107_479504_object95804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:00.003574+0000", 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0073681839999999998, 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074913560000000002, 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:35:00.018 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 195415 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003574+0000", 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003574+0000", 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:00.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003573+0000", 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003576+0000", 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3410000000000001e-06 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003581+0000", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3879999999999999e-06 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003586+0000", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.746e-06 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003674+0000", 2023-10-09T06:35:00.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8261000000000001e-05 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.003782+0000", 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010769599999999999 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.004277+0000", 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00049557799999999999 2023-10-09T06:35:00.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.010281+0000", 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0060032890000000002 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:00.010382+0000", 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101699 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:00.023 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:00.024 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:00.024 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:35:00.024 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 2 2023-10-09T06:35:00.024 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:35:00.042 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:00.069 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:00.116 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:00.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:00.221 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:00.257 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:00.321 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:00.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:00.433 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:00.464 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:00.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:00 smithi107 ceph-mon[220921]: pgmap v1296: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 365 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 886/17565 objects degraded (5.044%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:35:00.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:00 smithi107 ceph-mon[220921]: 12.2 scrub starts 2023-10-09T06:35:00.618 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:00 smithi107 ceph-mon[212750]: pgmap v1296: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 365 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 886/17565 objects degraded (5.044%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:35:00.618 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:00 smithi107 ceph-mon[212750]: 12.2 scrub starts 2023-10-09T06:35:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:00 smithi150 ceph-mon[194178]: pgmap v1296: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 365 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 886/17565 objects degraded (5.044%); 1.3 MiB/s, 90 objects/s recovering 2023-10-09T06:35:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:00 smithi150 ceph-mon[194178]: 12.2 scrub starts 2023-10-09T06:35:01.769 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:01.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:01.810 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:01.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:35:01.875 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:01.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:01 smithi107 ceph-mon[220921]: 1.18 deep-scrub starts 2023-10-09T06:35:01.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:01 smithi107 ceph-mon[220921]: 12.14 scrub starts 2023-10-09T06:35:01.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:01 smithi150 ceph-mon[194178]: 1.18 deep-scrub starts 2023-10-09T06:35:01.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:01 smithi150 ceph-mon[194178]: 12.14 scrub starts 2023-10-09T06:35:01.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:01 smithi107 ceph-mon[212750]: 1.18 deep-scrub starts 2023-10-09T06:35:01.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:01 smithi107 ceph-mon[212750]: 12.14 scrub starts 2023-10-09T06:35:01.905 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:01.963 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:01.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:02.068 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:02.072 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:02.148 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:02.208 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:02] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T06:35:02.313 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:02.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:02.422 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:02.461 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:02.764 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:02.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:02.774 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: pgmap v1297: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 300 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 603/14445 objects degraded (4.174%); 1.3 MiB/s, 80 objects/s recovering 2023-10-09T06:35:02.774 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:02.774 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:02.774 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: osdmap e420: 8 total, 8 up, 8 in 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: 12.3 scrub starts 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 603/14445 objects degraded (4.174%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: pgmap v1297: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 300 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 603/14445 objects degraded (4.174%); 1.3 MiB/s, 80 objects/s recovering 2023-10-09T06:35:02.775 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: osdmap e420: 8 total, 8 up, 8 in 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: 12.3 scrub starts 2023-10-09T06:35:02.776 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 603/14445 objects degraded (4.174%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:35:02.874 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: pgmap v1297: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 300 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 603/14445 objects degraded (4.174%); 1.3 MiB/s, 80 objects/s recovering 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: osdmap e420: 8 total, 8 up, 8 in 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: failed to encode map e420 with expected crc 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: 12.3 scrub starts 2023-10-09T06:35:02.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 603/14445 objects degraded (4.174%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:35:02.935 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:03.325 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:03.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:03.441 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:03.541 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:03.675 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: osdmap e421: 8 total, 8 up, 8 in 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 12.10 scrub starts 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 12.18 scrub starts 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 12.10 scrub ok 2023-10-09T06:35:03.726 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1539146579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 1.1b scrub ok 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 12.15 scrub starts 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[212750]: 12.15 scrub ok 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: osdmap e421: 8 total, 8 up, 8 in 2023-10-09T06:35:03.727 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 12.10 scrub starts 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 12.18 scrub starts 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 12.10 scrub ok 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1539146579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 1.1b scrub ok 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 12.15 scrub starts 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:35:03.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:03 smithi107 ceph-mon[220921]: 12.15 scrub ok 2023-10-09T06:35:03.755 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:03.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:03.872 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: osdmap e421: 8 total, 8 up, 8 in 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: failed to encode map e421 with expected crc 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 12.10 scrub starts 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 12.18 scrub starts 2023-10-09T06:35:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 12.10 scrub ok 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1539146579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 1.1b scrub ok 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 12.15 scrub starts 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:35:03.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:03 smithi150 ceph-mon[194178]: 12.15 scrub ok 2023-10-09T06:35:03.905 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:03.963 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:03.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:04.070 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:04.098 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:04.711 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:04.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:04.739 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:04.821 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:04.851 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:04.903 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: pgmap v1300: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 2.0 MiB/s, 125 objects/s recovering 2023-10-09T06:35:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: failed to encode map e422 with expected crc 2023-10-09T06:35:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:35:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: osdmap e422: 8 total, 8 up, 8 in 2023-10-09T06:35:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: failed to encode map e422 with expected crc 2023-10-09T06:35:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: failed to encode map e422 with expected crc 2023-10-09T06:35:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:04 smithi150 ceph-mon[194178]: failed to encode map e422 with expected crc 2023-10-09T06:35:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:04.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:05.010 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:05.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: pgmap v1300: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 2.0 MiB/s, 125 objects/s recovering 2023-10-09T06:35:05.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:05.021 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: osdmap e422: 8 total, 8 up, 8 in 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[212750]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: pgmap v1300: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 2.0 MiB/s, 125 objects/s recovering 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.022 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:05.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:35:05.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: osdmap e422: 8 total, 8 up, 8 in 2023-10-09T06:35:05.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:04 smithi107 ceph-mon[220921]: failed to encode map e422 with expected crc 2023-10-09T06:35:05.039 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:05.749 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:05.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:05.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: osdmap e423: 8 total, 8 up, 8 in 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: osdmap e423: 8 total, 8 up, 8 in 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.832 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T06:35:05.832 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:05.850 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:05.853 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:05.868 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:05.879 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:05.887 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:05.907 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:05.923 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:05.928 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: osdmap e423: 8 total, 8 up, 8 in 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.929 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: failed to encode map e423 with expected crc 2023-10-09T06:35:05.930 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T06:35:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:05.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:05.948 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:05.967 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:35:05.969 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:35:06.032 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:06.063 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:06.311 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:06.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:06.414 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:06.441 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:06.462 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:35:06.462 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1 pg undersized","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1719,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":424,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":8,"osd_in_since":1696832954,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":128},{"state_name":"active+undersized+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":2543,"data_bytes":165484200,"bytes_used":8305139712,"bytes_avail":759655145472,"bytes_total":767960285184,"recovering_objects_per_sec":75,"recovering_bytes_per_sec":2552078,"recovering_keys_per_sec":0,"num_objects_recovered":304,"num_bytes_recovered":10223616,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":588},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":27,"modified":"2023-10-09T06:34:19.394028+0000","services":{}},"progress_events":{"3403e6b5-aeb3-47f0-97ab-e50de33524c0":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 2s)","progress":0.98449611663818359,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:35:06.482 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:06.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:06.589 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:06.617 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: pgmap v1303: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2.4 MiB/s, 75 objects/s recovering 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: 12.14 starting backfill to osd.7 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3773238931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3607723589' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:35:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: pgmap v1303: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2.4 MiB/s, 75 objects/s recovering 2023-10-09T06:35:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: 12.14 starting backfill to osd.7 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3773238931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3607723589' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: pgmap v1303: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 158 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2.4 MiB/s, 75 objects/s recovering 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: osdmap e424: 8 total, 8 up, 8 in 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: 12.14 starting backfill to osd.7 from (0'0,0'0] MAX to 421'9015 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: failed to encode map e424 with expected crc 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3773238931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3607723589' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:35:07.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:07.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:07.160 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:07.188 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:07.905 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e424 with expected crc 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e424 with expected crc 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T06:35:08.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:35:08.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T06:35:08.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T06:35:08.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.108 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:07 smithi107 ceph-mon[220921]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.132 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:35:08.132 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e424 with expected crc 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: osdmap e425: 8 total, 8 up, 8 in 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e425 with expected crc 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T06:35:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: osdmap e426: 8 total, 8 up, 8 in 2023-10-09T06:35:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:07 smithi150 ceph-mon[194178]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.254 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:08.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:08.358 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:08.395 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:08.553 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:08.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:08.632 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:08.652 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:08.652 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:08.667 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:08.679 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:08.693 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:08.734 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:08.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:08.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:08.870 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:08.920 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:08.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:08.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:08.929 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: pgmap v1306: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 118 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:08.929 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.929 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T06:35:08.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.930 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: pgmap v1306: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 118 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e426 with expected crc 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.931 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:08.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:08 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.024 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:09.051 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: pgmap v1306: 129 pgs: 1 active+undersized+remapped, 128 active+clean; 118 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e426 with expected crc 2023-10-09T06:35:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: osdmap e427: 8 total, 8 up, 8 in 2023-10-09T06:35:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:08 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.816 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:35:09.827 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:09.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:09.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:09.931 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:09.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T06:35:09.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T06:35:09.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: failed to encode map e427 with expected crc 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: failed to encode map e428 with expected crc 2023-10-09T06:35:09.955 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:09 smithi107 ceph-mon[220921]: failed to encode map e428 with expected crc 2023-10-09T06:35:09.957 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: failed to encode map e427 with expected crc 2023-10-09T06:35:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T06:35:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T06:35:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: osdmap e428: 8 total, 8 up, 8 in 2023-10-09T06:35:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: failed to encode map e428 with expected crc 2023-10-09T06:35:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:09 smithi150 ceph-mon[194178]: failed to encode map e428 with expected crc 2023-10-09T06:35:10.180 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:35:10.273 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 99608 objects 2023-10-09T06:35:10.273 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :147.269 2023-10-09T06:35:10.391 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:10.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:10.496 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:10.530 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:10.580 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:10.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:10.684 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:10.713 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:10.786 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: pgmap v1309: 129 pgs: 129 active+clean; 49 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: 12.16 starting backfill to osd.4 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: 12.16 starting backfill to osd.6 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: failed to encode map e428 with expected crc 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:35:10.787 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: pgmap v1309: 129 pgs: 129 active+clean; 49 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: 12.16 starting backfill to osd.4 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: 12.16 starting backfill to osd.6 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: failed to encode map e428 with expected crc 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: failed to encode map e429 with expected crc 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T06:35:10.788 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:10 smithi107 ceph-mon[220921]: failed to encode map e429 with expected crc 2023-10-09T06:35:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: pgmap v1309: 129 pgs: 129 active+clean; 49 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: 12.16 starting backfill to osd.4 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: 12.16 starting backfill to osd.6 from (0'0,0'0] MAX to 425'9100 2023-10-09T06:35:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T06:35:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: failed to encode map e428 with expected crc 2023-10-09T06:35:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: failed to encode map e429 with expected crc 2023-10-09T06:35:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: osdmap e429: 8 total, 8 up, 8 in 2023-10-09T06:35:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:10 smithi150 ceph-mon[194178]: failed to encode map e429 with expected crc 2023-10-09T06:35:11.340 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:11.362 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:11.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:11.393 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:11.469 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:11.510 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:11.511 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: failed to encode map e429 with expected crc 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3085295500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:12.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: failed to encode map e429 with expected crc 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3085295500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T06:35:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T06:35:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:11 smithi107 ceph-mon[220921]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: failed to encode map e429 with expected crc 2023-10-09T06:35:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T06:35:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3085295500' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:12.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T06:35:12.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:12.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: osdmap e430: 8 total, 8 up, 8 in 2023-10-09T06:35:12.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:11 smithi150 ceph-mon[194178]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.335 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:12.342 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:12] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T06:35:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:12.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:12.445 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:12.483 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:12.530 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:12.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:12.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:12.636 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:12.668 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: pgmap v1312: 129 pgs: 129 active+clean; 33 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: 12.17 starting backfill to osd.3 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: 12.17 starting backfill to osd.6 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T06:35:12.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:12 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.969 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: pgmap v1312: 129 pgs: 129 active+clean; 33 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: 12.17 starting backfill to osd.3 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: 12.17 starting backfill to osd.6 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.970 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: pgmap v1312: 129 pgs: 129 active+clean; 33 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: failed to encode map e430 with expected crc 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: 12.17 starting backfill to osd.3 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: 12.17 starting backfill to osd.6 from (0'0,0'0] MAX to 427'9023 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: osdmap e431: 8 total, 8 up, 8 in 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:12.971 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:12 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.133 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:13.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:13.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:13.271 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:13.320 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:13.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:13.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:13.426 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:13.460 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:13.523 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:35:13.526 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:35:13.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: failed to encode map e431 with expected crc 2023-10-09T06:35:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: failed to encode map e432 with expected crc 2023-10-09T06:35:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:35:13.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:13 smithi107 ceph-mon[220921]: failed to encode map e432 with expected crc 2023-10-09T06:35:14.110 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:14.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: failed to encode map e431 with expected crc 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: failed to encode map e432 with expected crc 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: osdmap e432: 8 total, 8 up, 8 in 2023-10-09T06:35:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:13 smithi150 ceph-mon[194178]: failed to encode map e432 with expected crc 2023-10-09T06:35:14.215 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:14.241 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:14.282 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:14.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:14.387 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:14.413 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:14.454 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:14.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:14.562 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:14.591 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:14.656 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1316,"stamp":"2023-10-09T06:35:13.489581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":16455312,"num_objects":267,"num_object_clones":0,"num_object_copies":801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":267,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181943,"num_write_kb":5776958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9458,"num_bytes_recovered":484518652,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":201928,"ondisk_log_size":201928,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":397,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7461408,"kb_used_data":734560,"kb_used_omap":20,"kb_used_meta":6726763,"kb_avail":742499808,"statfs":{"total":767960285184,"available":760319803392,"internally_reserved":0,"allocated":752189440,"data_stored":740274142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6888205966},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-34570264,"num_objects":-529,"num_object_clones":0,"num_object_copies":-1587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":580,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1245184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.002295"},"pg_stats":[{"pgid":"12.15","version":"427'9102","reported_seq":9223,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500929+0000","last_change":"2023-10-09T06:35:06.702214+0000","last_active":"2023-10-09T06:35:11.500929+0000","last_peered":"2023-10-09T06:35:11.500929+0000","last_clean":"2023-10-09T06:35:11.500929+0000","last_became_active":"2023-10-09T06:35:06.677055+0000","last_became_peered":"2023-10-09T06:35:06.677055+0000","last_unstale":"2023-10-09T06:35:11.500929+0000","last_undegraded":"2023-10-09T06:35:11.500929+0000","last_fullsized":"2023-10-09T06:35:11.500929+0000","mapping_epoch":424,"log_start":"365'1030","ondisk_log_start":"365'1030","created":390,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":5,"last_scrub":"421'9062","last_scrub_stamp":"2023-10-09T06:35:03.439580+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.439580+0000","objects_scrubbed":40,"log_size":8072,"log_dups_size":0,"ondisk_log_size":8072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:19.487256+0000","scrub_duration":0.064842488000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":794,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.546923+0000","last_change":"2023-10-09T06:34:59.944040+0000","last_active":"2023-10-09T06:35:10.546923+0000","last_peered":"2023-10-09T06:35:10.546923+0000","last_clean":"2023-10-09T06:35:10.546923+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:35:10.546923+0000","last_undegraded":"2023-10-09T06:35:10.546923+0000","last_fullsized":"2023-10-09T06:35:10.546923+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":409,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.502327+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:35:10.502327+0000","last_peered":"2023-10-09T06:35:10.502327+0000","last_clean":"2023-10-09T06:35:10.502327+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:35:10.502327+0000","last_undegraded":"2023-10-09T06:35:10.502327+0000","last_fullsized":"2023-10-09T06:35:10.502327+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":131,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T06:35:12.487276+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:35:12.487276+0000","last_peered":"2023-10-09T06:35:12.487276+0000","last_clean":"2023-10-09T06:35:12.487276+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:35:12.487276+0000","last_undegraded":"2023-10-09T06:35:12.487276+0000","last_fullsized":"2023-10-09T06:35:12.487276+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"428'9108","reported_seq":9216,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.532550+0000","last_change":"2023-10-09T06:35:11.532207+0000","last_active":"2023-10-09T06:35:11.532550+0000","last_peered":"2023-10-09T06:35:11.532550+0000","last_clean":"2023-10-09T06:35:11.532550+0000","last_became_active":"2023-10-09T06:35:11.508792+0000","last_became_peered":"2023-10-09T06:35:11.508792+0000","last_unstale":"2023-10-09T06:35:11.532550+0000","last_undegraded":"2023-10-09T06:35:11.532550+0000","last_fullsized":"2023-10-09T06:35:11.532550+0000","mapping_epoch":429,"log_start":"365'1368","ondisk_log_start":"365'1368","created":390,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6224,"num_write_kb":198912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":761,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547140+0000","last_change":"2023-10-09T06:35:03.005446+0000","last_active":"2023-10-09T06:35:10.547140+0000","last_peered":"2023-10-09T06:35:10.547140+0000","last_clean":"2023-10-09T06:35:10.547140+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:35:10.547140+0000","last_undegraded":"2023-10-09T06:35:10.547140+0000","last_fullsized":"2023-10-09T06:35:10.547140+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:06.799059+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":425,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.497998+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:35:11.497998+0000","last_peered":"2023-10-09T06:35:11.497998+0000","last_clean":"2023-10-09T06:35:11.497998+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:35:11.497998+0000","last_undegraded":"2023-10-09T06:35:11.497998+0000","last_fullsized":"2023-10-09T06:35:11.497998+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":427,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.518629+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:35:09.518629+0000","last_peered":"2023-10-09T06:35:09.518629+0000","last_clean":"2023-10-09T06:35:09.518629+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:35:09.518629+0000","last_undegraded":"2023-10-09T06:35:09.518629+0000","last_fullsized":"2023-10-09T06:35:09.518629+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"427'9023","reported_seq":9124,"reported_epoch":431,"state":"peering","last_fresh":"2023-10-09T06:35:12.496739+0000","last_change":"2023-10-09T06:35:12.497111+0000","last_active":"2023-10-09T06:35:12.496710+0000","last_peered":"2023-10-09T06:35:11.618954+0000","last_clean":"2023-10-09T06:35:08.565333+0000","last_became_active":"2023-10-09T06:35:11.594598+0000","last_became_peered":"2023-10-09T06:35:11.594598+0000","last_unstale":"2023-10-09T06:35:12.496739+0000","last_undegraded":"2023-10-09T06:35:12.496739+0000","last_fullsized":"2023-10-09T06:35:12.496739+0000","mapping_epoch":431,"log_start":"365'1383","ondisk_log_start":"365'1383","created":390,"last_epoch_clean":430,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7640,"log_dups_size":0,"ondisk_log_size":7640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6179,"num_write_kb":194464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":677,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.502133+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:35:10.502133+0000","last_peered":"2023-10-09T06:35:10.502133+0000","last_clean":"2023-10-09T06:35:10.502133+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:35:10.502133+0000","last_undegraded":"2023-10-09T06:35:10.502133+0000","last_fullsized":"2023-10-09T06:35:10.502133+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":149,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503887+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:35:11.503887+0000","last_peered":"2023-10-09T06:35:11.503887+0000","last_clean":"2023-10-09T06:35:11.503887+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:35:11.503887+0000","last_undegraded":"2023-10-09T06:35:11.503887+0000","last_fullsized":"2023-10-09T06:35:11.503887+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":420,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502636+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:35:11.502636+0000","last_peered":"2023-10-09T06:35:11.502636+0000","last_clean":"2023-10-09T06:35:11.502636+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:35:11.502636+0000","last_undegraded":"2023-10-09T06:35:11.502636+0000","last_fullsized":"2023-10-09T06:35:11.502636+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.18","version":"426'8957","reported_seq":9042,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-10-09T06:35:08.054603+0000","last_change":"2023-10-09T06:35:02.880462+0000","last_active":"2023-10-09T06:35:08.054603+0000","last_peered":"2023-10-09T06:35:08.054603+0000","last_clean":"2023-10-09T06:35:08.054603+0000","last_became_active":"2023-10-09T06:32:33.077785+0000","last_became_peered":"2023-10-09T06:32:33.077785+0000","last_unstale":"2023-10-09T06:35:08.054603+0000","last_undegraded":"2023-10-09T06:35:08.054603+0000","last_fullsized":"2023-10-09T06:35:08.054603+0000","mapping_epoch":390,"log_start":"366'1727","ondisk_log_start":"366'1727","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7230,"log_dups_size":0,"ondisk_log_size":7230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6092,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":539,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502511+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:35:11.502511+0000","last_peered":"2023-10-09T06:35:11.502511+0000","last_clean":"2023-10-09T06:35:11.502511+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:35:11.502511+0000","last_undegraded":"2023-10-09T06:35:11.502511+0000","last_fullsized":"2023-10-09T06:35:11.502511+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.492052+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:35:09.492052+0000","last_peered":"2023-10-09T06:35:09.492052+0000","last_clean":"2023-10-09T06:35:09.492052+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:35:09.492052+0000","last_undegraded":"2023-10-09T06:35:09.492052+0000","last_fullsized":"2023-10-09T06:35:09.492052+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.504181+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:35:11.504181+0000","last_peered":"2023-10-09T06:35:11.504181+0000","last_clean":"2023-10-09T06:35:11.504181+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:35:11.504181+0000","last_undegraded":"2023-10-09T06:35:11.504181+0000","last_fullsized":"2023-10-09T06:35:11.504181+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"426'8974","reported_seq":9070,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500270+0000","last_change":"2023-10-09T06:34:21.023111+0000","last_active":"2023-10-09T06:35:11.500270+0000","last_peered":"2023-10-09T06:35:11.500270+0000","last_clean":"2023-10-09T06:35:11.500270+0000","last_became_active":"2023-10-09T06:32:33.074690+0000","last_became_peered":"2023-10-09T06:32:33.074690+0000","last_unstale":"2023-10-09T06:35:11.500270+0000","last_undegraded":"2023-10-09T06:35:11.500270+0000","last_fullsized":"2023-10-09T06:35:11.500270+0000","mapping_epoch":390,"log_start":"365'812","ondisk_log_start":"365'812","created":390,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8162,"log_dups_size":0,"ondisk_log_size":8162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1015808,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6106,"num_write_kb":195872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":576,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502698+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:35:11.502698+0000","last_peered":"2023-10-09T06:35:11.502698+0000","last_clean":"2023-10-09T06:35:11.502698+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:35:11.502698+0000","last_undegraded":"2023-10-09T06:35:11.502698+0000","last_fullsized":"2023-10-09T06:35:11.502698+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547260+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:35:10.547260+0000","last_peered":"2023-10-09T06:35:10.547260+0000","last_clean":"2023-10-09T06:35:10.547260+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:35:10.547260+0000","last_undegraded":"2023-10-09T06:35:10.547260+0000","last_fullsized":"2023-10-09T06:35:10.547260+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503757+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:35:11.503757+0000","last_peered":"2023-10-09T06:35:11.503757+0000","last_clean":"2023-10-09T06:35:11.503757+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:35:11.503757+0000","last_undegraded":"2023-10-09T06:35:11.503757+0000","last_fullsized":"2023-10-09T06:35:11.503757+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":628,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547380+0000","last_change":"2023-10-09T06:34:51.813303+0000","last_active":"2023-10-09T06:35:10.547380+0000","last_peered":"2023-10-09T06:35:10.547380+0000","last_clean":"2023-10-09T06:35:10.547380+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:35:10.547380+0000","last_undegraded":"2023-10-09T06:35:10.547380+0000","last_fullsized":"2023-10-09T06:35:10.547380+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":312,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503260+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:35:11.503260+0000","last_peered":"2023-10-09T06:35:11.503260+0000","last_clean":"2023-10-09T06:35:11.503260+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:35:11.503260+0000","last_undegraded":"2023-10-09T06:35:11.503260+0000","last_fullsized":"2023-10-09T06:35:11.503260+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":366,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.501566+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:35:11.501566+0000","last_peered":"2023-10-09T06:35:11.501566+0000","last_clean":"2023-10-09T06:35:11.501566+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:35:11.501566+0000","last_undegraded":"2023-10-09T06:35:11.501566+0000","last_fullsized":"2023-10-09T06:35:11.501566+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":538,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499546+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:35:11.499546+0000","last_peered":"2023-10-09T06:35:11.499546+0000","last_clean":"2023-10-09T06:35:11.499546+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:35:11.499546+0000","last_undegraded":"2023-10-09T06:35:11.499546+0000","last_fullsized":"2023-10-09T06:35:11.499546+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":436,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501759+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:35:10.501759+0000","last_peered":"2023-10-09T06:35:10.501759+0000","last_clean":"2023-10-09T06:35:10.501759+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:35:10.501759+0000","last_undegraded":"2023-10-09T06:35:10.501759+0000","last_fullsized":"2023-10-09T06:35:10.501759+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":400,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500876+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:35:10.500876+0000","last_peered":"2023-10-09T06:35:10.500876+0000","last_clean":"2023-10-09T06:35:10.500876+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:35:10.500876+0000","last_undegraded":"2023-10-09T06:35:10.500876+0000","last_fullsized":"2023-10-09T06:35:10.500876+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":700,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491286+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:35:09.491286+0000","last_peered":"2023-10-09T06:35:09.491286+0000","last_clean":"2023-10-09T06:35:09.491286+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:35:09.491286+0000","last_undegraded":"2023-10-09T06:35:09.491286+0000","last_fullsized":"2023-10-09T06:35:09.491286+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":280,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502537+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:35:11.502537+0000","last_peered":"2023-10-09T06:35:11.502537+0000","last_clean":"2023-10-09T06:35:11.502537+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:35:11.502537+0000","last_undegraded":"2023-10-09T06:35:11.502537+0000","last_fullsized":"2023-10-09T06:35:11.502537+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499549+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:35:11.499549+0000","last_peered":"2023-10-09T06:35:11.499549+0000","last_clean":"2023-10-09T06:35:11.499549+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:35:11.499549+0000","last_undegraded":"2023-10-09T06:35:11.499549+0000","last_fullsized":"2023-10-09T06:35:11.499549+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":544,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.546934+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:35:10.546934+0000","last_peered":"2023-10-09T06:35:10.546934+0000","last_clean":"2023-10-09T06:35:10.546934+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:35:10.546934+0000","last_undegraded":"2023-10-09T06:35:10.546934+0000","last_fullsized":"2023-10-09T06:35:10.546934+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":425,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491632+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:35:09.491632+0000","last_peered":"2023-10-09T06:35:09.491632+0000","last_clean":"2023-10-09T06:35:09.491632+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:35:09.491632+0000","last_undegraded":"2023-10-09T06:35:09.491632+0000","last_fullsized":"2023-10-09T06:35:09.491632+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547489+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:35:10.547489+0000","last_peered":"2023-10-09T06:35:10.547489+0000","last_clean":"2023-10-09T06:35:10.547489+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:35:10.547489+0000","last_undegraded":"2023-10-09T06:35:10.547489+0000","last_fullsized":"2023-10-09T06:35:10.547489+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":756,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498417+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:35:11.498417+0000","last_peered":"2023-10-09T06:35:11.498417+0000","last_clean":"2023-10-09T06:35:11.498417+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:35:11.498417+0000","last_undegraded":"2023-10-09T06:35:11.498417+0000","last_fullsized":"2023-10-09T06:35:11.498417+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":274,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502481+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:35:11.502481+0000","last_peered":"2023-10-09T06:35:11.502481+0000","last_clean":"2023-10-09T06:35:11.502481+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:35:11.502481+0000","last_undegraded":"2023-10-09T06:35:11.502481+0000","last_fullsized":"2023-10-09T06:35:11.502481+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.492028+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:35:09.492028+0000","last_peered":"2023-10-09T06:35:09.492028+0000","last_clean":"2023-10-09T06:35:09.492028+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:35:09.492028+0000","last_undegraded":"2023-10-09T06:35:09.492028+0000","last_fullsized":"2023-10-09T06:35:09.492028+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":544,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547073+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:35:10.547073+0000","last_peered":"2023-10-09T06:35:10.547073+0000","last_clean":"2023-10-09T06:35:10.547073+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:35:10.547073+0000","last_undegraded":"2023-10-09T06:35:10.547073+0000","last_fullsized":"2023-10-09T06:35:10.547073+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":509,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499579+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:35:11.499579+0000","last_peered":"2023-10-09T06:35:11.499579+0000","last_clean":"2023-10-09T06:35:11.499579+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:35:11.499579+0000","last_undegraded":"2023-10-09T06:35:11.499579+0000","last_fullsized":"2023-10-09T06:35:11.499579+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498495+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:35:11.498495+0000","last_peered":"2023-10-09T06:35:11.498495+0000","last_clean":"2023-10-09T06:35:11.498495+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:35:11.498495+0000","last_undegraded":"2023-10-09T06:35:11.498495+0000","last_fullsized":"2023-10-09T06:35:11.498495+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":558,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502620+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:35:11.502620+0000","last_peered":"2023-10-09T06:35:11.502620+0000","last_clean":"2023-10-09T06:35:11.502620+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:35:11.502620+0000","last_undegraded":"2023-10-09T06:35:11.502620+0000","last_fullsized":"2023-10-09T06:35:11.502620+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"427'9167","reported_seq":9314,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500963+0000","last_change":"2023-10-09T06:34:08.464064+0000","last_active":"2023-10-09T06:35:10.500963+0000","last_peered":"2023-10-09T06:35:10.500963+0000","last_clean":"2023-10-09T06:35:10.500963+0000","last_became_active":"2023-10-09T06:32:33.069345+0000","last_became_peered":"2023-10-09T06:32:33.069345+0000","last_unstale":"2023-10-09T06:35:10.500963+0000","last_undegraded":"2023-10-09T06:35:10.500963+0000","last_fullsized":"2023-10-09T06:35:10.500963+0000","mapping_epoch":361,"log_start":"365'442","ondisk_log_start":"365'442","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"414'7928","last_scrub_stamp":"2023-10-09T06:34:08.463956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:34:08.463956+0000","objects_scrubbed":1240,"log_size":8725,"log_dups_size":0,"ondisk_log_size":8725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:39.228319+0000","scrub_duration":0.275394735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1240,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":316,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503518+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:35:11.503518+0000","last_peered":"2023-10-09T06:35:11.503518+0000","last_clean":"2023-10-09T06:35:11.503518+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:35:11.503518+0000","last_undegraded":"2023-10-09T06:35:11.503518+0000","last_fullsized":"2023-10-09T06:35:11.503518+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547027+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:35:10.547027+0000","last_peered":"2023-10-09T06:35:10.547027+0000","last_clean":"2023-10-09T06:35:10.547027+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:35:10.547027+0000","last_undegraded":"2023-10-09T06:35:10.547027+0000","last_fullsized":"2023-10-09T06:35:10.547027+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":815,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499181+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:35:11.499181+0000","last_peered":"2023-10-09T06:35:11.499181+0000","last_clean":"2023-10-09T06:35:11.499181+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:35:11.499181+0000","last_undegraded":"2023-10-09T06:35:11.499181+0000","last_fullsized":"2023-10-09T06:35:11.499181+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.1","version":"427'9187","reported_seq":9288,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500163+0000","last_change":"2023-10-09T06:34:43.375542+0000","last_active":"2023-10-09T06:35:11.500163+0000","last_peered":"2023-10-09T06:35:11.500163+0000","last_clean":"2023-10-09T06:35:11.500163+0000","last_became_active":"2023-10-09T06:32:33.000165+0000","last_became_peered":"2023-10-09T06:32:33.000165+0000","last_unstale":"2023-10-09T06:35:11.500163+0000","last_undegraded":"2023-10-09T06:35:11.500163+0000","last_fullsized":"2023-10-09T06:35:11.500163+0000","mapping_epoch":361,"log_start":"365'1249","ondisk_log_start":"365'1249","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":201024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":278,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503164+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:35:11.503164+0000","last_peered":"2023-10-09T06:35:11.503164+0000","last_clean":"2023-10-09T06:35:11.503164+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:35:11.503164+0000","last_undegraded":"2023-10-09T06:35:11.503164+0000","last_fullsized":"2023-10-09T06:35:11.503164+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":427,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500716+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:35:10.500716+0000","last_peered":"2023-10-09T06:35:10.500716+0000","last_clean":"2023-10-09T06:35:10.500716+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:35:10.500716+0000","last_undegraded":"2023-10-09T06:35:10.500716+0000","last_fullsized":"2023-10-09T06:35:10.500716+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":732,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502273+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:35:11.502273+0000","last_peered":"2023-10-09T06:35:11.502273+0000","last_clean":"2023-10-09T06:35:11.502273+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:35:11.502273+0000","last_undegraded":"2023-10-09T06:35:11.502273+0000","last_fullsized":"2023-10-09T06:35:11.502273+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.2","version":"426'9144","reported_seq":9271,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.502217+0000","last_change":"2023-10-09T06:35:00.201300+0000","last_active":"2023-10-09T06:35:10.502217+0000","last_peered":"2023-10-09T06:35:10.502217+0000","last_clean":"2023-10-09T06:35:10.502217+0000","last_became_active":"2023-10-09T06:32:32.996849+0000","last_became_peered":"2023-10-09T06:32:32.996849+0000","last_unstale":"2023-10-09T06:35:10.502217+0000","last_undegraded":"2023-10-09T06:35:10.502217+0000","last_fullsized":"2023-10-09T06:35:10.502217+0000","mapping_epoch":361,"log_start":"365'206","ondisk_log_start":"365'206","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8938,"log_dups_size":0,"ondisk_log_size":8938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":199904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":334,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503058+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:35:11.503058+0000","last_peered":"2023-10-09T06:35:11.503058+0000","last_clean":"2023-10-09T06:35:11.503058+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:35:11.503058+0000","last_undegraded":"2023-10-09T06:35:11.503058+0000","last_fullsized":"2023-10-09T06:35:11.503058+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":407,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500925+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:35:11.500925+0000","last_peered":"2023-10-09T06:35:11.500925+0000","last_clean":"2023-10-09T06:35:11.500925+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:35:11.500925+0000","last_undegraded":"2023-10-09T06:35:11.500925+0000","last_fullsized":"2023-10-09T06:35:11.500925+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":554,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501687+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:35:10.501687+0000","last_peered":"2023-10-09T06:35:10.501687+0000","last_clean":"2023-10-09T06:35:10.501687+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:35:10.501687+0000","last_undegraded":"2023-10-09T06:35:10.501687+0000","last_fullsized":"2023-10-09T06:35:10.501687+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.3","version":"426'9159","reported_seq":9248,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491802+0000","last_change":"2023-10-09T06:35:01.843970+0000","last_active":"2023-10-09T06:35:09.491802+0000","last_peered":"2023-10-09T06:35:09.491802+0000","last_clean":"2023-10-09T06:35:09.491802+0000","last_became_active":"2023-10-09T06:32:32.993707+0000","last_became_peered":"2023-10-09T06:32:32.993707+0000","last_unstale":"2023-10-09T06:35:09.491802+0000","last_undegraded":"2023-10-09T06:35:09.491802+0000","last_fullsized":"2023-10-09T06:35:09.491802+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":9159,"log_dups_size":0,"ondisk_log_size":9159,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6267,"num_write_kb":200608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547226+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:35:10.547226+0000","last_peered":"2023-10-09T06:35:10.547226+0000","last_clean":"2023-10-09T06:35:10.547226+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:35:10.547226+0000","last_undegraded":"2023-10-09T06:35:10.547226+0000","last_fullsized":"2023-10-09T06:35:10.547226+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":389,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491862+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:35:09.491862+0000","last_peered":"2023-10-09T06:35:09.491862+0000","last_clean":"2023-10-09T06:35:09.491862+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:35:09.491862+0000","last_undegraded":"2023-10-09T06:35:09.491862+0000","last_fullsized":"2023-10-09T06:35:09.491862+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"426'9072","reported_seq":9171,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.501374+0000","last_change":"2023-10-09T06:34:49.477325+0000","last_active":"2023-10-09T06:35:11.501374+0000","last_peered":"2023-10-09T06:35:11.501374+0000","last_clean":"2023-10-09T06:35:11.501374+0000","last_became_active":"2023-10-09T06:32:32.995923+0000","last_became_peered":"2023-10-09T06:32:32.995923+0000","last_unstale":"2023-10-09T06:35:11.501374+0000","last_undegraded":"2023-10-09T06:35:11.501374+0000","last_fullsized":"2023-10-09T06:35:11.501374+0000","mapping_epoch":361,"log_start":"365'1110","ondisk_log_start":"365'1110","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":783,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502153+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:35:11.502153+0000","last_peered":"2023-10-09T06:35:11.502153+0000","last_clean":"2023-10-09T06:35:11.502153+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:35:11.502153+0000","last_undegraded":"2023-10-09T06:35:11.502153+0000","last_fullsized":"2023-10-09T06:35:11.502153+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":157,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499124+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:35:11.499124+0000","last_peered":"2023-10-09T06:35:11.499124+0000","last_clean":"2023-10-09T06:35:11.499124+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:35:11.499124+0000","last_undegraded":"2023-10-09T06:35:11.499124+0000","last_fullsized":"2023-10-09T06:35:11.499124+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":410,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501593+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:35:10.501593+0000","last_peered":"2023-10-09T06:35:10.501593+0000","last_clean":"2023-10-09T06:35:10.501593+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:35:10.501593+0000","last_undegraded":"2023-10-09T06:35:10.501593+0000","last_fullsized":"2023-10-09T06:35:10.501593+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"426'9153","reported_seq":9252,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.501049+0000","last_change":"2023-10-09T06:34:55.606440+0000","last_active":"2023-10-09T06:35:11.501049+0000","last_peered":"2023-10-09T06:35:11.501049+0000","last_clean":"2023-10-09T06:35:11.501049+0000","last_became_active":"2023-10-09T06:32:33.044765+0000","last_became_peered":"2023-10-09T06:32:33.044765+0000","last_unstale":"2023-10-09T06:35:11.501049+0000","last_undegraded":"2023-10-09T06:35:11.501049+0000","last_fullsized":"2023-10-09T06:35:11.501049+0000","mapping_epoch":361,"log_start":"365'1258","ondisk_log_start":"365'1258","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7895,"log_dups_size":0,"ondisk_log_size":7895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6258,"num_write_kb":199776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":762,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547006+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:35:10.547006+0000","last_peered":"2023-10-09T06:35:10.547006+0000","last_clean":"2023-10-09T06:35:10.547006+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:35:10.547006+0000","last_undegraded":"2023-10-09T06:35:10.547006+0000","last_fullsized":"2023-10-09T06:35:10.547006+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":388,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500760+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:35:11.500760+0000","last_peered":"2023-10-09T06:35:11.500760+0000","last_clean":"2023-10-09T06:35:11.500760+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:35:11.500760+0000","last_undegraded":"2023-10-09T06:35:11.500760+0000","last_fullsized":"2023-10-09T06:35:11.500760+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":407,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501359+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:35:10.501359+0000","last_peered":"2023-10-09T06:35:10.501359+0000","last_clean":"2023-10-09T06:35:10.501359+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:35:10.501359+0000","last_undegraded":"2023-10-09T06:35:10.501359+0000","last_fullsized":"2023-10-09T06:35:10.501359+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":765,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498507+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:35:11.498507+0000","last_peered":"2023-10-09T06:35:11.498507+0000","last_clean":"2023-10-09T06:35:11.498507+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:35:11.498507+0000","last_undegraded":"2023-10-09T06:35:11.498507+0000","last_fullsized":"2023-10-09T06:35:11.498507+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.6","version":"426'9124","reported_seq":9213,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499961+0000","last_change":"2023-10-09T06:32:32.976279+0000","last_active":"2023-10-09T06:35:11.499961+0000","last_peered":"2023-10-09T06:35:11.499961+0000","last_clean":"2023-10-09T06:35:11.499961+0000","last_became_active":"2023-10-09T06:32:32.973171+0000","last_became_peered":"2023-10-09T06:32:32.973171+0000","last_unstale":"2023-10-09T06:35:11.499961+0000","last_undegraded":"2023-10-09T06:35:11.499961+0000","last_fullsized":"2023-10-09T06:35:11.499961+0000","mapping_epoch":361,"log_start":"365'1326","ondisk_log_start":"365'1326","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7798,"log_dups_size":0,"ondisk_log_size":7798,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524288,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":411,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500498+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:35:10.500498+0000","last_peered":"2023-10-09T06:35:10.500498+0000","last_clean":"2023-10-09T06:35:10.500498+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:35:10.500498+0000","last_undegraded":"2023-10-09T06:35:10.500498+0000","last_fullsized":"2023-10-09T06:35:10.500498+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":390,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501943+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:35:10.501943+0000","last_peered":"2023-10-09T06:35:10.501943+0000","last_clean":"2023-10-09T06:35:10.501943+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:35:10.501943+0000","last_undegraded":"2023-10-09T06:35:10.501943+0000","last_fullsized":"2023-10-09T06:35:10.501943+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"427'8898","reported_seq":8988,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498765+0000","last_change":"2023-10-09T06:32:33.079508+0000","last_active":"2023-10-09T06:35:11.498765+0000","last_peered":"2023-10-09T06:35:11.498765+0000","last_clean":"2023-10-09T06:35:11.498765+0000","last_became_active":"2023-10-09T06:32:33.075123+0000","last_became_peered":"2023-10-09T06:32:33.075123+0000","last_unstale":"2023-10-09T06:35:11.498765+0000","last_undegraded":"2023-10-09T06:35:11.498765+0000","last_fullsized":"2023-10-09T06:35:11.498765+0000","mapping_epoch":361,"log_start":"365'833","ondisk_log_start":"365'833","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":8065,"log_dups_size":0,"ondisk_log_size":8065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6651904,"num_objects":101,"num_object_clones":0,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6053,"num_write_kb":196960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":740,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547172+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:35:10.547172+0000","last_peered":"2023-10-09T06:35:10.547172+0000","last_clean":"2023-10-09T06:35:10.547172+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:35:10.547172+0000","last_undegraded":"2023-10-09T06:35:10.547172+0000","last_fullsized":"2023-10-09T06:35:10.547172+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.504134+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:35:11.504134+0000","last_peered":"2023-10-09T06:35:11.504134+0000","last_clean":"2023-10-09T06:35:11.504134+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:35:11.504134+0000","last_undegraded":"2023-10-09T06:35:11.504134+0000","last_fullsized":"2023-10-09T06:35:11.504134+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":384,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498845+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:35:11.498845+0000","last_peered":"2023-10-09T06:35:11.498845+0000","last_clean":"2023-10-09T06:35:11.498845+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:35:11.498845+0000","last_undegraded":"2023-10-09T06:35:11.498845+0000","last_fullsized":"2023-10-09T06:35:11.498845+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"426'12458","reported_seq":12536,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491447+0000","last_change":"2023-10-09T06:31:03.215613+0000","last_active":"2023-10-09T06:35:09.491447+0000","last_peered":"2023-10-09T06:35:09.491447+0000","last_clean":"2023-10-09T06:35:09.491447+0000","last_became_active":"2023-10-09T06:31:03.215338+0000","last_became_peered":"2023-10-09T06:31:03.215338+0000","last_unstale":"2023-10-09T06:35:09.491447+0000","last_undegraded":"2023-10-09T06:35:09.491447+0000","last_fullsized":"2023-10-09T06:35:09.491447+0000","mapping_epoch":361,"log_start":"391'5900","ondisk_log_start":"391'5900","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:38.401834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12458,"num_write_kb":398656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":723,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502697+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:35:11.502697+0000","last_peered":"2023-10-09T06:35:11.502697+0000","last_clean":"2023-10-09T06:35:11.502697+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:35:11.502697+0000","last_undegraded":"2023-10-09T06:35:11.502697+0000","last_fullsized":"2023-10-09T06:35:11.502697+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":436,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499549+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:35:11.499549+0000","last_peered":"2023-10-09T06:35:11.499549+0000","last_clean":"2023-10-09T06:35:11.499549+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:35:11.499549+0000","last_undegraded":"2023-10-09T06:35:11.499549+0000","last_fullsized":"2023-10-09T06:35:11.499549+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547453+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:35:10.547453+0000","last_peered":"2023-10-09T06:35:10.547453+0000","last_clean":"2023-10-09T06:35:10.547453+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:35:10.547453+0000","last_undegraded":"2023-10-09T06:35:10.547453+0000","last_fullsized":"2023-10-09T06:35:10.547453+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":370,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.501307+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:35:11.501307+0000","last_peered":"2023-10-09T06:35:11.501307+0000","last_clean":"2023-10-09T06:35:11.501307+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:35:11.501307+0000","last_undegraded":"2023-10-09T06:35:11.501307+0000","last_fullsized":"2023-10-09T06:35:11.501307+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"427'12488","reported_seq":12566,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491609+0000","last_change":"2023-10-09T06:31:03.215123+0000","last_active":"2023-10-09T06:35:09.491609+0000","last_peered":"2023-10-09T06:35:09.491609+0000","last_clean":"2023-10-09T06:35:09.491609+0000","last_became_active":"2023-10-09T06:31:03.214876+0000","last_became_peered":"2023-10-09T06:31:03.214876+0000","last_unstale":"2023-10-09T06:35:09.491609+0000","last_undegraded":"2023-10-09T06:35:09.491609+0000","last_fullsized":"2023-10-09T06:35:09.491609+0000","mapping_epoch":361,"log_start":"391'5900","ondisk_log_start":"391'5900","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:29:14.356867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12488,"num_write_kb":399616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":776,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547147+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:35:10.547147+0000","last_peered":"2023-10-09T06:35:10.547147+0000","last_clean":"2023-10-09T06:35:10.547147+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:35:10.547147+0000","last_undegraded":"2023-10-09T06:35:10.547147+0000","last_fullsized":"2023-10-09T06:35:10.547147+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547264+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:35:10.547264+0000","last_peered":"2023-10-09T06:35:10.547264+0000","last_clean":"2023-10-09T06:35:10.547264+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:35:10.547264+0000","last_undegraded":"2023-10-09T06:35:10.547264+0000","last_fullsized":"2023-10-09T06:35:10.547264+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":411,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500714+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:35:10.500714+0000","last_peered":"2023-10-09T06:35:10.500714+0000","last_clean":"2023-10-09T06:35:10.500714+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:35:10.500714+0000","last_undegraded":"2023-10-09T06:35:10.500714+0000","last_fullsized":"2023-10-09T06:35:10.500714+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":400,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499475+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:35:11.499475+0000","last_peered":"2023-10-09T06:35:11.499475+0000","last_clean":"2023-10-09T06:35:11.499475+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:35:11.499475+0000","last_undegraded":"2023-10-09T06:35:11.499475+0000","last_fullsized":"2023-10-09T06:35:11.499475+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"427'12538","reported_seq":12616,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502381+0000","last_change":"2023-10-09T06:31:03.217015+0000","last_active":"2023-10-09T06:35:11.502381+0000","last_peered":"2023-10-09T06:35:11.502381+0000","last_clean":"2023-10-09T06:35:11.502381+0000","last_became_active":"2023-10-09T06:31:03.216705+0000","last_became_peered":"2023-10-09T06:31:03.216705+0000","last_unstale":"2023-10-09T06:35:11.502381+0000","last_undegraded":"2023-10-09T06:35:11.502381+0000","last_fullsized":"2023-10-09T06:35:11.502381+0000","mapping_epoch":361,"log_start":"403'6846","ondisk_log_start":"403'6846","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:49.920825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12538,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":773,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502421+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:35:11.502421+0000","last_peered":"2023-10-09T06:35:11.502421+0000","last_clean":"2023-10-09T06:35:11.502421+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:35:11.502421+0000","last_undegraded":"2023-10-09T06:35:11.502421+0000","last_fullsized":"2023-10-09T06:35:11.502421+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":422,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503531+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:35:11.503531+0000","last_peered":"2023-10-09T06:35:11.503531+0000","last_clean":"2023-10-09T06:35:11.503531+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:35:11.503531+0000","last_undegraded":"2023-10-09T06:35:11.503531+0000","last_fullsized":"2023-10-09T06:35:11.503531+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":409,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.504444+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:35:11.504444+0000","last_peered":"2023-10-09T06:35:11.504444+0000","last_clean":"2023-10-09T06:35:11.504444+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:35:11.504444+0000","last_undegraded":"2023-10-09T06:35:11.504444+0000","last_fullsized":"2023-10-09T06:35:11.504444+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":431,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498924+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:35:11.498924+0000","last_peered":"2023-10-09T06:35:11.498924+0000","last_clean":"2023-10-09T06:35:11.498924+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:35:11.498924+0000","last_undegraded":"2023-10-09T06:35:11.498924+0000","last_fullsized":"2023-10-09T06:35:11.498924+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"428'12614","reported_seq":12693,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502720+0000","last_change":"2023-10-09T06:31:03.218350+0000","last_active":"2023-10-09T06:35:11.502720+0000","last_peered":"2023-10-09T06:35:11.502720+0000","last_clean":"2023-10-09T06:35:11.502720+0000","last_became_active":"2023-10-09T06:31:03.218103+0000","last_became_peered":"2023-10-09T06:31:03.218103+0000","last_unstale":"2023-10-09T06:35:11.502720+0000","last_undegraded":"2023-10-09T06:35:11.502720+0000","last_fullsized":"2023-10-09T06:35:11.502720+0000","mapping_epoch":361,"log_start":"403'7305","ondisk_log_start":"403'7305","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:31.110196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12614,"num_write_kb":403585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":117,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547302+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:35:10.547302+0000","last_peered":"2023-10-09T06:35:10.547302+0000","last_clean":"2023-10-09T06:35:10.547302+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:35:10.547302+0000","last_undegraded":"2023-10-09T06:35:10.547302+0000","last_fullsized":"2023-10-09T06:35:10.547302+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":754,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502451+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:35:11.502451+0000","last_peered":"2023-10-09T06:35:11.502451+0000","last_clean":"2023-10-09T06:35:11.502451+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:35:11.502451+0000","last_undegraded":"2023-10-09T06:35:11.502451+0000","last_fullsized":"2023-10-09T06:35:11.502451+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":140,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499667+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:35:11.499667+0000","last_peered":"2023-10-09T06:35:11.499667+0000","last_clean":"2023-10-09T06:35:11.499667+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:35:11.499667+0000","last_undegraded":"2023-10-09T06:35:11.499667+0000","last_fullsized":"2023-10-09T06:35:11.499667+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":393,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.501287+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:35:11.501287+0000","last_peered":"2023-10-09T06:35:11.501287+0000","last_clean":"2023-10-09T06:35:11.501287+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:35:11.501287+0000","last_undegraded":"2023-10-09T06:35:11.501287+0000","last_fullsized":"2023-10-09T06:35:11.501287+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"427'12656","reported_seq":12734,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.518644+0000","last_change":"2023-10-09T06:31:02.762130+0000","last_active":"2023-10-09T06:35:09.518644+0000","last_peered":"2023-10-09T06:35:09.518644+0000","last_clean":"2023-10-09T06:35:09.518644+0000","last_became_active":"2023-10-09T06:31:02.761788+0000","last_became_peered":"2023-10-09T06:31:02.761788+0000","last_unstale":"2023-10-09T06:35:09.518644+0000","last_undegraded":"2023-10-09T06:35:09.518644+0000","last_fullsized":"2023-10-09T06:35:09.518644+0000","mapping_epoch":361,"log_start":"392'6100","ondisk_log_start":"392'6100","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:10:55.773058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12656,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":739,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498593+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:35:11.498593+0000","last_peered":"2023-10-09T06:35:11.498593+0000","last_clean":"2023-10-09T06:35:11.498593+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:35:11.498593+0000","last_undegraded":"2023-10-09T06:35:11.498593+0000","last_fullsized":"2023-10-09T06:35:11.498593+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":294,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502324+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:35:11.502324+0000","last_peered":"2023-10-09T06:35:11.502324+0000","last_clean":"2023-10-09T06:35:11.502324+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:35:11.502324+0000","last_undegraded":"2023-10-09T06:35:11.502324+0000","last_fullsized":"2023-10-09T06:35:11.502324+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491444+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:35:09.491444+0000","last_peered":"2023-10-09T06:35:09.491444+0000","last_clean":"2023-10-09T06:35:09.491444+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:35:09.491444+0000","last_undegraded":"2023-10-09T06:35:09.491444+0000","last_fullsized":"2023-10-09T06:35:09.491444+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":413,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.518709+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:35:09.518709+0000","last_peered":"2023-10-09T06:35:09.518709+0000","last_clean":"2023-10-09T06:35:09.518709+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:35:09.518709+0000","last_undegraded":"2023-10-09T06:35:09.518709+0000","last_fullsized":"2023-10-09T06:35:09.518709+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"426'12244","reported_seq":12324,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498865+0000","last_change":"2023-10-09T06:31:03.215149+0000","last_active":"2023-10-09T06:35:11.498865+0000","last_peered":"2023-10-09T06:35:11.498865+0000","last_clean":"2023-10-09T06:35:11.498865+0000","last_became_active":"2023-10-09T06:31:03.214860+0000","last_became_peered":"2023-10-09T06:31:03.214860+0000","last_unstale":"2023-10-09T06:35:11.498865+0000","last_undegraded":"2023-10-09T06:35:11.498865+0000","last_fullsized":"2023-10-09T06:35:11.498865+0000","mapping_epoch":361,"log_start":"403'7009","ondisk_log_start":"403'7009","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":5235,"log_dups_size":0,"ondisk_log_size":5235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:02.454977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12244,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":753,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502940+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:35:11.502940+0000","last_peered":"2023-10-09T06:35:11.502940+0000","last_clean":"2023-10-09T06:35:11.502940+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:35:11.502940+0000","last_undegraded":"2023-10-09T06:35:11.502940+0000","last_fullsized":"2023-10-09T06:35:11.502940+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":294,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503199+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:35:11.503199+0000","last_peered":"2023-10-09T06:35:11.503199+0000","last_clean":"2023-10-09T06:35:11.503199+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:35:11.503199+0000","last_undegraded":"2023-10-09T06:35:11.503199+0000","last_fullsized":"2023-10-09T06:35:11.503199+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":426,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499391+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:35:11.499391+0000","last_peered":"2023-10-09T06:35:11.499391+0000","last_clean":"2023-10-09T06:35:11.499391+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:35:11.499391+0000","last_undegraded":"2023-10-09T06:35:11.499391+0000","last_fullsized":"2023-10-09T06:35:11.499391+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":411,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491965+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:35:09.491965+0000","last_peered":"2023-10-09T06:35:09.491965+0000","last_clean":"2023-10-09T06:35:09.491965+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:35:09.491965+0000","last_undegraded":"2023-10-09T06:35:09.491965+0000","last_fullsized":"2023-10-09T06:35:09.491965+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"427'9141","reported_seq":9231,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503978+0000","last_change":"2023-10-09T06:32:47.459790+0000","last_active":"2023-10-09T06:35:11.503978+0000","last_peered":"2023-10-09T06:35:11.503978+0000","last_clean":"2023-10-09T06:35:11.503978+0000","last_became_active":"2023-10-09T06:32:33.070761+0000","last_became_peered":"2023-10-09T06:32:33.070761+0000","last_unstale":"2023-10-09T06:35:11.503978+0000","last_undegraded":"2023-10-09T06:35:11.503978+0000","last_fullsized":"2023-10-09T06:35:11.503978+0000","mapping_epoch":361,"log_start":"368'2311","ondisk_log_start":"368'2311","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6276,"num_write_kb":197824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":753,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501224+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:35:10.501224+0000","last_peered":"2023-10-09T06:35:10.501224+0000","last_clean":"2023-10-09T06:35:10.501224+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:35:10.501224+0000","last_undegraded":"2023-10-09T06:35:10.501224+0000","last_fullsized":"2023-10-09T06:35:10.501224+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":161,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498581+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:35:11.498581+0000","last_peered":"2023-10-09T06:35:11.498581+0000","last_clean":"2023-10-09T06:35:11.498581+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:35:11.498581+0000","last_undegraded":"2023-10-09T06:35:11.498581+0000","last_fullsized":"2023-10-09T06:35:11.498581+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":394,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499744+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:35:11.499744+0000","last_peered":"2023-10-09T06:35:11.499744+0000","last_clean":"2023-10-09T06:35:11.499744+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:35:11.499744+0000","last_undegraded":"2023-10-09T06:35:11.499744+0000","last_fullsized":"2023-10-09T06:35:11.499744+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":149,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.504048+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:35:11.504048+0000","last_peered":"2023-10-09T06:35:11.504048+0000","last_clean":"2023-10-09T06:35:11.504048+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:35:11.504048+0000","last_undegraded":"2023-10-09T06:35:11.504048+0000","last_fullsized":"2023-10-09T06:35:11.504048+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"427'9069","reported_seq":9169,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.500486+0000","last_change":"2023-10-09T06:34:47.445399+0000","last_active":"2023-10-09T06:35:11.500486+0000","last_peered":"2023-10-09T06:35:11.500486+0000","last_clean":"2023-10-09T06:35:11.500486+0000","last_became_active":"2023-10-09T06:32:33.044852+0000","last_became_peered":"2023-10-09T06:32:33.044852+0000","last_unstale":"2023-10-09T06:35:11.500486+0000","last_undegraded":"2023-10-09T06:35:11.500486+0000","last_fullsized":"2023-10-09T06:35:11.500486+0000","mapping_epoch":361,"log_start":"365'1128","ondisk_log_start":"365'1128","created":361,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7941,"log_dups_size":0,"ondisk_log_size":7941,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":197920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":783,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499685+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:35:11.499685+0000","last_peered":"2023-10-09T06:35:11.499685+0000","last_clean":"2023-10-09T06:35:11.499685+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:35:11.499685+0000","last_undegraded":"2023-10-09T06:35:11.499685+0000","last_fullsized":"2023-10-09T06:35:11.499685+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":415,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502841+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:35:11.502841+0000","last_peered":"2023-10-09T06:35:11.502841+0000","last_clean":"2023-10-09T06:35:11.502841+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:35:11.502841+0000","last_undegraded":"2023-10-09T06:35:11.502841+0000","last_fullsized":"2023-10-09T06:35:11.502841+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":394,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.492141+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:35:09.492141+0000","last_peered":"2023-10-09T06:35:09.492141+0000","last_clean":"2023-10-09T06:35:09.492141+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:35:09.492141+0000","last_undegraded":"2023-10-09T06:35:09.492141+0000","last_fullsized":"2023-10-09T06:35:09.492141+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":393,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500638+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:35:10.500638+0000","last_peered":"2023-10-09T06:35:10.500638+0000","last_clean":"2023-10-09T06:35:10.500638+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:35:10.500638+0000","last_undegraded":"2023-10-09T06:35:10.500638+0000","last_fullsized":"2023-10-09T06:35:10.500638+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"427'9054","reported_seq":9234,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547329+0000","last_change":"2023-10-09T06:35:08.532747+0000","last_active":"2023-10-09T06:35:10.547329+0000","last_peered":"2023-10-09T06:35:10.547329+0000","last_clean":"2023-10-09T06:35:10.547329+0000","last_became_active":"2023-10-09T06:35:08.499024+0000","last_became_peered":"2023-10-09T06:35:08.499024+0000","last_unstale":"2023-10-09T06:35:10.547329+0000","last_undegraded":"2023-10-09T06:35:10.547329+0000","last_fullsized":"2023-10-09T06:35:10.547329+0000","mapping_epoch":426,"log_start":"365'1211","ondisk_log_start":"365'1211","created":390,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7843,"log_dups_size":0,"ondisk_log_size":7843,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6182,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1376256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":710,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503362+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:35:11.503362+0000","last_peered":"2023-10-09T06:35:11.503362+0000","last_clean":"2023-10-09T06:35:11.503362+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:35:11.503362+0000","last_undegraded":"2023-10-09T06:35:11.503362+0000","last_fullsized":"2023-10-09T06:35:11.503362+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":395,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491958+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:35:09.491958+0000","last_peered":"2023-10-09T06:35:09.491958+0000","last_clean":"2023-10-09T06:35:09.491958+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:35:09.491958+0000","last_undegraded":"2023-10-09T06:35:09.491958+0000","last_fullsized":"2023-10-09T06:35:09.491958+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":419,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.499219+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:35:11.499219+0000","last_peered":"2023-10-09T06:35:11.499219+0000","last_clean":"2023-10-09T06:35:11.499219+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:35:11.499219+0000","last_undegraded":"2023-10-09T06:35:11.499219+0000","last_fullsized":"2023-10-09T06:35:11.499219+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.13","version":"426'9132","reported_seq":15480,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498330+0000","last_change":"2023-10-09T06:35:02.658700+0000","last_active":"2023-10-09T06:35:11.498330+0000","last_peered":"2023-10-09T06:35:11.498330+0000","last_clean":"2023-10-09T06:35:11.498330+0000","last_became_active":"2023-10-09T06:35:02.637405+0000","last_became_peered":"2023-10-09T06:35:02.637405+0000","last_unstale":"2023-10-09T06:35:11.498330+0000","last_undegraded":"2023-10-09T06:35:11.498330+0000","last_fullsized":"2023-10-09T06:35:11.498330+0000","mapping_epoch":420,"log_start":"365'1367","ondisk_log_start":"365'1367","created":390,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7765,"log_dups_size":0,"ondisk_log_size":7765,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6241,"num_write_kb":199648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3121,"num_bytes_recovered":90112000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":640,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498385+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:35:11.498385+0000","last_peered":"2023-10-09T06:35:11.498385+0000","last_clean":"2023-10-09T06:35:11.498385+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:35:11.498385+0000","last_undegraded":"2023-10-09T06:35:11.498385+0000","last_fullsized":"2023-10-09T06:35:11.498385+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":121,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547297+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:35:10.547297+0000","last_peered":"2023-10-09T06:35:10.547297+0000","last_clean":"2023-10-09T06:35:10.547297+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:35:10.547297+0000","last_undegraded":"2023-10-09T06:35:10.547297+0000","last_fullsized":"2023-10-09T06:35:10.547297+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":437,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.500645+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:35:10.500645+0000","last_peered":"2023-10-09T06:35:10.500645+0000","last_clean":"2023-10-09T06:35:10.500645+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:35:10.500645+0000","last_undegraded":"2023-10-09T06:35:10.500645+0000","last_fullsized":"2023-10-09T06:35:10.500645+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.12","version":"427'9163","reported_seq":18668,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-09T06:35:12.486227+0000","last_change":"2023-10-09T06:35:04.683490+0000","last_active":"2023-10-09T06:35:12.486227+0000","last_peered":"2023-10-09T06:35:12.486227+0000","last_clean":"2023-10-09T06:35:12.486227+0000","last_became_active":"2023-10-09T06:35:04.664610+0000","last_became_peered":"2023-10-09T06:35:04.664610+0000","last_unstale":"2023-10-09T06:35:12.486227+0000","last_undegraded":"2023-10-09T06:35:12.486227+0000","last_fullsized":"2023-10-09T06:35:12.486227+0000","mapping_epoch":422,"log_start":"366'2155","ondisk_log_start":"366'2155","created":390,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6260,"num_write_kb":200096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3133,"num_bytes_recovered":195952640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":121,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547332+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:35:10.547332+0000","last_peered":"2023-10-09T06:35:10.547332+0000","last_clean":"2023-10-09T06:35:10.547332+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:35:10.547332+0000","last_undegraded":"2023-10-09T06:35:10.547332+0000","last_fullsized":"2023-10-09T06:35:10.547332+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":447,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.502496+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:35:10.502496+0000","last_peered":"2023-10-09T06:35:10.502496+0000","last_clean":"2023-10-09T06:35:10.502496+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:35:10.502496+0000","last_undegraded":"2023-10-09T06:35:10.502496+0000","last_fullsized":"2023-10-09T06:35:10.502496+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.11","version":"427'9159","reported_seq":18654,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.547015+0000","last_change":"2023-10-09T06:34:57.893466+0000","last_active":"2023-10-09T06:35:10.547015+0000","last_peered":"2023-10-09T06:35:10.547015+0000","last_clean":"2023-10-09T06:35:10.547015+0000","last_became_active":"2023-10-09T06:33:43.622574+0000","last_became_peered":"2023-10-09T06:33:43.622574+0000","last_unstale":"2023-10-09T06:35:10.547015+0000","last_undegraded":"2023-10-09T06:35:10.547015+0000","last_fullsized":"2023-10-09T06:35:10.547015+0000","mapping_epoch":407,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:31:01.745080+0000","objects_scrubbed":0,"log_size":9159,"log_dups_size":0,"ondisk_log_size":9159,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6251,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3127,"num_bytes_recovered":196673536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":787,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502818+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:35:11.502818+0000","last_peered":"2023-10-09T06:35:11.502818+0000","last_clean":"2023-10-09T06:35:11.502818+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:35:11.502818+0000","last_undegraded":"2023-10-09T06:35:11.502818+0000","last_fullsized":"2023-10-09T06:35:11.502818+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":131,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.498609+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:35:11.498609+0000","last_peered":"2023-10-09T06:35:11.498609+0000","last_clean":"2023-10-09T06:35:11.498609+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:35:11.498609+0000","last_undegraded":"2023-10-09T06:35:11.498609+0000","last_fullsized":"2023-10-09T06:35:11.498609+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":149,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.503669+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:35:11.503669+0000","last_peered":"2023-10-09T06:35:11.503669+0000","last_clean":"2023-10-09T06:35:11.503669+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:35:11.503669+0000","last_undegraded":"2023-10-09T06:35:11.503669+0000","last_fullsized":"2023-10-09T06:35:11.503669+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"427'9074","reported_seq":21372,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491175+0000","last_change":"2023-10-09T06:35:02.860602+0000","last_active":"2023-10-09T06:35:09.491175+0000","last_peered":"2023-10-09T06:35:09.491175+0000","last_clean":"2023-10-09T06:35:09.491175+0000","last_became_active":"2023-10-09T06:34:05.174880+0000","last_became_peered":"2023-10-09T06:34:05.174880+0000","last_unstale":"2023-10-09T06:35:09.491175+0000","last_undegraded":"2023-10-09T06:35:09.491175+0000","last_fullsized":"2023-10-09T06:35:09.491175+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":390,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"421'9019","last_scrub_stamp":"2023-10-09T06:35:02.859778+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:31:01.745080+0000","last_clean_scrub_stamp":"2023-10-09T06:35:02.859778+0000","objects_scrubbed":56,"log_size":9074,"log_dups_size":0,"ondisk_log_size":9074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:11.580452+0000","scrub_duration":0.026883792,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":56,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":552,"reported_epoch":430,"state":"active+clean","last_fresh":"2023-10-09T06:35:11.502730+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:35:11.502730+0000","last_peered":"2023-10-09T06:35:11.502730+0000","last_clean":"2023-10-09T06:35:11.502730+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:35:11.502730+0000","last_undegraded":"2023-10-09T06:35:11.502730+0000","last_fullsized":"2023-10-09T06:35:11.502730+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":424,"reported_epoch":428,"state":"active+clean","last_fresh":"2023-10-09T06:35:09.491961+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:35:09.491961+0000","last_peered":"2023-10-09T06:35:09.491961+0000","last_clean":"2023-10-09T06:35:09.491961+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:35:09.491961+0000","last_undegraded":"2023-10-09T06:35:09.491961+0000","last_fullsized":"2023-10-09T06:35:09.491961+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":390,"reported_epoch":429,"state":"active+clean","last_fresh":"2023-10-09T06:35:10.501137+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:35:10.501137+0000","last_peered":"2023-10-09T06:35:10.501137+0000","last_clean":"2023-10-09T06:35:10.501137+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:35:10.501137+0000","last_undegraded":"2023-10-09T06:35:10.501137+0000","last_fullsized":"2023-10-09T06:35:10.501137+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":15990784,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181882,"num_write_kb":5776929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9402,"num_bytes_recovered":484114432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":710017024,"data_stored":710017024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":196882,"ondisk_log_size":196882,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234455,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101992,"kb_used_data":201760,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92643160,"statfs":{"total":95995035648,"available":94866595840,"internally_reserved":0,"allocated":206602240,"data_stored":205568191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:08 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":4.6699999999999999,"15min":6.1829999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":2.984,"5min":196.28899999999999,"15min":219.41900000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":4.79,"15min":6.3479999999999999},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":3.1160000000000001,"5min":196.333,"15min":219.399},"last":0.627}]},{"osd":1,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":3.8900000000000001,"15min":4.3540000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":6.3920000000000003,"5min":107.87,"15min":215.61699999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.3400000000000001,"5min":3.3660000000000001,"15min":4.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":8.0830000000000002,"5min":107.904,"15min":216.22200000000001},"last":0.41899999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":4.2050000000000001,"15min":6.9450000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":39.747,"5min":198.94200000000001,"15min":216.34299999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":4.4509999999999996,"5min":4.2679999999999998,"15min":7.0730000000000004},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":39.658999999999999,"5min":198.92099999999999,"15min":216.25200000000001},"last":0.59699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":6.0700000000000003,"15min":4.6120000000000001},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":4.9809999999999999,"5min":187.34200000000001,"15min":215.922},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.929,"5min":6.0670000000000002,"15min":4.6959999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.9189999999999996,"5min":187.08199999999999,"15min":215.94900000000001},"last":0.66400000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.79600000000000004,"15min":2.4729999999999999},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":0.56999999999999995,"5min":6.1689999999999996,"15min":218.446},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.74299999999999999,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":1.7190000000000001,"5min":4.891,"15min":216.172},"last":0.57699999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.86699999999999999,"15min":2.4620000000000002},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":4.2309999999999999,"5min":8.8170000000000002,"15min":216.37200000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.087,"15min":2.3889999999999998},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":2.1000000000000001,"5min":6.3019999999999996,"15min":218.15899999999999},"last":0.67500000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.123,"15min":2.2909999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":2.6560000000000001,"5min":15.162000000000001,"15min":218.19999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.89000000000000001,"15min":2.0979999999999999},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":1.7589999999999999,"5min":8.2919999999999998,"15min":218.05500000000001},"last":0.69099999999999995}]}]},{"osd":6,"up_from":46,"seq":197568496091,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991120,"kb_used_data":92104,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92754032,"statfs":{"total":95995035648,"available":94980128768,"internally_reserved":0,"allocated":94314496,"data_stored":93303745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.6379999999999999,"5min":3.577,"15min":3.7759999999999998},"min":{"1min":0.22500000000000001,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":47.363999999999997,"5min":75.061999999999998,"15min":110.926},"last":0.438},{"interface":"front","average":{"1min":4.7809999999999997,"5min":3.73,"15min":4.0439999999999996},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":47.381,"5min":75.021000000000001,"15min":110.911},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":3.3300000000000001,"15min":5.0759999999999996},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.7949999999999999,"5min":82.159000000000006,"15min":292.92200000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.853,"5min":3.5569999999999999,"15min":4.9569999999999999},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":9.2560000000000002,"5min":82.174000000000007,"15min":292.904},"last":0.45500000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":3.3450000000000002,"15min":3.1800000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":10.239000000000001,"5min":74.748999999999995,"15min":75.191999999999993},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.8080000000000001,"5min":3.3799999999999999,"15min":2.9249999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.281000000000001,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.47599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:34:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2720000000000002,"5min":5.3849999999999998,"15min":3.9159999999999999},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.761000000000003,"5min":91.174000000000007,"15min":110.358},"last":0.51100000000000001},{"interface":"front","average":{"1min":4.1799999999999997,"5min":5.3019999999999996,"15min":3.9209999999999998},"min":{"1min":0.38200000000000001,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":47.606000000000002,"5min":91.438999999999993,"15min":110.43899999999999},"last":0.375}]},{"osd":4,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.73,"15min":1.488},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":86.171999999999997,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":4.556,"5min":1.617,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.223},"max":{"1min":86.727000000000004,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.38900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.93799999999999994,"15min":1.3540000000000001},"min":{"1min":0.33300000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.907,"5min":5.7539999999999996,"15min":22.379999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1599999999999999,"15min":1.393},"min":{"1min":0.33300000000000002,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":4.6219999999999999,"5min":8.8000000000000007,"15min":24.401},"last":0.497}]},{"osd":7,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.204,"15min":2.528},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":19.295999999999999,"15min":122.937},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.4690000000000001,"15min":2.673},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4049999999999998,"5min":19.584,"15min":122.92100000000001},"last":0.53200000000000003}]}]},{"osd":1,"up_from":110,"seq":472446402828,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815448,"kb_used_data":5200,"kb_used_omap":4,"kb_used_meta":810235,"kb_avail":92929704,"statfs":{"total":95995035648,"available":95160016896,"internally_reserved":0,"allocated":5324800,"data_stored":3380116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":829681550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":6.2249999999999996,"15min":6.3029999999999999},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":19.699999999999999,"5min":143.465,"15min":269.79199999999997},"last":0.24099999999999999},{"interface":"front","average":{"1min":4.1660000000000004,"5min":6.2460000000000004,"15min":6.7720000000000002},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.17899999999999999},"max":{"1min":11.663,"5min":143.66399999999999,"15min":271.20999999999998},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:34:45 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":4.0869999999999997,"15min":3.6629999999999998},"min":{"1min":0.23400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":13.217000000000001,"5min":138.452,"15min":139.75399999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":3.3069999999999999,"5min":4.21,"15min":4.4119999999999999},"min":{"1min":0.254,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.244,"5min":138.47499999999999,"15min":140.29900000000001},"last":0.30599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":3.2890000000000001,"15min":3.9620000000000002},"min":{"1min":0.28499999999999998,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":8.9420000000000002,"5min":27.064,"15min":153.18199999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.891,"5min":2.1920000000000002,"15min":2.819},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":13.129,"5min":26.626999999999999,"15min":152.52000000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6739999999999999,"15min":3.7789999999999999},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.6159999999999997,"5min":38.835999999999999,"15min":199.43799999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.895,"15min":3.4140000000000001},"min":{"1min":0.36599999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":8.8089999999999993,"5min":38.819000000000003,"15min":199.32900000000001},"last":0.64400000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.5750000000000002,"15min":2.339},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.163,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.5619999999999998,"15min":2.7040000000000002},"min":{"1min":0.40999999999999998,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":8.7759999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.46400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":3.6949999999999998,"15min":3.1890000000000001},"min":{"1min":0.39900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.9250000000000007,"5min":107.27200000000001,"15min":107.27200000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.52,"5min":4.452,"15min":3.4700000000000002},"min":{"1min":0.379,"5min":0.316,"15min":0.316},"max":{"1min":7.6950000000000003,"5min":107.252,"15min":107.252},"last":0.54100000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":4.2190000000000003,"15min":3.6389999999999998},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":7.6529999999999996,"5min":104.07299999999999,"15min":104.07299999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.7309999999999999,"5min":3.4750000000000001,"15min":3.0110000000000001},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.191000000000001,"5min":104.056,"15min":104.056},"last":0.57099999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024261,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":943360,"kb_used_data":5816,"kb_used_omap":5,"kb_used_meta":937530,"kb_avail":92801792,"statfs":{"total":95995035648,"available":95029035008,"internally_reserved":0,"allocated":5955584,"data_stored":3728654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":960031354},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.9930000000000001,"15min":3.4260000000000002},"min":{"1min":0.19900000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":33.091999999999999,"5min":33.091999999999999,"15min":191.446},"last":0.26500000000000001},{"interface":"front","average":{"1min":4.25,"5min":2.6120000000000001,"15min":3.7109999999999999},"min":{"1min":0.23799999999999999,"5min":0.187,"15min":0.187},"max":{"1min":24.838999999999999,"5min":24.838999999999999,"15min":191.393},"last":0.312}]},{"osd":2,"last update":"Mon Oct 9 06:34:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4199999999999999,"5min":2.8119999999999998,"15min":2.6429999999999998},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":12.532,"5min":16.936,"15min":57.162999999999997},"last":0.24099999999999999},{"interface":"front","average":{"1min":4.133,"5min":3.524,"15min":3.121},"min":{"1min":0.22800000000000001,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":13.579000000000001,"5min":15.794,"15min":57.585000000000001},"last":0.28000000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:34:11 2023","interfaces":[{"interface":"back","average":{"1min":4.5170000000000003,"5min":6.5800000000000001,"15min":5.9820000000000002},"min":{"1min":0.28399999999999997,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":15.413,"5min":187.172,"15min":208.11199999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":3.9609999999999999,"5min":6.2960000000000003,"15min":5.9489999999999998},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":15.266,"5min":194.851,"15min":208.25899999999999},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.649,"15min":1.5760000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":9.7159999999999993},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.504,"5min":2.2090000000000001,"15min":1.929},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.385999999999999,"5min":12.385999999999999,"15min":18.081},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.7669999999999999,"15min":1.873},"min":{"1min":0.309,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":9.0099999999999998,"5min":9.0099999999999998,"15min":62.984999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.601,"5min":2.3380000000000001,"15min":2.3730000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.8239999999999998,"5min":13.968999999999999,"15min":64.983999999999995},"last":0.499}]},{"osd":6,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.1829999999999998,"5min":2.4550000000000001,"15min":3.036},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":22.292000000000002,"5min":22.292000000000002,"15min":120.572},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9910000000000001,"15min":2.52},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30299999999999999},"max":{"1min":22.475000000000001,"5min":22.475000000000001,"15min":120.64400000000001},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.472,"15min":2.734},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.60399999999999998},{"interface":"front","average":{"1min":4.1139999999999999,"5min":2.79,"15min":3.125},"min":{"1min":0.39700000000000002,"5min":0.39100000000000001,"15min":0.377},"max":{"1min":22.683,"5min":22.683,"15min":122.32299999999999},"last":0.54300000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814130,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":730548,"kb_used_data":5356,"kb_used_omap":7,"kb_used_meta":725176,"kb_avail":93014604,"statfs":{"total":95995035648,"available":95246954496,"internally_reserved":0,"allocated":5484544,"data_stored":3613862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":742580305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":15.65,"5min":7.032,"15min":5.5129999999999999},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":156.94499999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":15.272,"5min":6.734,"15min":5.0049999999999999},"min":{"1min":0.315,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":156.922,"5min":156.922,"15min":168.49700000000001},"last":0.29299999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":14.833,"5min":6.202,"15min":4.7000000000000002},"min":{"1min":0.41499999999999998,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":140.452,"5min":140.452,"15min":205.148},"last":0.308},{"interface":"front","average":{"1min":14.69,"5min":5.9409999999999998,"15min":4.8739999999999997},"min":{"1min":0.35599999999999998,"5min":0.183,"15min":0.183},"max":{"1min":139.75299999999999,"5min":139.75299999999999,"15min":205.267},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 06:34:15 2023","interfaces":[{"interface":"back","average":{"1min":6.2880000000000003,"5min":3.968,"15min":3.5510000000000002},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":75.498999999999995,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.26600000000000001},{"interface":"front","average":{"1min":6.2220000000000004,"5min":4.3140000000000001,"15min":3.9409999999999998},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.25700000000000001},"max":{"1min":69.784000000000006,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.33400000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.0249999999999999,"15min":2.27},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":3.423,"5min":44.314,"15min":90.938999999999993},"last":0.75600000000000001},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.4329999999999998,"15min":2.6019999999999999},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":4.6950000000000003,"5min":43.896000000000001,"15min":90.950000000000003},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.794,"15min":1.675},"min":{"1min":0.379,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7250000000000001,"5min":13.102,"15min":17.800000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.958,"5min":2.0009999999999999,"15min":1.9630000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.6779999999999999,"5min":11.285,"15min":11.285},"last":0.78000000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":4.524,"15min":3.4060000000000001},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":20.445,"5min":123.47799999999999,"15min":138.291},"last":0.55100000000000005},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.3840000000000003,"15min":3.202},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.148,"5min":122.517,"15min":138.01599999999999},"last":0.65900000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":2.7839999999999998,"15min":2.3820000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":16.574999999999999,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.444,"5min":2.3570000000000002,"15min":2.1549999999999998},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":3.3330000000000002,"5min":47.206000000000003,"15min":55.731000000000002},"last":0.68600000000000005}]}]},{"osd":3,"up_from":149,"seq":639950127328,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":859892,"kb_used_data":173292,"kb_used_omap":2,"kb_used_meta":686589,"kb_avail":92885260,"statfs":{"total":95995035648,"available":95114506240,"internally_reserved":0,"allocated":177451008,"data_stored":175655494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":703067189},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":7.1669999999999998,"15min":5.8540000000000001},"min":{"1min":0.20000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":22.367000000000001,"5min":161.821,"15min":187.852},"last":0.20300000000000001},{"interface":"front","average":{"1min":2.3140000000000001,"5min":6.1689999999999996,"15min":5.8490000000000002},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":13.443,"5min":163.52000000000001,"15min":187.923},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9209999999999998,"5min":4.4930000000000003,"15min":4.375},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":22.349,"5min":155.23099999999999,"15min":187.89099999999999},"last":0.44},{"interface":"front","average":{"1min":4.1050000000000004,"5min":6,"15min":4.2480000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":15.066000000000001,"5min":155.24799999999999,"15min":155.24799999999999},"last":0.252}]},{"osd":2,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.7329999999999997,"15min":4.234},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":22.297000000000001,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":3.5720000000000001,"5min":5.4390000000000001,"15min":4.2380000000000004},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":13.631,"5min":168.16900000000001,"15min":187.79499999999999},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.9159999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.661,"5min":48.249000000000002,"15min":48.249000000000002},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.887,"15min":2.645},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":16.516999999999999,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.54400000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.7389999999999999,"15min":2.7250000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":16.494,"5min":48.162999999999997,"15min":187.833},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.1520000000000001,"15min":2.4300000000000002},"min":{"1min":0.42299999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":18.835999999999999,"5min":48.106000000000002,"15min":187.952},"last":0.497}]},{"osd":6,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":4.3339999999999996,"15min":3.3319999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.462,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.3199999999999998,"5min":3.8679999999999999,"15min":2.9900000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":22.260000000000002,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":4.5229999999999997,"15min":3.3769999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":13.695,"5min":121.613,"15min":192.20400000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":3.0110000000000001,"5min":4.7270000000000003,"15min":3.4569999999999999},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.542000000000002,"5min":121.559,"15min":187.86500000000001},"last":0.50600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019364,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129304,"kb_used_data":246608,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":92615848,"statfs":{"total":95995035648,"available":94838628352,"internally_reserved":0,"allocated":252526592,"data_stored":251508026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":11.006,"5min":11.769,"15min":5.6020000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":118.98999999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":11.007,"5min":11.597,"15min":5.8220000000000001},"min":{"1min":0.32300000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":118.956,"5min":287.57100000000003,"15min":287.57100000000003},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":10.643000000000001,"5min":5.5620000000000003,"15min":3.4590000000000001},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":134.37100000000001,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.497},{"interface":"front","average":{"1min":10.766,"5min":5.7549999999999999,"15min":3.669},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":134.21700000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.41399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":5.3170000000000002,"15min":3.7749999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":148.36099999999999,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.44},{"interface":"front","average":{"1min":12.159000000000001,"5min":5.2990000000000004,"15min":3.7559999999999998},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":145.755,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":8.2629999999999999,"5min":10.750999999999999,"15min":5.8639999999999999},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":65.915000000000006,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":8.6899999999999995,"5min":10.590999999999999,"15min":5.5880000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":68.054000000000002,"5min":161.934,"15min":161.934},"last":0.34999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.70699999999999996,"15min":0.98999999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80600000000000005,"5min":6.0199999999999996,"15min":10.673},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.82799999999999996,"15min":1.0029999999999999},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.548,"5min":5.7000000000000002,"15min":12.552},"last":0.317}]},{"osd":6,"last update":"Mon Oct 9 06:34:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.4620000000000002,"15min":1.2929999999999999},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":2.5179999999999998,"5min":163.36699999999999,"15min":163.36699999999999},"last":0.308},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.641,"15min":1.6419999999999999},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":2.9340000000000002,"5min":163.381,"15min":163.381},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:34:20 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.6499999999999999,"15min":1.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":2.9649999999999999,"5min":70.247,"15min":70.247},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.774,"15min":1.6160000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":2.0680000000000001,"5min":70.159000000000006,"15min":70.159000000000006},"last":0.47799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757729,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889744,"kb_used_data":4424,"kb_used_omap":0,"kb_used_meta":885312,"kb_avail":92855408,"statfs":{"total":95995035648,"available":95083937792,"internally_reserved":0,"allocated":4530176,"data_stored":3516054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906559488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":12.871,"5min":8.1099999999999994,"15min":5.742},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":149.149,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":12.737,"5min":7.6159999999999997,"15min":5.5460000000000003},"min":{"1min":0.245,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":149.21100000000001,"5min":294.30399999999997,"15min":294.30399999999997},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":7.4349999999999996,"5min":4.0570000000000004,"15min":3.274},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":119.024,"5min":119.024,"15min":126.71899999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":7.5579999999999998,"5min":4.2779999999999996,"15min":3.4369999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":119.133,"5min":119.133,"15min":126.931},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.4609999999999999,"15min":2.4969999999999999},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.5149999999999997,"5min":64.150000000000006,"15min":81.260000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.2290000000000001,"15min":2.4289999999999998},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":4.8550000000000004,"5min":64.507000000000005,"15min":81.245000000000005},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:35:10 2023","interfaces":[{"interface":"back","average":{"1min":9.0340000000000007,"5min":4.4349999999999996,"15min":5.1299999999999999},"min":{"1min":0.35399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":85.671000000000006,"5min":92.521000000000001,"15min":144.56399999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":8.4719999999999995,"5min":4.1699999999999999,"15min":4.9400000000000004},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":85.593999999999994,"5min":94.492000000000004,"15min":144.62799999999999},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.74199999999999999,"15min":1.1859999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.8700000000000001,"5min":4.9119999999999999,"15min":32.850999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.78900000000000003,"15min":1.044},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":2.8919999999999999,"5min":6.2610000000000001,"15min":32.82},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.387,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.9460000000000002,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.5169999999999999,"15min":1.7470000000000001},"min":{"1min":0.35099999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.173,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.624}]},{"osd":7,"last update":"Mon Oct 9 06:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.147,"15min":1.397},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1829999999999998,"5min":7.4290000000000003,"15min":30.658999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.022,"15min":1.399},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":2.3540000000000001,"5min":6.2320000000000002,"15min":30.683},"last":0.70099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":171966464,"data_stored":171966464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":246284288,"data_stored":246284288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":89718784,"data_stored":89718784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":202047488,"data_stored":202047488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:35:14.741 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:14.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:14.848 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:14.850 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:35:14.876 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: pgmap v1315: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: 12.18 starting backfill to osd.1 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: 12.18 starting backfill to osd.4 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[212750]: 12.18 starting backfill to osd.7 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.877 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: pgmap v1315: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:14.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: failed to encode map e432 with expected crc 2023-10-09T06:35:14.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: 12.18 starting backfill to osd.1 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: 12.18 starting backfill to osd.4 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:14.878 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:14 smithi107 ceph-mon[220921]: 12.18 starting backfill to osd.7 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:14.879 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:14.979 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:14.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:15.032 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:15.033 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:15.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:15.117 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: pgmap v1315: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: failed to encode map e432 with expected crc 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: 12.18 starting backfill to osd.1 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: 12.18 starting backfill to osd.4 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: from='client.36699 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:14 smithi150 ceph-mon[194178]: 12.18 starting backfill to osd.7 from (0'0,0'0] MAX to 426'8957 2023-10-09T06:35:15.262 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:15.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:15.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:15.366 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:15.393 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:15.434 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:15.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:15.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:35:15.539 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:15.565 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:15.606 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:15.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:15.711 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:15.740 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:15.782 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:15.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:15.882 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-10-09T06:35:15.882 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-10-09T06:35:15.890 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:15.954 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:16.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e432 with expected crc 2023-10-09T06:35:16.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:35:16.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2755675164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e432 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2755675164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:16.153 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e432 with expected crc 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: osdmap e433: 8 total, 8 up, 8 in 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: failed to encode map e433 with expected crc 2023-10-09T06:35:16.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2755675164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:16.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:16.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:16.171 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:35:16.260 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:16.285 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:16.390 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:16.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:16.494 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:16.523 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:16.564 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:16.568 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:16.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:16.627 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:16.678 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:16.699 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:16.710 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:16.758 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:16.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:16.863 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:16.893 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:16.939 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:16.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:17.037 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: pgmap v1318: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:17.038 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: pgmap v1318: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:17.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:17.045 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:17.074 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: pgmap v1318: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 304 KiB/s, 4 objects/s recovering 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: osdmap e434: 8 total, 8 up, 8 in 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: failed to encode map e434 with expected crc 2023-10-09T06:35:17.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:17.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:17.619 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:17.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:17.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:35:17.723 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:17.750 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:18 smithi107 ceph-mon[212750]: pgmap v1320: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s 2023-10-09T06:35:18.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:18 smithi107 ceph-mon[220921]: pgmap v1320: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s 2023-10-09T06:35:18.473 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:35:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:18 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:18.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:18 smithi150 ceph-mon[194178]: pgmap v1320: 129 pgs: 1 peering, 128 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s 2023-10-09T06:35:18.949 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:18.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:18.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:19.052 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:19.086 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:19.186 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:19.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:19.290 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:19.319 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:19.544 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:19.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:19.585 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:19 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:35:19.586 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:19 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:35:19.648 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:19.654 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:19.680 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:19.711 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:19.728 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:19.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:19.752 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:35:19.753 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:35:19.753 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:35:19.753 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:35:19.837 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:19.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:19 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T06:35:19.865 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:19.910 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:19.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:20.016 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:20.043 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:20.087 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:20.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:20.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:20.195 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:20.222 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:20.263 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:20.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:20.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:20.370 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:20.400 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:20.554 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:35:20.558 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_7' removed 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: pgmap v1321: 129 pgs: 129 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3338697149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2285637992' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: pgmap v1321: 129 pgs: 129 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3338697149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2285637992' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:20.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:20 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: pgmap v1321: 129 pgs: 129 active+clean; 16 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: osdmap e435: 8 total, 8 up, 8 in 2023-10-09T06:35:20.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3338697149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:35:20.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: failed to encode map e435 with expected crc 2023-10-09T06:35:20.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2285637992' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:20.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:20 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:35:21.347 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:21 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.705 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:21.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:21.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:21.809 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:21.835 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: osdmap e436: 8 total, 8 up, 8 in 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:21.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:21 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.275 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:22] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2023-10-09T06:35:22.278 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:22.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:22.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:22.384 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:22.416 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:22.432 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:35:22.433 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:35:22.433 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-10-09T06:35:22.433 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-10-09T06:35:22.464 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:22.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:22.557 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[212750]: pgmap v1324: 103 pgs: 103 active+clean; 454 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[212750]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[220921]: pgmap v1324: 103 pgs: 103 active+clean; 454 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:22.558 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.559 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:22 smithi107 ceph-mon[220921]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.571 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:22.599 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:22.643 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:22.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:22.750 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:22.775 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:22 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.775 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:22 smithi150 ceph-mon[194178]: pgmap v1324: 103 pgs: 103 active+clean; 454 KiB data, 7.1 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:22.775 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:22 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.775 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:22 smithi150 ceph-mon[194178]: failed to encode map e436 with expected crc 2023-10-09T06:35:22.777 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:22.820 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:22.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:22.927 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:22.956 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:23.023 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:23.032 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:23.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:23.137 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:23.167 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:23.270 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:23.361 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:23.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:23.466 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:23.494 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:23.587 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:35:23.589 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:35:23.591 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:23.613 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:23.639 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:23.656 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:23.677 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:23.700 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:23.719 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:23.739 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:23.740 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.740 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.740 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.740 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.740 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.741 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.741 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:35:23.741 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:35:24.009 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:24.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:24.113 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:24.141 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:24.297 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:35:24.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[220921]: pgmap v1325: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:24.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2650791435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:24.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[220921]: from='client.46661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:24.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[212750]: pgmap v1325: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:24.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2650791435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:24.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:24 smithi107 ceph-mon[212750]: from='client.46661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:24 smithi150 ceph-mon[194178]: pgmap v1325: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2650791435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:24.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:24 smithi150 ceph-mon[194178]: from='client.46661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:24.971 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1325,"stamp":"2023-10-09T06:35:23.429169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":464528,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":404220,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":373,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7351576,"kb_used_data":606680,"kb_used_omap":20,"kb_used_meta":6744811,"kb_avail":742609640,"statfs":{"total":767960285184,"available":760432271360,"internally_reserved":0,"allocated":621240320,"data_stored":609210186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6906687118},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":117,"apply_latency_ms":117,"commit_latency_ns":117000000,"apply_latency_ns":117000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-15990784,"num_objects":-243,"num_object_clones":0,"num_object_copies":-729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-243,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-181882,"num_write_kb":-5776929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-9402,"num_bytes_recovered":-484114432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.705501"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":800,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495898+0000","last_change":"2023-10-09T06:34:59.944040+0000","last_active":"2023-10-09T06:35:19.495898+0000","last_peered":"2023-10-09T06:35:19.495898+0000","last_clean":"2023-10-09T06:35:19.495898+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:35:19.495898+0000","last_undegraded":"2023-10-09T06:35:19.495898+0000","last_fullsized":"2023-10-09T06:35:19.495898+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:22:41.675186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:22:41.675186+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":415,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099007+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:35:20.099007+0000","last_peered":"2023-10-09T06:35:20.099007+0000","last_clean":"2023-10-09T06:35:20.099007+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:35:20.099007+0000","last_undegraded":"2023-10-09T06:35:20.099007+0000","last_fullsized":"2023-10-09T06:35:20.099007+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493959+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:35:19.493959+0000","last_peered":"2023-10-09T06:35:19.493959+0000","last_clean":"2023-10-09T06:35:19.493959+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:35:19.493959+0000","last_undegraded":"2023-10-09T06:35:19.493959+0000","last_fullsized":"2023-10-09T06:35:19.493959+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":767,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496787+0000","last_change":"2023-10-09T06:35:03.005446+0000","last_active":"2023-10-09T06:35:19.496787+0000","last_peered":"2023-10-09T06:35:19.496787+0000","last_clean":"2023-10-09T06:35:19.496787+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:35:19.496787+0000","last_undegraded":"2023-10-09T06:35:19.496787+0000","last_fullsized":"2023-10-09T06:35:19.496787+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:06.799059+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":430,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.490844+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:35:19.490844+0000","last_peered":"2023-10-09T06:35:19.490844+0000","last_clean":"2023-10-09T06:35:19.490844+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:35:19.490844+0000","last_undegraded":"2023-10-09T06:35:19.490844+0000","last_fullsized":"2023-10-09T06:35:19.490844+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":434,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.489116+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:35:19.489116+0000","last_peered":"2023-10-09T06:35:19.489116+0000","last_clean":"2023-10-09T06:35:19.489116+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:35:19.489116+0000","last_undegraded":"2023-10-09T06:35:19.489116+0000","last_fullsized":"2023-10-09T06:35:19.489116+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":683,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099250+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:35:20.099250+0000","last_peered":"2023-10-09T06:35:20.099250+0000","last_clean":"2023-10-09T06:35:20.099250+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:35:20.099250+0000","last_undegraded":"2023-10-09T06:35:20.099250+0000","last_fullsized":"2023-10-09T06:35:20.099250+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":154,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494741+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:35:19.494741+0000","last_peered":"2023-10-09T06:35:19.494741+0000","last_clean":"2023-10-09T06:35:19.494741+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:35:19.494741+0000","last_undegraded":"2023-10-09T06:35:19.494741+0000","last_fullsized":"2023-10-09T06:35:19.494741+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":425,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497496+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:35:19.497496+0000","last_peered":"2023-10-09T06:35:19.497496+0000","last_clean":"2023-10-09T06:35:19.497496+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:35:19.497496+0000","last_undegraded":"2023-10-09T06:35:19.497496+0000","last_fullsized":"2023-10-09T06:35:19.497496+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":544,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653411+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:35:19.653411+0000","last_peered":"2023-10-09T06:35:19.653411+0000","last_clean":"2023-10-09T06:35:19.653411+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:35:19.653411+0000","last_undegraded":"2023-10-09T06:35:19.653411+0000","last_fullsized":"2023-10-09T06:35:19.653411+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":109,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488971+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:35:19.488971+0000","last_peered":"2023-10-09T06:35:19.488971+0000","last_clean":"2023-10-09T06:35:19.488971+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:35:19.488971+0000","last_undegraded":"2023-10-09T06:35:19.488971+0000","last_fullsized":"2023-10-09T06:35:19.488971+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494916+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:35:19.494916+0000","last_peered":"2023-10-09T06:35:19.494916+0000","last_clean":"2023-10-09T06:35:19.494916+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:35:19.494916+0000","last_undegraded":"2023-10-09T06:35:19.494916+0000","last_fullsized":"2023-10-09T06:35:19.494916+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":581,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653608+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:35:19.653608+0000","last_peered":"2023-10-09T06:35:19.653608+0000","last_clean":"2023-10-09T06:35:19.653608+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:35:19.653608+0000","last_undegraded":"2023-10-09T06:35:19.653608+0000","last_fullsized":"2023-10-09T06:35:19.653608+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497517+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:35:19.497517+0000","last_peered":"2023-10-09T06:35:19.497517+0000","last_clean":"2023-10-09T06:35:19.497517+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:35:19.497517+0000","last_undegraded":"2023-10-09T06:35:19.497517+0000","last_fullsized":"2023-10-09T06:35:19.497517+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495981+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:35:19.495981+0000","last_peered":"2023-10-09T06:35:19.495981+0000","last_clean":"2023-10-09T06:35:19.495981+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:35:19.495981+0000","last_undegraded":"2023-10-09T06:35:19.495981+0000","last_fullsized":"2023-10-09T06:35:19.495981+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":634,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496721+0000","last_change":"2023-10-09T06:34:51.813303+0000","last_active":"2023-10-09T06:35:19.496721+0000","last_peered":"2023-10-09T06:35:19.496721+0000","last_clean":"2023-10-09T06:35:19.496721+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:35:19.496721+0000","last_undegraded":"2023-10-09T06:35:19.496721+0000","last_fullsized":"2023-10-09T06:35:19.496721+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:21:31.860637+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:21:31.860637+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":317,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653835+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:35:19.653835+0000","last_peered":"2023-10-09T06:35:19.653835+0000","last_clean":"2023-10-09T06:35:19.653835+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:35:19.653835+0000","last_undegraded":"2023-10-09T06:35:19.653835+0000","last_fullsized":"2023-10-09T06:35:19.653835+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":371,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491497+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:35:19.491497+0000","last_peered":"2023-10-09T06:35:19.491497+0000","last_clean":"2023-10-09T06:35:19.491497+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:35:19.491497+0000","last_undegraded":"2023-10-09T06:35:19.491497+0000","last_fullsized":"2023-10-09T06:35:19.491497+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":543,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.492530+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:35:19.492530+0000","last_peered":"2023-10-09T06:35:19.492530+0000","last_clean":"2023-10-09T06:35:19.492530+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:35:19.492530+0000","last_undegraded":"2023-10-09T06:35:19.492530+0000","last_fullsized":"2023-10-09T06:35:19.492530+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":442,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099678+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:35:20.099678+0000","last_peered":"2023-10-09T06:35:20.099678+0000","last_clean":"2023-10-09T06:35:20.099678+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:35:20.099678+0000","last_undegraded":"2023-10-09T06:35:20.099678+0000","last_fullsized":"2023-10-09T06:35:20.099678+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":406,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099791+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:35:20.099791+0000","last_peered":"2023-10-09T06:35:20.099791+0000","last_clean":"2023-10-09T06:35:20.099791+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:35:20.099791+0000","last_undegraded":"2023-10-09T06:35:20.099791+0000","last_fullsized":"2023-10-09T06:35:20.099791+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":707,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488374+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:35:19.488374+0000","last_peered":"2023-10-09T06:35:19.488374+0000","last_clean":"2023-10-09T06:35:19.488374+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:35:19.488374+0000","last_undegraded":"2023-10-09T06:35:19.488374+0000","last_fullsized":"2023-10-09T06:35:19.488374+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":285,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653300+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:35:19.653300+0000","last_peered":"2023-10-09T06:35:19.653300+0000","last_clean":"2023-10-09T06:35:19.653300+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:35:19.653300+0000","last_undegraded":"2023-10-09T06:35:19.653300+0000","last_fullsized":"2023-10-09T06:35:19.653300+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494299+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:35:19.494299+0000","last_peered":"2023-10-09T06:35:19.494299+0000","last_clean":"2023-10-09T06:35:19.494299+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:35:19.494299+0000","last_undegraded":"2023-10-09T06:35:19.494299+0000","last_fullsized":"2023-10-09T06:35:19.494299+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":550,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496008+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:35:19.496008+0000","last_peered":"2023-10-09T06:35:19.496008+0000","last_clean":"2023-10-09T06:35:19.496008+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:35:19.496008+0000","last_undegraded":"2023-10-09T06:35:19.496008+0000","last_fullsized":"2023-10-09T06:35:19.496008+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":432,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488457+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:35:19.488457+0000","last_peered":"2023-10-09T06:35:19.488457+0000","last_clean":"2023-10-09T06:35:19.488457+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:35:19.488457+0000","last_undegraded":"2023-10-09T06:35:19.488457+0000","last_fullsized":"2023-10-09T06:35:19.488457+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496983+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:35:19.496983+0000","last_peered":"2023-10-09T06:35:19.496983+0000","last_clean":"2023-10-09T06:35:19.496983+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:35:19.496983+0000","last_undegraded":"2023-10-09T06:35:19.496983+0000","last_fullsized":"2023-10-09T06:35:19.496983+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":761,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491680+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:35:19.491680+0000","last_peered":"2023-10-09T06:35:19.491680+0000","last_clean":"2023-10-09T06:35:19.491680+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:35:19.491680+0000","last_undegraded":"2023-10-09T06:35:19.491680+0000","last_fullsized":"2023-10-09T06:35:19.491680+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":279,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653044+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:35:19.653044+0000","last_peered":"2023-10-09T06:35:19.653044+0000","last_clean":"2023-10-09T06:35:19.653044+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:35:19.653044+0000","last_undegraded":"2023-10-09T06:35:19.653044+0000","last_fullsized":"2023-10-09T06:35:19.653044+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":109,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488881+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:35:19.488881+0000","last_peered":"2023-10-09T06:35:19.488881+0000","last_clean":"2023-10-09T06:35:19.488881+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:35:19.488881+0000","last_undegraded":"2023-10-09T06:35:19.488881+0000","last_fullsized":"2023-10-09T06:35:19.488881+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":550,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496311+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:35:19.496311+0000","last_peered":"2023-10-09T06:35:19.496311+0000","last_clean":"2023-10-09T06:35:19.496311+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:35:19.496311+0000","last_undegraded":"2023-10-09T06:35:19.496311+0000","last_fullsized":"2023-10-09T06:35:19.496311+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":514,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495821+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:35:19.495821+0000","last_peered":"2023-10-09T06:35:19.495821+0000","last_clean":"2023-10-09T06:35:19.495821+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:35:19.495821+0000","last_undegraded":"2023-10-09T06:35:19.495821+0000","last_fullsized":"2023-10-09T06:35:19.495821+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494517+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:35:19.494517+0000","last_peered":"2023-10-09T06:35:19.494517+0000","last_clean":"2023-10-09T06:35:19.494517+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:35:19.494517+0000","last_undegraded":"2023-10-09T06:35:19.494517+0000","last_fullsized":"2023-10-09T06:35:19.494517+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":563,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653475+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:35:19.653475+0000","last_peered":"2023-10-09T06:35:19.653475+0000","last_clean":"2023-10-09T06:35:19.653475+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:35:19.653475+0000","last_undegraded":"2023-10-09T06:35:19.653475+0000","last_fullsized":"2023-10-09T06:35:19.653475+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":321,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653918+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:35:19.653918+0000","last_peered":"2023-10-09T06:35:19.653918+0000","last_clean":"2023-10-09T06:35:19.653918+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:35:19.653918+0000","last_undegraded":"2023-10-09T06:35:19.653918+0000","last_fullsized":"2023-10-09T06:35:19.653918+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496184+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:35:19.496184+0000","last_peered":"2023-10-09T06:35:19.496184+0000","last_clean":"2023-10-09T06:35:19.496184+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:35:19.496184+0000","last_undegraded":"2023-10-09T06:35:19.496184+0000","last_fullsized":"2023-10-09T06:35:19.496184+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":820,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493889+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:35:19.493889+0000","last_peered":"2023-10-09T06:35:19.493889+0000","last_clean":"2023-10-09T06:35:19.493889+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:35:19.493889+0000","last_undegraded":"2023-10-09T06:35:19.493889+0000","last_fullsized":"2023-10-09T06:35:19.493889+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":283,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653872+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:35:19.653872+0000","last_peered":"2023-10-09T06:35:19.653872+0000","last_clean":"2023-10-09T06:35:19.653872+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:35:19.653872+0000","last_undegraded":"2023-10-09T06:35:19.653872+0000","last_fullsized":"2023-10-09T06:35:19.653872+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":433,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099435+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:35:20.099435+0000","last_peered":"2023-10-09T06:35:20.099435+0000","last_clean":"2023-10-09T06:35:20.099435+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:35:20.099435+0000","last_undegraded":"2023-10-09T06:35:20.099435+0000","last_fullsized":"2023-10-09T06:35:20.099435+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":737,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497315+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:35:19.497315+0000","last_peered":"2023-10-09T06:35:19.497315+0000","last_clean":"2023-10-09T06:35:19.497315+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:35:19.497315+0000","last_undegraded":"2023-10-09T06:35:19.497315+0000","last_fullsized":"2023-10-09T06:35:19.497315+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":339,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653765+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:35:19.653765+0000","last_peered":"2023-10-09T06:35:19.653765+0000","last_clean":"2023-10-09T06:35:19.653765+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:35:19.653765+0000","last_undegraded":"2023-10-09T06:35:19.653765+0000","last_fullsized":"2023-10-09T06:35:19.653765+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":412,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496884+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:35:19.496884+0000","last_peered":"2023-10-09T06:35:19.496884+0000","last_clean":"2023-10-09T06:35:19.496884+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:35:19.496884+0000","last_undegraded":"2023-10-09T06:35:19.496884+0000","last_fullsized":"2023-10-09T06:35:19.496884+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":560,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099816+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:35:20.099816+0000","last_peered":"2023-10-09T06:35:20.099816+0000","last_clean":"2023-10-09T06:35:20.099816+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:35:20.099816+0000","last_undegraded":"2023-10-09T06:35:20.099816+0000","last_fullsized":"2023-10-09T06:35:20.099816+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497172+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:35:19.497172+0000","last_peered":"2023-10-09T06:35:19.497172+0000","last_clean":"2023-10-09T06:35:19.497172+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:35:19.497172+0000","last_undegraded":"2023-10-09T06:35:19.497172+0000","last_fullsized":"2023-10-09T06:35:19.497172+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":397,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488820+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:35:19.488820+0000","last_peered":"2023-10-09T06:35:19.488820+0000","last_clean":"2023-10-09T06:35:19.488820+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:35:19.488820+0000","last_undegraded":"2023-10-09T06:35:19.488820+0000","last_fullsized":"2023-10-09T06:35:19.488820+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":788,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.652898+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:35:19.652898+0000","last_peered":"2023-10-09T06:35:19.652898+0000","last_clean":"2023-10-09T06:35:19.652898+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:35:19.652898+0000","last_undegraded":"2023-10-09T06:35:19.652898+0000","last_fullsized":"2023-10-09T06:35:19.652898+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":162,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493969+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:35:19.493969+0000","last_peered":"2023-10-09T06:35:19.493969+0000","last_clean":"2023-10-09T06:35:19.493969+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:35:19.493969+0000","last_undegraded":"2023-10-09T06:35:19.493969+0000","last_fullsized":"2023-10-09T06:35:19.493969+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":416,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099557+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:35:20.099557+0000","last_peered":"2023-10-09T06:35:20.099557+0000","last_clean":"2023-10-09T06:35:20.099557+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:35:20.099557+0000","last_undegraded":"2023-10-09T06:35:20.099557+0000","last_fullsized":"2023-10-09T06:35:20.099557+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":768,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496107+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:35:19.496107+0000","last_peered":"2023-10-09T06:35:19.496107+0000","last_clean":"2023-10-09T06:35:19.496107+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:35:19.496107+0000","last_undegraded":"2023-10-09T06:35:19.496107+0000","last_fullsized":"2023-10-09T06:35:19.496107+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":393,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496736+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:35:19.496736+0000","last_peered":"2023-10-09T06:35:19.496736+0000","last_clean":"2023-10-09T06:35:19.496736+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:35:19.496736+0000","last_undegraded":"2023-10-09T06:35:19.496736+0000","last_fullsized":"2023-10-09T06:35:19.496736+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":413,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099016+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:35:20.099016+0000","last_peered":"2023-10-09T06:35:20.099016+0000","last_clean":"2023-10-09T06:35:20.099016+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:35:20.099016+0000","last_undegraded":"2023-10-09T06:35:20.099016+0000","last_fullsized":"2023-10-09T06:35:20.099016+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":770,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491758+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:35:19.491758+0000","last_peered":"2023-10-09T06:35:19.491758+0000","last_clean":"2023-10-09T06:35:19.491758+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:35:19.491758+0000","last_undegraded":"2023-10-09T06:35:19.491758+0000","last_fullsized":"2023-10-09T06:35:19.491758+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":417,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099317+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:35:20.099317+0000","last_peered":"2023-10-09T06:35:20.099317+0000","last_clean":"2023-10-09T06:35:20.099317+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:35:20.099317+0000","last_undegraded":"2023-10-09T06:35:20.099317+0000","last_fullsized":"2023-10-09T06:35:20.099317+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":396,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099951+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:35:20.099951+0000","last_peered":"2023-10-09T06:35:20.099951+0000","last_clean":"2023-10-09T06:35:20.099951+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:35:20.099951+0000","last_undegraded":"2023-10-09T06:35:20.099951+0000","last_fullsized":"2023-10-09T06:35:20.099951+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":746,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496443+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:35:19.496443+0000","last_peered":"2023-10-09T06:35:19.496443+0000","last_clean":"2023-10-09T06:35:19.496443+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:35:19.496443+0000","last_undegraded":"2023-10-09T06:35:19.496443+0000","last_fullsized":"2023-10-09T06:35:19.496443+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494880+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:35:19.494880+0000","last_peered":"2023-10-09T06:35:19.494880+0000","last_clean":"2023-10-09T06:35:19.494880+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:35:19.494880+0000","last_undegraded":"2023-10-09T06:35:19.494880+0000","last_fullsized":"2023-10-09T06:35:19.494880+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":389,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491111+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:35:19.491111+0000","last_peered":"2023-10-09T06:35:19.491111+0000","last_clean":"2023-10-09T06:35:19.491111+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:35:19.491111+0000","last_undegraded":"2023-10-09T06:35:19.491111+0000","last_fullsized":"2023-10-09T06:35:19.491111+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":728,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497593+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:35:19.497593+0000","last_peered":"2023-10-09T06:35:19.497593+0000","last_clean":"2023-10-09T06:35:19.497593+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:35:19.497593+0000","last_undegraded":"2023-10-09T06:35:19.497593+0000","last_fullsized":"2023-10-09T06:35:19.497593+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":441,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493523+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:35:19.493523+0000","last_peered":"2023-10-09T06:35:19.493523+0000","last_clean":"2023-10-09T06:35:19.493523+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:35:19.493523+0000","last_undegraded":"2023-10-09T06:35:19.493523+0000","last_fullsized":"2023-10-09T06:35:19.493523+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496891+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:35:19.496891+0000","last_peered":"2023-10-09T06:35:19.496891+0000","last_clean":"2023-10-09T06:35:19.496891+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:35:19.496891+0000","last_undegraded":"2023-10-09T06:35:19.496891+0000","last_fullsized":"2023-10-09T06:35:19.496891+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":375,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491190+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:35:19.491190+0000","last_peered":"2023-10-09T06:35:19.491190+0000","last_clean":"2023-10-09T06:35:19.491190+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:35:19.491190+0000","last_undegraded":"2023-10-09T06:35:19.491190+0000","last_fullsized":"2023-10-09T06:35:19.491190+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":782,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496255+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:35:19.496255+0000","last_peered":"2023-10-09T06:35:19.496255+0000","last_clean":"2023-10-09T06:35:19.496255+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:35:19.496255+0000","last_undegraded":"2023-10-09T06:35:19.496255+0000","last_fullsized":"2023-10-09T06:35:19.496255+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496601+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:35:19.496601+0000","last_peered":"2023-10-09T06:35:19.496601+0000","last_clean":"2023-10-09T06:35:19.496601+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:35:19.496601+0000","last_undegraded":"2023-10-09T06:35:19.496601+0000","last_fullsized":"2023-10-09T06:35:19.496601+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":417,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099698+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:35:20.099698+0000","last_peered":"2023-10-09T06:35:20.099698+0000","last_clean":"2023-10-09T06:35:20.099698+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:35:20.099698+0000","last_undegraded":"2023-10-09T06:35:20.099698+0000","last_fullsized":"2023-10-09T06:35:20.099698+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":405,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493453+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:35:19.493453+0000","last_peered":"2023-10-09T06:35:19.493453+0000","last_clean":"2023-10-09T06:35:19.493453+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:35:19.493453+0000","last_undegraded":"2023-10-09T06:35:19.493453+0000","last_fullsized":"2023-10-09T06:35:19.493453+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":778,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653149+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:35:19.653149+0000","last_peered":"2023-10-09T06:35:19.653149+0000","last_clean":"2023-10-09T06:35:19.653149+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:35:19.653149+0000","last_undegraded":"2023-10-09T06:35:19.653149+0000","last_fullsized":"2023-10-09T06:35:19.653149+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":427,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495707+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:35:19.495707+0000","last_peered":"2023-10-09T06:35:19.495707+0000","last_clean":"2023-10-09T06:35:19.495707+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:35:19.495707+0000","last_undegraded":"2023-10-09T06:35:19.495707+0000","last_fullsized":"2023-10-09T06:35:19.495707+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":414,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491869+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:35:19.491869+0000","last_peered":"2023-10-09T06:35:19.491869+0000","last_clean":"2023-10-09T06:35:19.491869+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:35:19.491869+0000","last_undegraded":"2023-10-09T06:35:19.491869+0000","last_fullsized":"2023-10-09T06:35:19.491869+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":436,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493708+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:35:19.493708+0000","last_peered":"2023-10-09T06:35:19.493708+0000","last_clean":"2023-10-09T06:35:19.493708+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:35:19.493708+0000","last_undegraded":"2023-10-09T06:35:19.493708+0000","last_fullsized":"2023-10-09T06:35:19.493708+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":123,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496933+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:35:19.496933+0000","last_peered":"2023-10-09T06:35:19.496933+0000","last_clean":"2023-10-09T06:35:19.496933+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:35:19.496933+0000","last_undegraded":"2023-10-09T06:35:19.496933+0000","last_fullsized":"2023-10-09T06:35:19.496933+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":759,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653057+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:35:19.653057+0000","last_peered":"2023-10-09T06:35:19.653057+0000","last_clean":"2023-10-09T06:35:19.653057+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:35:19.653057+0000","last_undegraded":"2023-10-09T06:35:19.653057+0000","last_fullsized":"2023-10-09T06:35:19.653057+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":145,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494672+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:35:19.494672+0000","last_peered":"2023-10-09T06:35:19.494672+0000","last_clean":"2023-10-09T06:35:19.494672+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:35:19.494672+0000","last_undegraded":"2023-10-09T06:35:19.494672+0000","last_fullsized":"2023-10-09T06:35:19.494672+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":398,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496788+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:35:19.496788+0000","last_peered":"2023-10-09T06:35:19.496788+0000","last_clean":"2023-10-09T06:35:19.496788+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:35:19.496788+0000","last_undegraded":"2023-10-09T06:35:19.496788+0000","last_fullsized":"2023-10-09T06:35:19.496788+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":744,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.491835+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:35:19.491835+0000","last_peered":"2023-10-09T06:35:19.491835+0000","last_clean":"2023-10-09T06:35:19.491835+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:35:19.491835+0000","last_undegraded":"2023-10-09T06:35:19.491835+0000","last_fullsized":"2023-10-09T06:35:19.491835+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":299,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.652932+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:35:19.652932+0000","last_peered":"2023-10-09T06:35:19.652932+0000","last_clean":"2023-10-09T06:35:19.652932+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:35:19.652932+0000","last_undegraded":"2023-10-09T06:35:19.652932+0000","last_fullsized":"2023-10-09T06:35:19.652932+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":109,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488571+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:35:19.488571+0000","last_peered":"2023-10-09T06:35:19.488571+0000","last_clean":"2023-10-09T06:35:19.488571+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:35:19.488571+0000","last_undegraded":"2023-10-09T06:35:19.488571+0000","last_fullsized":"2023-10-09T06:35:19.488571+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":420,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.489212+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:35:19.489212+0000","last_peered":"2023-10-09T06:35:19.489212+0000","last_clean":"2023-10-09T06:35:19.489212+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:35:19.489212+0000","last_undegraded":"2023-10-09T06:35:19.489212+0000","last_fullsized":"2023-10-09T06:35:19.489212+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":758,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497160+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:35:19.497160+0000","last_peered":"2023-10-09T06:35:19.497160+0000","last_clean":"2023-10-09T06:35:19.497160+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:35:19.497160+0000","last_undegraded":"2023-10-09T06:35:19.497160+0000","last_fullsized":"2023-10-09T06:35:19.497160+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":299,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653705+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:35:19.653705+0000","last_peered":"2023-10-09T06:35:19.653705+0000","last_clean":"2023-10-09T06:35:19.653705+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:35:19.653705+0000","last_undegraded":"2023-10-09T06:35:19.653705+0000","last_fullsized":"2023-10-09T06:35:19.653705+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":431,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493381+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:35:19.493381+0000","last_peered":"2023-10-09T06:35:19.493381+0000","last_clean":"2023-10-09T06:35:19.493381+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:35:19.493381+0000","last_undegraded":"2023-10-09T06:35:19.493381+0000","last_fullsized":"2023-10-09T06:35:19.493381+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":418,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.489003+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:35:19.489003+0000","last_peered":"2023-10-09T06:35:19.489003+0000","last_clean":"2023-10-09T06:35:19.489003+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:35:19.489003+0000","last_undegraded":"2023-10-09T06:35:19.489003+0000","last_fullsized":"2023-10-09T06:35:19.489003+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":759,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099570+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:35:20.099570+0000","last_peered":"2023-10-09T06:35:20.099570+0000","last_clean":"2023-10-09T06:35:20.099570+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:35:20.099570+0000","last_undegraded":"2023-10-09T06:35:20.099570+0000","last_fullsized":"2023-10-09T06:35:20.099570+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":166,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494645+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:35:19.494645+0000","last_peered":"2023-10-09T06:35:19.494645+0000","last_clean":"2023-10-09T06:35:19.494645+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:35:19.494645+0000","last_undegraded":"2023-10-09T06:35:19.494645+0000","last_fullsized":"2023-10-09T06:35:19.494645+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":399,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.492736+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:35:19.492736+0000","last_peered":"2023-10-09T06:35:19.492736+0000","last_clean":"2023-10-09T06:35:19.492736+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:35:19.492736+0000","last_undegraded":"2023-10-09T06:35:19.492736+0000","last_fullsized":"2023-10-09T06:35:19.492736+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":154,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494841+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:35:19.494841+0000","last_peered":"2023-10-09T06:35:19.494841+0000","last_clean":"2023-10-09T06:35:19.494841+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:35:19.494841+0000","last_undegraded":"2023-10-09T06:35:19.494841+0000","last_fullsized":"2023-10-09T06:35:19.494841+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":788,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494418+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:35:19.494418+0000","last_peered":"2023-10-09T06:35:19.494418+0000","last_clean":"2023-10-09T06:35:19.494418+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:35:19.494418+0000","last_undegraded":"2023-10-09T06:35:19.494418+0000","last_fullsized":"2023-10-09T06:35:19.494418+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":420,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497667+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:35:19.497667+0000","last_peered":"2023-10-09T06:35:19.497667+0000","last_clean":"2023-10-09T06:35:19.497667+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:35:19.497667+0000","last_undegraded":"2023-10-09T06:35:19.497667+0000","last_fullsized":"2023-10-09T06:35:19.497667+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":401,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.489039+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:35:19.489039+0000","last_peered":"2023-10-09T06:35:19.489039+0000","last_clean":"2023-10-09T06:35:19.489039+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:35:19.489039+0000","last_undegraded":"2023-10-09T06:35:19.489039+0000","last_fullsized":"2023-10-09T06:35:19.489039+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":399,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099410+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:35:20.099410+0000","last_peered":"2023-10-09T06:35:20.099410+0000","last_clean":"2023-10-09T06:35:20.099410+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:35:20.099410+0000","last_undegraded":"2023-10-09T06:35:20.099410+0000","last_fullsized":"2023-10-09T06:35:20.099410+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":715,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495530+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:35:19.495530+0000","last_peered":"2023-10-09T06:35:19.495530+0000","last_clean":"2023-10-09T06:35:19.495530+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:35:19.495530+0000","last_undegraded":"2023-10-09T06:35:19.495530+0000","last_fullsized":"2023-10-09T06:35:19.495530+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":402,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488926+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:35:19.488926+0000","last_peered":"2023-10-09T06:35:19.488926+0000","last_clean":"2023-10-09T06:35:19.488926+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:35:19.488926+0000","last_undegraded":"2023-10-09T06:35:19.488926+0000","last_fullsized":"2023-10-09T06:35:19.488926+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":424,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.493303+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:35:19.493303+0000","last_peered":"2023-10-09T06:35:19.493303+0000","last_clean":"2023-10-09T06:35:19.493303+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:35:19.493303+0000","last_undegraded":"2023-10-09T06:35:19.493303+0000","last_fullsized":"2023-10-09T06:35:19.493303+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":645,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494031+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:35:19.494031+0000","last_peered":"2023-10-09T06:35:19.494031+0000","last_clean":"2023-10-09T06:35:19.494031+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:35:19.494031+0000","last_undegraded":"2023-10-09T06:35:19.494031+0000","last_fullsized":"2023-10-09T06:35:19.494031+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":127,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.497185+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:35:19.497185+0000","last_peered":"2023-10-09T06:35:19.497185+0000","last_clean":"2023-10-09T06:35:19.497185+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:35:19.497185+0000","last_undegraded":"2023-10-09T06:35:19.497185+0000","last_fullsized":"2023-10-09T06:35:19.497185+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":443,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099652+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:35:20.099652+0000","last_peered":"2023-10-09T06:35:20.099652+0000","last_clean":"2023-10-09T06:35:20.099652+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:35:20.099652+0000","last_undegraded":"2023-10-09T06:35:20.099652+0000","last_fullsized":"2023-10-09T06:35:20.099652+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":127,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.496706+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:35:19.496706+0000","last_peered":"2023-10-09T06:35:19.496706+0000","last_clean":"2023-10-09T06:35:19.496706+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:35:19.496706+0000","last_undegraded":"2023-10-09T06:35:19.496706+0000","last_fullsized":"2023-10-09T06:35:19.496706+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":453,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099424+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:35:20.099424+0000","last_peered":"2023-10-09T06:35:20.099424+0000","last_clean":"2023-10-09T06:35:20.099424+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:35:20.099424+0000","last_undegraded":"2023-10-09T06:35:20.099424+0000","last_fullsized":"2023-10-09T06:35:20.099424+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":792,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.653614+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:35:19.653614+0000","last_peered":"2023-10-09T06:35:19.653614+0000","last_clean":"2023-10-09T06:35:19.653614+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:35:19.653614+0000","last_undegraded":"2023-10-09T06:35:19.653614+0000","last_fullsized":"2023-10-09T06:35:19.653614+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":136,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.494142+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:35:19.494142+0000","last_peered":"2023-10-09T06:35:19.494142+0000","last_clean":"2023-10-09T06:35:19.494142+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:35:19.494142+0000","last_undegraded":"2023-10-09T06:35:19.494142+0000","last_fullsized":"2023-10-09T06:35:19.494142+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":154,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.495880+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:35:19.495880+0000","last_peered":"2023-10-09T06:35:19.495880+0000","last_clean":"2023-10-09T06:35:19.495880+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:35:19.495880+0000","last_undegraded":"2023-10-09T06:35:19.495880+0000","last_fullsized":"2023-10-09T06:35:19.495880+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":557,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.492415+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:35:19.492415+0000","last_peered":"2023-10-09T06:35:19.492415+0000","last_clean":"2023-10-09T06:35:19.492415+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:35:19.492415+0000","last_undegraded":"2023-10-09T06:35:19.492415+0000","last_fullsized":"2023-10-09T06:35:19.492415+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":431,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:19.488881+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:35:19.488881+0000","last_peered":"2023-10-09T06:35:19.488881+0000","last_clean":"2023-10-09T06:35:19.488881+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:35:19.488881+0000","last_undegraded":"2023-10-09T06:35:19.488881+0000","last_fullsized":"2023-10-09T06:35:19.488881+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":396,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-09T06:35:20.099267+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:35:20.099267+0000","last_peered":"2023-10-09T06:35:20.099267+0000","last_clean":"2023-10-09T06:35:20.099267+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:35:20.099267+0000","last_undegraded":"2023-10-09T06:35:20.099267+0000","last_fullsized":"2023-10-09T06:35:20.099267+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":459024,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":394032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234457,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1067484,"kb_used_data":167252,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92677668,"statfs":{"total":95995035648,"available":94901932032,"internally_reserved":0,"allocated":171266048,"data_stored":170227603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:08 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":4.6699999999999999,"15min":6.1829999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":2.984,"5min":196.28899999999999,"15min":219.41900000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.2470000000000001,"5min":4.79,"15min":6.3479999999999999},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":3.1160000000000001,"5min":196.333,"15min":219.399},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":3.8900000000000001,"15min":4.3540000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":6.3920000000000003,"5min":107.87,"15min":215.61699999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":3.3660000000000001,"15min":4.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":8.0830000000000002,"5min":107.904,"15min":216.22200000000001},"last":0.86299999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":4.2050000000000001,"15min":6.9450000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":39.747,"5min":198.94200000000001,"15min":216.34299999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":4.4509999999999996,"5min":4.2679999999999998,"15min":7.0730000000000004},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":39.658999999999999,"5min":198.92099999999999,"15min":216.25200000000001},"last":0.61599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":6.0700000000000003,"15min":4.6120000000000001},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":4.9809999999999999,"5min":187.34200000000001,"15min":215.922},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.929,"5min":6.0670000000000002,"15min":4.6959999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":5.9189999999999996,"5min":187.08199999999999,"15min":215.94900000000001},"last":0.65800000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.79600000000000004,"15min":2.4729999999999999},"min":{"1min":0.26900000000000002,"5min":0.26800000000000002,"15min":0.23300000000000001},"max":{"1min":0.56999999999999995,"5min":6.1689999999999996,"15min":218.446},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.74299999999999999,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.23699999999999999},"max":{"1min":1.7190000000000001,"5min":4.891,"15min":216.172},"last":0.71599999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.86699999999999999,"15min":2.4620000000000002},"min":{"1min":0.29599999999999999,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":4.2309999999999999,"5min":8.8170000000000002,"15min":216.37200000000001},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.087,"15min":2.3889999999999998},"min":{"1min":0.33500000000000002,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":2.1000000000000001,"5min":6.3019999999999996,"15min":218.15899999999999},"last":0.98699999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.123,"15min":2.2909999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28000000000000003},"max":{"1min":2.6560000000000001,"5min":15.162000000000001,"15min":218.19999999999999},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.89000000000000001,"15min":2.0979999999999999},"min":{"1min":0.36799999999999999,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":1.7589999999999999,"5min":8.2919999999999998,"15min":218.05500000000001},"last":0.755}]}]},{"osd":6,"up_from":46,"seq":197568496093,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":954756,"kb_used_data":55740,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92790396,"statfs":{"total":95995035648,"available":95017365504,"internally_reserved":0,"allocated":57077760,"data_stored":56062613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4930000000000003,"5min":4.1479999999999997,"15min":3.2290000000000001},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":66.867000000000004,"5min":75.061999999999998,"15min":110.926},"last":0.36799999999999999},{"interface":"front","average":{"1min":5.625,"5min":4.2839999999999998,"15min":3.4769999999999999},"min":{"1min":0.30299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":66.856999999999999,"5min":75.021000000000001,"15min":110.911},"last":0.41199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":3.3300000000000001,"15min":5.0759999999999996},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.7949999999999999,"5min":82.159000000000006,"15min":292.92200000000003},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.853,"5min":3.5569999999999999,"15min":4.9569999999999999},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":9.2560000000000002,"5min":82.174000000000007,"15min":292.904},"last":0.53500000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":3.3450000000000002,"15min":3.1800000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":10.239000000000001,"5min":74.748999999999995,"15min":75.191999999999993},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.8080000000000001,"5min":3.3799999999999999,"15min":2.9249999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.281000000000001,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":4.0359999999999996,"15min":3.5680000000000001},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.336,"5min":88.763000000000005,"15min":110.358},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.744,"5min":3.8959999999999999,"15min":3.6150000000000002},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":6.7350000000000003,"5min":86.831999999999994,"15min":110.43899999999999},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.73,"15min":1.488},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":86.171999999999997,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.556,"5min":1.617,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.223},"max":{"1min":86.727000000000004,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.93799999999999994,"15min":1.3540000000000001},"min":{"1min":0.33300000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.907,"5min":5.7539999999999996,"15min":22.379999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1599999999999999,"15min":1.393},"min":{"1min":0.33300000000000002,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":4.6219999999999999,"5min":8.8000000000000007,"15min":24.401},"last":0.72499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.204,"15min":2.528},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.26600000000000001},"max":{"1min":2.105,"5min":19.295999999999999,"15min":122.937},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.4690000000000001,"15min":2.673},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.4049999999999998,"5min":19.584,"15min":122.92100000000001},"last":0.751}]}]},{"osd":1,"up_from":110,"seq":472446402830,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815512,"kb_used_data":5264,"kb_used_omap":4,"kb_used_meta":810235,"kb_avail":92929640,"statfs":{"total":95995035648,"available":95159951360,"internally_reserved":0,"allocated":5390336,"data_stored":3423034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":829681550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7599999999999998,"5min":6.2249999999999996,"15min":6.3029999999999999},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":19.699999999999999,"5min":143.465,"15min":269.79199999999997},"last":0.26600000000000001},{"interface":"front","average":{"1min":4.1660000000000004,"5min":6.2460000000000004,"15min":6.7720000000000002},"min":{"1min":0.23200000000000001,"5min":0.215,"15min":0.17899999999999999},"max":{"1min":11.663,"5min":143.66399999999999,"15min":271.20999999999998},"last":0.33300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:34:45 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":4.0869999999999997,"15min":3.6629999999999998},"min":{"1min":0.23400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":13.217000000000001,"5min":138.452,"15min":139.75399999999999},"last":0.317},{"interface":"front","average":{"1min":3.3069999999999999,"5min":4.21,"15min":4.4119999999999999},"min":{"1min":0.254,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.244,"5min":138.47499999999999,"15min":140.29900000000001},"last":0.37}]},{"osd":3,"last update":"Mon Oct 9 06:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":3.2890000000000001,"15min":3.9620000000000002},"min":{"1min":0.28499999999999998,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":8.9420000000000002,"5min":27.064,"15min":153.18199999999999},"last":0.379},{"interface":"front","average":{"1min":2.891,"5min":2.1920000000000002,"15min":2.819},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":13.129,"5min":26.626999999999999,"15min":152.52000000000001},"last":0.39100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6739999999999999,"15min":3.7789999999999999},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":7.6159999999999997,"5min":38.835999999999999,"15min":199.43799999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.895,"15min":3.4140000000000001},"min":{"1min":0.36599999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":8.8089999999999993,"5min":38.819000000000003,"15min":199.32900000000001},"last":0.59699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.5750000000000002,"15min":2.339},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.163,"5min":52.512999999999998,"15min":77.436999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.5619999999999998,"15min":2.7040000000000002},"min":{"1min":0.40999999999999998,"5min":0.371,"15min":0.32200000000000001},"max":{"1min":8.7759999999999998,"5min":52.664999999999999,"15min":77.388000000000005},"last":0.50900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":3.6949999999999998,"15min":3.1890000000000001},"min":{"1min":0.39900000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.9250000000000007,"5min":107.27200000000001,"15min":107.27200000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.52,"5min":4.452,"15min":3.4700000000000002},"min":{"1min":0.379,"5min":0.316,"15min":0.316},"max":{"1min":7.6950000000000003,"5min":107.252,"15min":107.252},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":4.2190000000000003,"15min":3.6389999999999998},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":7.6529999999999996,"5min":104.07299999999999,"15min":104.07299999999999},"last":0.5},{"interface":"front","average":{"1min":2.7309999999999999,"5min":3.4750000000000001,"15min":3.0110000000000001},"min":{"1min":0.35399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.191000000000001,"5min":104.056,"15min":104.056},"last":0.61499999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024263,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961496,"kb_used_data":5904,"kb_used_omap":5,"kb_used_meta":955578,"kb_avail":92783656,"statfs":{"total":95995035648,"available":95010463744,"internally_reserved":0,"allocated":6045696,"data_stored":3789038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":978512506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":24,"apply_latency_ms":24,"commit_latency_ns":24000000,"apply_latency_ns":24000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.9930000000000001,"15min":3.4260000000000002},"min":{"1min":0.19900000000000001,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":33.091999999999999,"5min":33.091999999999999,"15min":191.446},"last":0.28299999999999997},{"interface":"front","average":{"1min":4.25,"5min":2.6120000000000001,"15min":3.7109999999999999},"min":{"1min":0.23799999999999999,"5min":0.187,"15min":0.187},"max":{"1min":24.838999999999999,"5min":24.838999999999999,"15min":191.393},"last":0.29799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:34:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4199999999999999,"5min":2.8119999999999998,"15min":2.6429999999999998},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":12.532,"5min":16.936,"15min":57.162999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":4.133,"5min":3.524,"15min":3.121},"min":{"1min":0.22800000000000001,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":13.579000000000001,"5min":15.794,"15min":57.585000000000001},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":5.7140000000000004,"15min":6.0190000000000001},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":24.129000000000001,"5min":187.172,"15min":208.11199999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":5.3150000000000004,"5min":5.6699999999999999,"15min":5.9800000000000004},"min":{"1min":0.34399999999999997,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":24.797000000000001,"5min":194.851,"15min":208.25899999999999},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.649,"15min":1.5760000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":9.7159999999999993},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.504,"5min":2.2090000000000001,"15min":1.929},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.385999999999999,"5min":12.385999999999999,"15min":18.081},"last":0.60499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.7669999999999999,"15min":1.873},"min":{"1min":0.309,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":9.0099999999999998,"5min":9.0099999999999998,"15min":62.984999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.601,"5min":2.3380000000000001,"15min":2.3730000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.8239999999999998,"5min":13.968999999999999,"15min":64.983999999999995},"last":0.59399999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.1829999999999998,"5min":2.4550000000000001,"15min":3.036},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":22.292000000000002,"5min":22.292000000000002,"15min":120.572},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9910000000000001,"15min":2.52},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30299999999999999},"max":{"1min":22.475000000000001,"5min":22.475000000000001,"15min":120.64400000000001},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.472,"15min":2.734},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.56399999999999995},{"interface":"front","average":{"1min":4.1139999999999999,"5min":2.79,"15min":3.125},"min":{"1min":0.39700000000000002,"5min":0.39100000000000001,"15min":0.377},"max":{"1min":22.683,"5min":22.683,"15min":122.32299999999999},"last":0.63800000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814132,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":730632,"kb_used_data":5440,"kb_used_omap":7,"kb_used_meta":725176,"kb_avail":93014520,"statfs":{"total":95995035648,"available":95246868480,"internally_reserved":0,"allocated":5570560,"data_stored":3673870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":742580305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":33,"apply_latency_ms":33,"commit_latency_ns":33000000,"apply_latency_ns":33000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":15.65,"5min":7.032,"15min":5.5129999999999999},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":156.94499999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":15.272,"5min":6.734,"15min":5.0049999999999999},"min":{"1min":0.315,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":156.922,"5min":156.922,"15min":168.49700000000001},"last":0.38}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":14.833,"5min":6.202,"15min":4.7000000000000002},"min":{"1min":0.41499999999999998,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":140.452,"5min":140.452,"15min":205.148},"last":0.39500000000000002},{"interface":"front","average":{"1min":14.69,"5min":5.9409999999999998,"15min":4.8739999999999997},"min":{"1min":0.35599999999999998,"5min":0.183,"15min":0.183},"max":{"1min":139.75299999999999,"5min":139.75299999999999,"15min":205.267},"last":0.34000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":3.6709999999999998,"15min":3.27},"min":{"1min":0.26600000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.243,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.8260000000000001,"5min":4.0609999999999999,"15min":3.698},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":11.176,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.41599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.0249999999999999,"15min":2.27},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":3.423,"5min":44.314,"15min":90.938999999999993},"last":0.92100000000000004},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.4329999999999998,"15min":2.6019999999999999},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":4.6950000000000003,"5min":43.896000000000001,"15min":90.950000000000003},"last":0.90900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.794,"15min":1.675},"min":{"1min":0.379,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7250000000000001,"5min":13.102,"15min":17.800000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.958,"5min":2.0009999999999999,"15min":1.9630000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.6779999999999999,"5min":11.285,"15min":11.285},"last":0.81999999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":4.524,"15min":3.4060000000000001},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":20.445,"5min":123.47799999999999,"15min":138.291},"last":0.61399999999999999},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.3840000000000003,"15min":3.202},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.148,"5min":122.517,"15min":138.01599999999999},"last":0.67100000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":2.7839999999999998,"15min":2.3820000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":16.574999999999999,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.444,"5min":2.3570000000000002,"15min":2.1549999999999998},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":3.3330000000000002,"5min":47.206000000000003,"15min":55.731000000000002},"last":0.627}]}]},{"osd":3,"up_from":149,"seq":639950127330,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840816,"kb_used_data":154216,"kb_used_omap":2,"kb_used_meta":686589,"kb_avail":92904336,"statfs":{"total":95995035648,"available":95134040064,"internally_reserved":0,"allocated":157917184,"data_stored":156102772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":703067189},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":7.1669999999999998,"15min":5.8540000000000001},"min":{"1min":0.20000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":22.367000000000001,"5min":161.821,"15min":187.852},"last":0.439},{"interface":"front","average":{"1min":2.3140000000000001,"5min":6.1689999999999996,"15min":5.8490000000000002},"min":{"1min":0.215,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":13.443,"5min":163.52000000000001,"15min":187.923},"last":0.32000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9209999999999998,"5min":4.4930000000000003,"15min":4.375},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":22.349,"5min":155.23099999999999,"15min":187.89099999999999},"last":0.373},{"interface":"front","average":{"1min":4.1050000000000004,"5min":6,"15min":4.2480000000000002},"min":{"1min":0.25,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":15.066000000000001,"5min":155.24799999999999,"15min":155.24799999999999},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":4.7329999999999997,"15min":4.234},"min":{"1min":0.40799999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":22.297000000000001,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":3.5720000000000001,"5min":5.4390000000000001,"15min":4.2380000000000004},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":13.631,"5min":168.16900000000001,"15min":187.79499999999999},"last":0.497}]},{"osd":4,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.9159999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.661,"5min":48.249000000000002,"15min":48.249000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.7589999999999999,"5min":2.887,"15min":2.645},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":16.516999999999999,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.50900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":2.7389999999999999,"15min":2.7250000000000001},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":16.494,"5min":48.162999999999997,"15min":187.833},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.2090000000000001,"5min":2.1520000000000001,"15min":2.4300000000000002},"min":{"1min":0.42299999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":18.835999999999999,"5min":48.106000000000002,"15min":187.952},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8500000000000001,"5min":4.3339999999999996,"15min":3.3319999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.462,"5min":130.39599999999999,"15min":187.78200000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.3199999999999998,"5min":3.8679999999999999,"15min":2.9900000000000002},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.32900000000000001},"max":{"1min":22.260000000000002,"5min":130.67699999999999,"15min":187.81800000000001},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:34:21 2023","interfaces":[{"interface":"back","average":{"1min":2.512,"5min":4.5229999999999997,"15min":3.3769999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":13.695,"5min":121.613,"15min":192.20400000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.0110000000000001,"5min":4.7270000000000003,"15min":3.4569999999999999},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":16.542000000000002,"5min":121.559,"15min":187.86500000000001},"last":0.56899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019366,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1091084,"kb_used_data":208388,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":92654068,"statfs":{"total":95995035648,"available":94877765632,"internally_reserved":0,"allocated":213389312,"data_stored":212366350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":17.963999999999999,"5min":14.997,"15min":6.7309999999999999},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":286.30099999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":18.295000000000002,"5min":14.887,"15min":6.9299999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":286.36000000000001,"5min":287.57100000000003,"15min":287.57100000000003},"last":0.66700000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":10.643000000000001,"5min":5.5620000000000003,"15min":3.4590000000000001},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":134.37100000000001,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":10.766,"5min":5.7549999999999999,"15min":3.669},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":134.21700000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.41499999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":5.3170000000000002,"15min":3.7749999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":148.36099999999999,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":12.159000000000001,"5min":5.2990000000000004,"15min":3.7559999999999998},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":145.755,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:35:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":6.649,"15min":5.702},"min":{"1min":0.312,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":4.5529999999999999,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.258,"5min":6.5609999999999999,"15min":5.5380000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":4.3360000000000003,"5min":161.934,"15min":161.934},"last":2.1549999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.70699999999999996,"15min":0.98999999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80600000000000005,"5min":6.0199999999999996,"15min":10.673},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.82799999999999996,"15min":1.0029999999999999},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.548,"5min":5.7000000000000002,"15min":12.552},"last":0.77300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0860000000000003,"5min":2.2149999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":115.70999999999999,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":8.4830000000000005,"5min":2.3029999999999999,"15min":2.1150000000000002},"min":{"1min":0.35199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":115.68600000000001,"5min":115.68600000000001,"15min":163.381},"last":0.69399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:35:20 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":2.7650000000000001,"15min":1.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":106.801,"5min":106.801,"15min":106.801},"last":0.75600000000000001},{"interface":"front","average":{"1min":6.7919999999999998,"5min":2.9340000000000002,"15min":1.871},"min":{"1min":0.41899999999999998,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":106.827,"5min":106.827,"15min":106.827},"last":0.78800000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757731,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889796,"kb_used_data":4476,"kb_used_omap":0,"kb_used_meta":885312,"kb_avail":92855356,"statfs":{"total":95995035648,"available":95083884544,"internally_reserved":0,"allocated":4583424,"data_stored":3564906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906559488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":12.871,"5min":8.1099999999999994,"15min":5.742},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":149.149,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":12.737,"5min":7.6159999999999997,"15min":5.5460000000000003},"min":{"1min":0.245,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":149.21100000000001,"5min":294.30399999999997,"15min":294.30399999999997},"last":0.57899999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":7.4349999999999996,"5min":4.0570000000000004,"15min":3.274},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":119.024,"5min":119.024,"15min":126.71899999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":7.5579999999999998,"5min":4.2779999999999996,"15min":3.4369999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":119.133,"5min":119.133,"15min":126.931},"last":0.51300000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.4609999999999999,"15min":2.4969999999999999},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.5149999999999997,"5min":64.150000000000006,"15min":81.260000000000005},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.2290000000000001,"15min":2.4289999999999998},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":4.8550000000000004,"5min":64.507000000000005,"15min":81.245000000000005},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 06:35:10 2023","interfaces":[{"interface":"back","average":{"1min":9.0340000000000007,"5min":4.4349999999999996,"15min":5.1299999999999999},"min":{"1min":0.35399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":85.671000000000006,"5min":92.521000000000001,"15min":144.56399999999999},"last":1.0389999999999999},{"interface":"front","average":{"1min":8.4719999999999995,"5min":4.1699999999999999,"15min":4.9400000000000004},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":85.593999999999994,"5min":94.492000000000004,"15min":144.62799999999999},"last":0.52800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.74199999999999999,"15min":1.1859999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.8700000000000001,"5min":4.9119999999999999,"15min":32.850999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.78900000000000003,"15min":1.044},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":2.8919999999999999,"5min":6.2610000000000001,"15min":32.82},"last":0.60399999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.387,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.9460000000000002,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.028,"5min":1.5169999999999999,"15min":1.7470000000000001},"min":{"1min":0.35099999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.173,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.54800000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.147,"15min":1.397},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1829999999999998,"5min":7.4290000000000003,"15min":30.658999999999999},"last":1.079},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.022,"15min":1.399},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":2.3540000000000001,"5min":6.2320000000000002,"15min":30.683},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:35:24.977 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:35:24.978 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:35:25.593 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:25.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:25.698 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:25.726 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:25.919 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:25.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:26.023 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:26.056 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:26.104 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:26.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:26.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:26.209 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:26.228 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:26.243 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:26.332 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:26.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:26.341 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:26.341 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:26.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:26.470 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:26.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:26 smithi107 ceph-mon[212750]: pgmap v1326: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:26.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:26 smithi107 ceph-mon[220921]: pgmap v1326: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:26.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:26 smithi150 ceph-mon[194178]: pgmap v1326: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:27.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:35:27.304 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:27.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:27.409 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:27.441 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:27.492 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:27.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:27.531 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_8' created 2023-10-09T06:35:27.600 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:27.628 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:27.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T06:35:27.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:27.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T06:35:27.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:27.671 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:27.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:27.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:27.778 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:27.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T06:35:27.803 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:27.805 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:27.847 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:27.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:27.952 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:27.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:35:27.981 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:28.472 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:28.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:28.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:28.576 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:28.585 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-10-09T06:35:28.603 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:28.646 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:28.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:28.738 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:28.754 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:28.781 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: pgmap v1327: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:28.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T06:35:28.782 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[212750]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: pgmap v1327: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T06:35:28.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:28 smithi107 ceph-mon[220921]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:35:28.785 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:28.842 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:28.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: pgmap v1327: 103 pgs: 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: failed to encode map e437 with expected crc 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1183201712' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T06:35:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:28 smithi150 ceph-mon[194178]: osdmap e437: 8 total, 8 up, 8 in 2023-10-09T06:35:28.926 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:28.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:28.981 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:29.033 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:29.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:29.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:29.137 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:29.166 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:29.210 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:29.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:29.316 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:29.327 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:29.342 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:29.384 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:29.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:29.491 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:29.519 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:29.545 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.570 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.571 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: osdmap e438: 8 total, 8 up, 8 in 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: failed to encode map e438 with expected crc 2023-10-09T06:35:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:30.085 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:30.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:30.189 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:30.216 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: pgmap v1330: 119 pgs: 8 creating+peering, 8 unknown, 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: pgmap v1330: 119 pgs: 8 creating+peering, 8 unknown, 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:30 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: pgmap v1330: 119 pgs: 8 creating+peering, 8 unknown, 103 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2385560726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: osdmap e439: 8 total, 8 up, 8 in 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:30.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:30 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.310 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:31.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:31.414 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:31.446 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:31.494 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:31.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:31.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:31.600 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:31.628 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:31.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[212750]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[212750]: 1.18 scrub ok 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[220921]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:35:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:31 smithi107 ceph-mon[220921]: 1.18 scrub ok 2023-10-09T06:35:31.672 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:31.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:31.777 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:31.806 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:31 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:31 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:31 smithi150 ceph-mon[194178]: failed to encode map e439 with expected crc 2023-10-09T06:35:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:31 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:35:31.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:31 smithi150 ceph-mon[194178]: 1.18 scrub ok 2023-10-09T06:35:31.944 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:31.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:32.048 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:32.073 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:32] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.43.0" 2023-10-09T06:35:32.075 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:32.183 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:32.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:32.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:32.288 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:32.320 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:32.365 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:32.366 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:32.370 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:32.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:32.390 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:32.409 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:32.427 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:32.446 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:32.466 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:32.475 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:32.489 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:32.509 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:32.512 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:32.538 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:32.539 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.539 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.539 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.540 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.540 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.540 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.540 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.540 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:35:32.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: 1.17 scrub ok 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: pgmap v1332: 119 pgs: 8 creating+peering, 7 unknown, 104 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4174429091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: 1.17 scrub ok 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: pgmap v1332: 119 pgs: 8 creating+peering, 7 unknown, 104 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:32.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:32.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:32.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4174429091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: 1.17 scrub ok 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: pgmap v1332: 119 pgs: 8 creating+peering, 7 unknown, 104 active+clean; 454 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4174429091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:32.678 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:35:33.213 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:33.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:33.317 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:33.349 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:33.398 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:33.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:33.504 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:33.532 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:33.577 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:33.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:33.685 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:33.711 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:33.753 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:33.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:33.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:35:33.858 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:33.900 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:33.931 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:33.957 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:33.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:34.061 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:34.088 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:34.131 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:34.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:34.238 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:34.265 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:34.310 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:34.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:34.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:34.414 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:34.441 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:34.483 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:34.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:34.560 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-10-09T06:35:34.589 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:34.615 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:34.658 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:34.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:34.765 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:34.790 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:34 smithi150 ceph-mon[194178]: pgmap v1333: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:34.790 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2196486635' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.790 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:34 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.792 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:34.834 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:34.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:34.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[212750]: pgmap v1333: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2196486635' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[220921]: pgmap v1333: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2196486635' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:34 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:35:34.939 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:34.965 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:35.006 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:35.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:35.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:35.143 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:35.421 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:35.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:35.526 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:35.539 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:35.554 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:35.598 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:35.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:35.646 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:35:35.675 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[212750]: failed to encode map e440 with expected crc 2023-10-09T06:35:35.675 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:35:35.676 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[212750]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T06:35:35.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[220921]: failed to encode map e440 with expected crc 2023-10-09T06:35:35.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:35:35.676 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:35 smithi107 ceph-mon[220921]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T06:35:35.703 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.730 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:35.772 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:35.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:35.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:35 smithi150 ceph-mon[194178]: failed to encode map e440 with expected crc 2023-10-09T06:35:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:35:35.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:35 smithi150 ceph-mon[194178]: osdmap e440: 8 total, 8 up, 8 in 2023-10-09T06:35:35.905 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:35.947 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:35.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:36.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:36.081 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:36.123 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:36.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:36.228 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.258 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:36.301 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:36.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:36.405 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.431 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:36.485 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-10-09T06:35:36.486 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-10-09T06:35:36.486 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:35:36.488 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:35:36.488 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:35:36.558 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:36 smithi107 ceph-mon[212750]: pgmap v1335: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:36.558 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:35:36.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:35:36.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_535588 2023-10-09T06:35:36.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:35:36.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:35:36.575 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:35:36.684 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:36.838 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:36.839 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:36 smithi107 ceph-mon[220921]: pgmap v1335: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:36.839 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:36.861 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:36.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:36 smithi150 ceph-mon[194178]: pgmap v1335: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:35:36.967 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:36.994 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:37.405 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:37.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:37.510 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:37.554 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:37.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2863 2847 177.928 177.938 0.00622612 0.00556965 2023-10-09T06:35:37.658 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:37.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:37.672 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:37.672 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.672 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:37.672 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[212750]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:37.673 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.674 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:37 smithi107 ceph-mon[220921]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.768 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:37.831 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1417550609' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: osdmap e441: 8 total, 8 up, 8 in 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:37 smithi150 ceph-mon[194178]: failed to encode map e441 with expected crc 2023-10-09T06:35:37.959 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:38.218 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:35:38.218 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:35:38.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5013 13.9 e441/437 13:969ee87f:::benchmark_data_smithi107_535588_object5012:head v 441'326, mlcod=441'323)", 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.215603+0000", 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0018255299999999999, 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018479049999999999, 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.219 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215603+0000", 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215603+0000", 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215603+0000", 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215665+0000", 2023-10-09T06:35:38.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2639999999999997e-05 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215674+0000", 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9609999999999994e-06 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215681+0000", 2023-10-09T06:35:38.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0330000000000003e-06 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215733+0000", 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1490999999999999e-05 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.215752+0000", 2023-10-09T06:35:38.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9638000000000001e-05 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5004 13.b e441/437 13:dcec4f7b:::benchmark_data_smithi107_535588_object5003:head v 441'294, mlcod=441'290)", 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.216065+0000", 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.001363433, 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014666460000000001, 2023-10-09T06:35:38.224 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216065+0000", 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216065+0000", 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216065+0000", 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216096+0000", 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0759000000000001e-05 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216108+0000", 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2679e-05 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216115+0000", 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9770000000000003e-06 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216151+0000", 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6083e-05 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216171+0000", 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9306000000000001e-05 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5011 13.a e441/437 13:5cdbbe7e:::benchmark_data_smithi107_535588_object5010:head v 441'324, mlcod=441'321)", 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.216685+0000", 2023-10-09T06:35:38.229 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.00074432399999999998, 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00089860600000000004, 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216685+0000", 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216685+0000", 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216684+0000", 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216708+0000", 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4496999999999999e-05 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216718+0000", 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9729999999999992e-06 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216725+0000", 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7479999999999996e-06 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216763+0000", 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7886999999999998e-05 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216796+0000", 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3494999999999997e-05 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.234 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5014 13.9 e441/437 13:91d350b1:::benchmark_data_smithi107_535588_object5013:head v 441'327, mlcod=441'323)", 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.216826+0000", 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.00060275699999999999, 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00081181899999999997, 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216826+0000", 2023-10-09T06:35:38.235 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216826+0000", 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216826+0000", 2023-10-09T06:35:38.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216892+0000", 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6940000000000006e-05 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216901+0000", 2023-10-09T06:35:38.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1799999999999996e-06 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216907+0000", 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9100000000000002e-06 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216963+0000", 2023-10-09T06:35:38.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6725999999999999e-05 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.216983+0000", 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9979999999999998e-05 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.239 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5015 13.f e441/437 13:fbe0a6d9:::benchmark_data_smithi107_535588_object5014:head v 441'318, mlcod=441'316)", 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.217479+0000", 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 4294967295.9999495, 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00022646999999999999, 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.240 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217479+0000", 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217479+0000", 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.241 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217479+0000", 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217558+0000", 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9975999999999995e-05 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.242 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217572+0000", 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3242e-05 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217579+0000", 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9999999999999999e-06 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.243 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217628+0000", 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9177e-05 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217679+0000", 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0862000000000002e-05 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.244 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:5016 13.1 e441/437 13:898defd3:::benchmark_data_smithi107_535588_object5015:head v 441'328, mlcod=441'326)", 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:38.217593+0000", 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 4294967295.9998364, 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017626499999999999, 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.245 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217593+0000", 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217593+0000", 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.246 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217592+0000", 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217622+0000", 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9369999999999998e-05 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.247 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217628+0000", 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6640000000000001e-06 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217634+0000", 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4620000000000004e-06 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.248 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217681+0000", 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7485999999999999e-05 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:38.217722+0000", 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0636000000000002e-05 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:38.249 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.250 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:38.250 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:35:38.250 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 6 2023-10-09T06:35:38.250 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:35:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:38.251 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:38.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:38.352 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:38.400 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:38.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 6190 6174 192.923 207.938 0.00414578 0.00517602 2023-10-09T06:35:38.573 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:38 smithi107 ceph-mon[212750]: pgmap v1337: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:38.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:38 smithi107 ceph-mon[220921]: pgmap v1337: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:38.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:38 smithi150 ceph-mon[194178]: pgmap v1337: 119 pgs: 119 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2023-10-09T06:35:39.559 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 9246 9230 192.277 191 0.00359598 0.00519337 2023-10-09T06:35:39.585 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:35:40.179 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:40.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:40.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:40.328 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:40.560 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 11948 11932 186.412 168.875 0.00780427 0.00535864 2023-10-09T06:35:40.685 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:40.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:40.790 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:40.841 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:40 smithi107 ceph-mon[212750]: pgmap v1338: 119 pgs: 119 active+clean; 168 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s wr, 334 op/s 2023-10-09T06:35:40.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:35:40.842 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:40 smithi107 ceph-mon[220921]: pgmap v1338: 119 pgs: 119 active+clean; 168 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s wr, 334 op/s 2023-10-09T06:35:40.882 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:40.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:40 smithi150 ceph-mon[194178]: pgmap v1338: 119 pgs: 119 active+clean; 168 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s wr, 334 op/s 2023-10-09T06:35:40.985 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:40.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:41.095 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:41.188 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:41.271 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:41.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:41.377 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:41.447 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:41.560 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 14230 14214 177.636 142.625 0.00753015 0.0056206 2023-10-09T06:35:41.594 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2275915208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:41.611 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:41.643 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:41.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:41.650 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:41.688 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:41.750 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:41.777 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:41.817 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:41.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:41.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2275915208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:41.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2275915208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:41.924 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:41.950 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:42.007 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:42.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:42.112 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:42.138 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.158 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.159 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:35:42.244 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:42.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:42.352 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:42.397 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:42] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.43.0" 2023-10-09T06:35:42.403 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:42.560 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 15846 15830 164.863 101 0.00328921 0.00605838 2023-10-09T06:35:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:42 smithi107 ceph-mon[212750]: pgmap v1339: 119 pgs: 119 active+clean; 220 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 27 MiB/s wr, 437 op/s 2023-10-09T06:35:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:42 smithi107 ceph-mon[220921]: pgmap v1339: 119 pgs: 119 active+clean; 220 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 27 MiB/s wr, 437 op/s 2023-10-09T06:35:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:42.892 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:42.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:42.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:42.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:42 smithi150 ceph-mon[194178]: pgmap v1339: 119 pgs: 119 active+clean; 220 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 27 MiB/s wr, 437 op/s 2023-10-09T06:35:42.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:43.002 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:43.053 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:43.317 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:43.421 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:43.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:43.445 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:43.542 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:43.561 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 17953 17937 160.109 131.688 0.00983082 0.00624149 2023-10-09T06:35:43.616 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:43.706 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:43.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:43.817 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:43.878 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:44.237 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:44.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:44.343 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:44.415 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:44.505 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:35:44.512 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:35:44.561 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 20167 20151 157.392 138.375 0.00450835 0.00634826 2023-10-09T06:35:44.597 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:44 smithi107 ceph-mon[220921]: pgmap v1340: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T06:35:44.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:44 smithi107 ceph-mon[212750]: pgmap v1340: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T06:35:44.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:44 smithi150 ceph-mon[194178]: pgmap v1340: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-10-09T06:35:45.403 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:35:45.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 21852 21836 151.597 105.312 0.00553207 0.00659041 2023-10-09T06:35:45.570 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:45.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:45.677 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:45.705 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:45 smithi107 ceph-mon[220921]: from='client.46718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:45.708 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:45 smithi107 ceph-mon[212750]: from='client.46718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:45.737 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:45.794 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:45.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:45.902 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:45.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:45 smithi150 ceph-mon[194178]: from='client.46718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:35:45.945 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:45.994 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:46.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:46.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:46.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:46.152 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:46.257 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:46.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:46.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:46.367 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:46.416 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:46.497 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1340,"stamp":"2023-10-09T06:35:43.434051+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1021253536,"num_objects":15598,"num_object_clones":0,"num_object_copies":46794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15635,"num_write_kb":996765,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20620,"ondisk_log_size":20620,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10291924,"kb_used_data":3228308,"kb_used_omap":20,"kb_used_meta":7063531,"kb_avail":739669292,"statfs":{"total":767960285184,"available":757421355008,"internally_reserved":0,"allocated":3305787392,"data_stored":3293701462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7233056398},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,6,11,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1020657664,"num_objects":15574,"num_object_clones":0,"num_object_copies":46722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15574,"num_write_kb":996736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.881470"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":815,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708022+0000","last_change":"2023-10-09T06:35:30.160523+0000","last_active":"2023-10-09T06:35:34.708022+0000","last_peered":"2023-10-09T06:35:34.708022+0000","last_clean":"2023-10-09T06:35:34.708022+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:35:34.708022+0000","last_undegraded":"2023-10-09T06:35:34.708022+0000","last_fullsized":"2023-10-09T06:35:34.708022+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:30.160439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:30.160439+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:36.130795+0000","scrub_duration":0.002097706,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":420,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555627+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:35:34.555627+0000","last_peered":"2023-10-09T06:35:34.555627+0000","last_clean":"2023-10-09T06:35:34.555627+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:35:34.555627+0000","last_undegraded":"2023-10-09T06:35:34.555627+0000","last_fullsized":"2023-10-09T06:35:34.555627+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708395+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:35:34.708395+0000","last_peered":"2023-10-09T06:35:34.708395+0000","last_clean":"2023-10-09T06:35:34.708395+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:35:34.708395+0000","last_undegraded":"2023-10-09T06:35:34.708395+0000","last_fullsized":"2023-10-09T06:35:34.708395+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":772,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709210+0000","last_change":"2023-10-09T06:35:03.005446+0000","last_active":"2023-10-09T06:35:34.709210+0000","last_peered":"2023-10-09T06:35:34.709210+0000","last_clean":"2023-10-09T06:35:34.709210+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:35:34.709210+0000","last_undegraded":"2023-10-09T06:35:34.709210+0000","last_fullsized":"2023-10-09T06:35:34.709210+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:06.799059+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":435,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707761+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:35:34.707761+0000","last_peered":"2023-10-09T06:35:34.707761+0000","last_clean":"2023-10-09T06:35:34.707761+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:35:34.707761+0000","last_undegraded":"2023-10-09T06:35:34.707761+0000","last_fullsized":"2023-10-09T06:35:34.707761+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":439,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.553001+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:35:34.553001+0000","last_peered":"2023-10-09T06:35:34.553001+0000","last_clean":"2023-10-09T06:35:34.553001+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:35:34.553001+0000","last_undegraded":"2023-10-09T06:35:34.553001+0000","last_fullsized":"2023-10-09T06:35:34.553001+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":688,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555615+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:35:34.555615+0000","last_peered":"2023-10-09T06:35:34.555615+0000","last_clean":"2023-10-09T06:35:34.555615+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:35:34.555615+0000","last_undegraded":"2023-10-09T06:35:34.555615+0000","last_fullsized":"2023-10-09T06:35:34.555615+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":159,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707701+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:35:34.707701+0000","last_peered":"2023-10-09T06:35:34.707701+0000","last_clean":"2023-10-09T06:35:34.707701+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:35:34.707701+0000","last_undegraded":"2023-10-09T06:35:34.707701+0000","last_fullsized":"2023-10-09T06:35:34.707701+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":430,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709464+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:35:34.709464+0000","last_peered":"2023-10-09T06:35:34.709464+0000","last_clean":"2023-10-09T06:35:34.709464+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:35:34.709464+0000","last_undegraded":"2023-10-09T06:35:34.709464+0000","last_fullsized":"2023-10-09T06:35:34.709464+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":549,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708728+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:35:34.708728+0000","last_peered":"2023-10-09T06:35:34.708728+0000","last_clean":"2023-10-09T06:35:34.708728+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:35:34.708728+0000","last_undegraded":"2023-10-09T06:35:34.708728+0000","last_fullsized":"2023-10-09T06:35:34.708728+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":114,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.553071+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:35:34.553071+0000","last_peered":"2023-10-09T06:35:34.553071+0000","last_clean":"2023-10-09T06:35:34.553071+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:35:34.553071+0000","last_undegraded":"2023-10-09T06:35:34.553071+0000","last_fullsized":"2023-10-09T06:35:34.553071+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708567+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:35:34.708567+0000","last_peered":"2023-10-09T06:35:34.708567+0000","last_clean":"2023-10-09T06:35:34.708567+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:35:34.708567+0000","last_undegraded":"2023-10-09T06:35:34.708567+0000","last_fullsized":"2023-10-09T06:35:34.708567+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":586,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708613+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:35:34.708613+0000","last_peered":"2023-10-09T06:35:34.708613+0000","last_clean":"2023-10-09T06:35:34.708613+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:35:34.708613+0000","last_undegraded":"2023-10-09T06:35:34.708613+0000","last_fullsized":"2023-10-09T06:35:34.708613+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709016+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:35:34.709016+0000","last_peered":"2023-10-09T06:35:34.709016+0000","last_clean":"2023-10-09T06:35:34.709016+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:35:34.709016+0000","last_undegraded":"2023-10-09T06:35:34.709016+0000","last_fullsized":"2023-10-09T06:35:34.709016+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708696+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:35:34.708696+0000","last_peered":"2023-10-09T06:35:34.708696+0000","last_clean":"2023-10-09T06:35:34.708696+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:35:34.708696+0000","last_undegraded":"2023-10-09T06:35:34.708696+0000","last_fullsized":"2023-10-09T06:35:34.708696+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":649,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709986+0000","last_change":"2023-10-09T06:35:31.210236+0000","last_active":"2023-10-09T06:35:34.709986+0000","last_peered":"2023-10-09T06:35:34.709986+0000","last_clean":"2023-10-09T06:35:34.709986+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:35:34.709986+0000","last_undegraded":"2023-10-09T06:35:34.709986+0000","last_fullsized":"2023-10-09T06:35:34.709986+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:31.210137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:31.210137+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:04.153669+0000","scrub_duration":0.0017668409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":322,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709216+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:35:34.709216+0000","last_peered":"2023-10-09T06:35:34.709216+0000","last_clean":"2023-10-09T06:35:34.709216+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:35:34.709216+0000","last_undegraded":"2023-10-09T06:35:34.709216+0000","last_fullsized":"2023-10-09T06:35:34.709216+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":376,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708028+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:35:34.708028+0000","last_peered":"2023-10-09T06:35:34.708028+0000","last_clean":"2023-10-09T06:35:34.708028+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:35:34.708028+0000","last_undegraded":"2023-10-09T06:35:34.708028+0000","last_fullsized":"2023-10-09T06:35:34.708028+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":548,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708041+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:35:34.708041+0000","last_peered":"2023-10-09T06:35:34.708041+0000","last_clean":"2023-10-09T06:35:34.708041+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:35:34.708041+0000","last_undegraded":"2023-10-09T06:35:34.708041+0000","last_fullsized":"2023-10-09T06:35:34.708041+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":447,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555817+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:35:34.555817+0000","last_peered":"2023-10-09T06:35:34.555817+0000","last_clean":"2023-10-09T06:35:34.555817+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:35:34.555817+0000","last_undegraded":"2023-10-09T06:35:34.555817+0000","last_fullsized":"2023-10-09T06:35:34.555817+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":411,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.556144+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:35:34.556144+0000","last_peered":"2023-10-09T06:35:34.556144+0000","last_clean":"2023-10-09T06:35:34.556144+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:35:34.556144+0000","last_undegraded":"2023-10-09T06:35:34.556144+0000","last_fullsized":"2023-10-09T06:35:34.556144+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":712,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552644+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:35:34.552644+0000","last_peered":"2023-10-09T06:35:34.552644+0000","last_clean":"2023-10-09T06:35:34.552644+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:35:34.552644+0000","last_undegraded":"2023-10-09T06:35:34.552644+0000","last_fullsized":"2023-10-09T06:35:34.552644+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":290,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708902+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:35:34.708902+0000","last_peered":"2023-10-09T06:35:34.708902+0000","last_clean":"2023-10-09T06:35:34.708902+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:35:34.708902+0000","last_undegraded":"2023-10-09T06:35:34.708902+0000","last_fullsized":"2023-10-09T06:35:34.708902+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708815+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:35:34.708815+0000","last_peered":"2023-10-09T06:35:34.708815+0000","last_clean":"2023-10-09T06:35:34.708815+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:35:34.708815+0000","last_undegraded":"2023-10-09T06:35:34.708815+0000","last_fullsized":"2023-10-09T06:35:34.708815+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":555,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708111+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:35:34.708111+0000","last_peered":"2023-10-09T06:35:34.708111+0000","last_clean":"2023-10-09T06:35:34.708111+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:35:34.708111+0000","last_undegraded":"2023-10-09T06:35:34.708111+0000","last_fullsized":"2023-10-09T06:35:34.708111+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":437,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552713+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:35:34.552713+0000","last_peered":"2023-10-09T06:35:34.552713+0000","last_clean":"2023-10-09T06:35:34.552713+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:35:34.552713+0000","last_undegraded":"2023-10-09T06:35:34.552713+0000","last_fullsized":"2023-10-09T06:35:34.552713+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708801+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:35:34.708801+0000","last_peered":"2023-10-09T06:35:34.708801+0000","last_clean":"2023-10-09T06:35:34.708801+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:35:34.708801+0000","last_undegraded":"2023-10-09T06:35:34.708801+0000","last_fullsized":"2023-10-09T06:35:34.708801+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":766,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707819+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:35:34.707819+0000","last_peered":"2023-10-09T06:35:34.707819+0000","last_clean":"2023-10-09T06:35:34.707819+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:35:34.707819+0000","last_undegraded":"2023-10-09T06:35:34.707819+0000","last_fullsized":"2023-10-09T06:35:34.707819+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708290+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:35:34.708290+0000","last_peered":"2023-10-09T06:35:34.708290+0000","last_clean":"2023-10-09T06:35:34.708290+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:35:34.708290+0000","last_undegraded":"2023-10-09T06:35:34.708290+0000","last_fullsized":"2023-10-09T06:35:34.708290+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":114,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552883+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:35:34.552883+0000","last_peered":"2023-10-09T06:35:34.552883+0000","last_clean":"2023-10-09T06:35:34.552883+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:35:34.552883+0000","last_undegraded":"2023-10-09T06:35:34.552883+0000","last_fullsized":"2023-10-09T06:35:34.552883+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":555,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709856+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:35:34.709856+0000","last_peered":"2023-10-09T06:35:34.709856+0000","last_clean":"2023-10-09T06:35:34.709856+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:35:34.709856+0000","last_undegraded":"2023-10-09T06:35:34.709856+0000","last_fullsized":"2023-10-09T06:35:34.709856+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":519,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707810+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:35:34.707810+0000","last_peered":"2023-10-09T06:35:34.707810+0000","last_clean":"2023-10-09T06:35:34.707810+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:35:34.707810+0000","last_undegraded":"2023-10-09T06:35:34.707810+0000","last_fullsized":"2023-10-09T06:35:34.707810+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708470+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:35:34.708470+0000","last_peered":"2023-10-09T06:35:34.708470+0000","last_clean":"2023-10-09T06:35:34.708470+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:35:34.708470+0000","last_undegraded":"2023-10-09T06:35:34.708470+0000","last_fullsized":"2023-10-09T06:35:34.708470+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"441'973","reported_seq":985,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.551738+0000","last_change":"2023-10-09T06:35:28.749051+0000","last_active":"2023-10-09T06:35:42.551738+0000","last_peered":"2023-10-09T06:35:42.551738+0000","last_clean":"2023-10-09T06:35:42.551738+0000","last_became_active":"2023-10-09T06:35:28.748772+0000","last_became_peered":"2023-10-09T06:35:28.748772+0000","last_unstale":"2023-10-09T06:35:42.551738+0000","last_undegraded":"2023-10-09T06:35:42.551738+0000","last_fullsized":"2023-10-09T06:35:42.551738+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":973,"log_dups_size":0,"ondisk_log_size":973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:58.923271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":973,"num_write_kb":62272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":568,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708813+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:35:34.708813+0000","last_peered":"2023-10-09T06:35:34.708813+0000","last_clean":"2023-10-09T06:35:34.708813+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:35:34.708813+0000","last_undegraded":"2023-10-09T06:35:34.708813+0000","last_fullsized":"2023-10-09T06:35:34.708813+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":326,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709049+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:35:34.709049+0000","last_peered":"2023-10-09T06:35:34.709049+0000","last_clean":"2023-10-09T06:35:34.709049+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:35:34.709049+0000","last_undegraded":"2023-10-09T06:35:34.709049+0000","last_fullsized":"2023-10-09T06:35:34.709049+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708355+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:35:34.708355+0000","last_peered":"2023-10-09T06:35:34.708355+0000","last_clean":"2023-10-09T06:35:34.708355+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:35:34.708355+0000","last_undegraded":"2023-10-09T06:35:34.708355+0000","last_fullsized":"2023-10-09T06:35:34.708355+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"441'974","reported_seq":986,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.542721+0000","last_change":"2023-10-09T06:35:28.749790+0000","last_active":"2023-10-09T06:35:42.542721+0000","last_peered":"2023-10-09T06:35:42.542721+0000","last_clean":"2023-10-09T06:35:42.542721+0000","last_became_active":"2023-10-09T06:35:28.748466+0000","last_became_peered":"2023-10-09T06:35:28.748466+0000","last_unstale":"2023-10-09T06:35:42.542721+0000","last_undegraded":"2023-10-09T06:35:42.542721+0000","last_fullsized":"2023-10-09T06:35:42.542721+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":974,"log_dups_size":0,"ondisk_log_size":974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:34.659448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63832064,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":974,"num_write_kb":62336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":825,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708912+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:35:34.708912+0000","last_peered":"2023-10-09T06:35:34.708912+0000","last_clean":"2023-10-09T06:35:34.708912+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:35:34.708912+0000","last_undegraded":"2023-10-09T06:35:34.708912+0000","last_fullsized":"2023-10-09T06:35:34.708912+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":288,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709437+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:35:34.709437+0000","last_peered":"2023-10-09T06:35:34.709437+0000","last_clean":"2023-10-09T06:35:34.709437+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:35:34.709437+0000","last_undegraded":"2023-10-09T06:35:34.709437+0000","last_fullsized":"2023-10-09T06:35:34.709437+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":438,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555608+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:35:34.555608+0000","last_peered":"2023-10-09T06:35:34.555608+0000","last_clean":"2023-10-09T06:35:34.555608+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:35:34.555608+0000","last_undegraded":"2023-10-09T06:35:34.555608+0000","last_fullsized":"2023-10-09T06:35:34.555608+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"441'828","reported_seq":841,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:41.023096+0000","last_change":"2023-10-09T06:35:28.532035+0000","last_active":"2023-10-09T06:35:41.023096+0000","last_peered":"2023-10-09T06:35:41.023096+0000","last_clean":"2023-10-09T06:35:41.023096+0000","last_became_active":"2023-10-09T06:35:28.531723+0000","last_became_peered":"2023-10-09T06:35:28.531723+0000","last_unstale":"2023-10-09T06:35:41.023096+0000","last_undegraded":"2023-10-09T06:35:41.023096+0000","last_fullsized":"2023-10-09T06:35:41.023096+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":828,"log_dups_size":0,"ondisk_log_size":828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:19.173492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54263808,"num_objects":828,"num_object_clones":0,"num_object_copies":2484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":828,"num_write_kb":52992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":742,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709327+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:35:34.709327+0000","last_peered":"2023-10-09T06:35:34.709327+0000","last_clean":"2023-10-09T06:35:34.709327+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:35:34.709327+0000","last_undegraded":"2023-10-09T06:35:34.709327+0000","last_fullsized":"2023-10-09T06:35:34.709327+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":344,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708945+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:35:34.708945+0000","last_peered":"2023-10-09T06:35:34.708945+0000","last_clean":"2023-10-09T06:35:34.708945+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:35:34.708945+0000","last_undegraded":"2023-10-09T06:35:34.708945+0000","last_fullsized":"2023-10-09T06:35:34.708945+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":417,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708117+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:35:34.708117+0000","last_peered":"2023-10-09T06:35:34.708117+0000","last_clean":"2023-10-09T06:35:34.708117+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:35:34.708117+0000","last_undegraded":"2023-10-09T06:35:34.708117+0000","last_fullsized":"2023-10-09T06:35:34.708117+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"441'913","reported_seq":925,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.448415+0000","last_change":"2023-10-09T06:35:28.543526+0000","last_active":"2023-10-09T06:35:42.448415+0000","last_peered":"2023-10-09T06:35:42.448415+0000","last_clean":"2023-10-09T06:35:42.448415+0000","last_became_active":"2023-10-09T06:35:28.543240+0000","last_became_peered":"2023-10-09T06:35:28.543240+0000","last_unstale":"2023-10-09T06:35:42.448415+0000","last_undegraded":"2023-10-09T06:35:42.448415+0000","last_fullsized":"2023-10-09T06:35:42.448415+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":913,"log_dups_size":0,"ondisk_log_size":913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:16.584504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":913,"num_write_kb":58432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":565,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555964+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:35:34.555964+0000","last_peered":"2023-10-09T06:35:34.555964+0000","last_clean":"2023-10-09T06:35:34.555964+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:35:34.555964+0000","last_undegraded":"2023-10-09T06:35:34.555964+0000","last_fullsized":"2023-10-09T06:35:34.555964+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.710350+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:35:34.710350+0000","last_peered":"2023-10-09T06:35:34.710350+0000","last_clean":"2023-10-09T06:35:34.710350+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:35:34.710350+0000","last_undegraded":"2023-10-09T06:35:34.710350+0000","last_fullsized":"2023-10-09T06:35:34.710350+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":402,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552768+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:35:34.552768+0000","last_peered":"2023-10-09T06:35:34.552768+0000","last_clean":"2023-10-09T06:35:34.552768+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:35:34.552768+0000","last_undegraded":"2023-10-09T06:35:34.552768+0000","last_fullsized":"2023-10-09T06:35:34.552768+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"441'972","reported_seq":984,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.455343+0000","last_change":"2023-10-09T06:35:28.543354+0000","last_active":"2023-10-09T06:35:42.455343+0000","last_peered":"2023-10-09T06:35:42.455343+0000","last_clean":"2023-10-09T06:35:42.455343+0000","last_became_active":"2023-10-09T06:35:28.543001+0000","last_became_peered":"2023-10-09T06:35:28.543001+0000","last_unstale":"2023-10-09T06:35:42.455343+0000","last_undegraded":"2023-10-09T06:35:42.455343+0000","last_fullsized":"2023-10-09T06:35:42.455343+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":972,"log_dups_size":0,"ondisk_log_size":972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:38.235566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":972,"num_write_kb":62208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":793,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708147+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:35:34.708147+0000","last_peered":"2023-10-09T06:35:34.708147+0000","last_clean":"2023-10-09T06:35:34.708147+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:35:34.708147+0000","last_undegraded":"2023-10-09T06:35:34.708147+0000","last_fullsized":"2023-10-09T06:35:34.708147+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":167,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708085+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:35:34.708085+0000","last_peered":"2023-10-09T06:35:34.708085+0000","last_clean":"2023-10-09T06:35:34.708085+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:35:34.708085+0000","last_undegraded":"2023-10-09T06:35:34.708085+0000","last_fullsized":"2023-10-09T06:35:34.708085+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":421,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555728+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:35:34.555728+0000","last_peered":"2023-10-09T06:35:34.555728+0000","last_clean":"2023-10-09T06:35:34.555728+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:35:34.555728+0000","last_undegraded":"2023-10-09T06:35:34.555728+0000","last_fullsized":"2023-10-09T06:35:34.555728+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"441'977","reported_seq":988,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.556214+0000","last_change":"2023-10-09T06:35:28.750085+0000","last_active":"2023-10-09T06:35:42.556214+0000","last_peered":"2023-10-09T06:35:42.556214+0000","last_clean":"2023-10-09T06:35:42.556214+0000","last_became_active":"2023-10-09T06:35:28.749727+0000","last_became_peered":"2023-10-09T06:35:28.749727+0000","last_unstale":"2023-10-09T06:35:42.556214+0000","last_undegraded":"2023-10-09T06:35:42.556214+0000","last_fullsized":"2023-10-09T06:35:42.556214+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":977,"log_dups_size":0,"ondisk_log_size":977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:00.559675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64028672,"num_objects":977,"num_object_clones":0,"num_object_copies":2931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":977,"num_write_kb":62528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":773,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708193+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:35:34.708193+0000","last_peered":"2023-10-09T06:35:34.708193+0000","last_clean":"2023-10-09T06:35:34.708193+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:35:34.708193+0000","last_undegraded":"2023-10-09T06:35:34.708193+0000","last_fullsized":"2023-10-09T06:35:34.708193+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":398,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708009+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:35:34.708009+0000","last_peered":"2023-10-09T06:35:34.708009+0000","last_clean":"2023-10-09T06:35:34.708009+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:35:34.708009+0000","last_undegraded":"2023-10-09T06:35:34.708009+0000","last_fullsized":"2023-10-09T06:35:34.708009+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":418,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.556123+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:35:34.556123+0000","last_peered":"2023-10-09T06:35:34.556123+0000","last_clean":"2023-10-09T06:35:34.556123+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:35:34.556123+0000","last_undegraded":"2023-10-09T06:35:34.556123+0000","last_fullsized":"2023-10-09T06:35:34.556123+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"441'943","reported_seq":955,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.550370+0000","last_change":"2023-10-09T06:35:28.543670+0000","last_active":"2023-10-09T06:35:42.550370+0000","last_peered":"2023-10-09T06:35:42.550370+0000","last_clean":"2023-10-09T06:35:42.550370+0000","last_became_active":"2023-10-09T06:35:28.543089+0000","last_became_peered":"2023-10-09T06:35:28.543089+0000","last_unstale":"2023-10-09T06:35:42.550370+0000","last_undegraded":"2023-10-09T06:35:42.550370+0000","last_fullsized":"2023-10-09T06:35:42.550370+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":943,"log_dups_size":0,"ondisk_log_size":943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:02.123068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61800448,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":943,"num_write_kb":60352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":775,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707924+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:35:34.707924+0000","last_peered":"2023-10-09T06:35:34.707924+0000","last_clean":"2023-10-09T06:35:34.707924+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:35:34.707924+0000","last_undegraded":"2023-10-09T06:35:34.707924+0000","last_fullsized":"2023-10-09T06:35:34.707924+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":422,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555520+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:35:34.555520+0000","last_peered":"2023-10-09T06:35:34.555520+0000","last_clean":"2023-10-09T06:35:34.555520+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:35:34.555520+0000","last_undegraded":"2023-10-09T06:35:34.555520+0000","last_fullsized":"2023-10-09T06:35:34.555520+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":401,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.556052+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:35:34.556052+0000","last_peered":"2023-10-09T06:35:34.556052+0000","last_clean":"2023-10-09T06:35:34.556052+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:35:34.556052+0000","last_undegraded":"2023-10-09T06:35:34.556052+0000","last_fullsized":"2023-10-09T06:35:34.556052+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"441'987","reported_seq":999,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.455026+0000","last_change":"2023-10-09T06:35:28.540580+0000","last_active":"2023-10-09T06:35:42.455026+0000","last_peered":"2023-10-09T06:35:42.455026+0000","last_clean":"2023-10-09T06:35:42.455026+0000","last_became_active":"2023-10-09T06:35:28.540221+0000","last_became_peered":"2023-10-09T06:35:28.540221+0000","last_unstale":"2023-10-09T06:35:42.455026+0000","last_undegraded":"2023-10-09T06:35:42.455026+0000","last_fullsized":"2023-10-09T06:35:42.455026+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":987,"log_dups_size":0,"ondisk_log_size":987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:31.385902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":987,"num_write_kb":63168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":751,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709702+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:35:34.709702+0000","last_peered":"2023-10-09T06:35:34.709702+0000","last_clean":"2023-10-09T06:35:34.709702+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:35:34.709702+0000","last_undegraded":"2023-10-09T06:35:34.709702+0000","last_fullsized":"2023-10-09T06:35:34.709702+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":141,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708490+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:35:34.708490+0000","last_peered":"2023-10-09T06:35:34.708490+0000","last_clean":"2023-10-09T06:35:34.708490+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:35:34.708490+0000","last_undegraded":"2023-10-09T06:35:34.708490+0000","last_fullsized":"2023-10-09T06:35:34.708490+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":395,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708444+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:35:34.708444+0000","last_peered":"2023-10-09T06:35:34.708444+0000","last_clean":"2023-10-09T06:35:34.708444+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:35:34.708444+0000","last_undegraded":"2023-10-09T06:35:34.708444+0000","last_fullsized":"2023-10-09T06:35:34.708444+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"441'943","reported_seq":955,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.437031+0000","last_change":"2023-10-09T06:35:28.750966+0000","last_active":"2023-10-09T06:35:42.437031+0000","last_peered":"2023-10-09T06:35:42.437031+0000","last_clean":"2023-10-09T06:35:42.437031+0000","last_became_active":"2023-10-09T06:35:28.750668+0000","last_became_peered":"2023-10-09T06:35:28.750668+0000","last_unstale":"2023-10-09T06:35:42.437031+0000","last_undegraded":"2023-10-09T06:35:42.437031+0000","last_fullsized":"2023-10-09T06:35:42.437031+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":943,"log_dups_size":0,"ondisk_log_size":943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:47.331236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61800448,"num_objects":943,"num_object_clones":0,"num_object_copies":2829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":943,"num_write_kb":60352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":733,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709546+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:35:34.709546+0000","last_peered":"2023-10-09T06:35:34.709546+0000","last_clean":"2023-10-09T06:35:34.709546+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:35:34.709546+0000","last_undegraded":"2023-10-09T06:35:34.709546+0000","last_fullsized":"2023-10-09T06:35:34.709546+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":446,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708875+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:35:34.708875+0000","last_peered":"2023-10-09T06:35:34.708875+0000","last_clean":"2023-10-09T06:35:34.708875+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:35:34.708875+0000","last_undegraded":"2023-10-09T06:35:34.708875+0000","last_fullsized":"2023-10-09T06:35:34.708875+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708718+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:35:34.708718+0000","last_peered":"2023-10-09T06:35:34.708718+0000","last_clean":"2023-10-09T06:35:34.708718+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:35:34.708718+0000","last_undegraded":"2023-10-09T06:35:34.708718+0000","last_fullsized":"2023-10-09T06:35:34.708718+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":380,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707813+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:35:34.707813+0000","last_peered":"2023-10-09T06:35:34.707813+0000","last_clean":"2023-10-09T06:35:34.707813+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:35:34.707813+0000","last_undegraded":"2023-10-09T06:35:34.707813+0000","last_fullsized":"2023-10-09T06:35:34.707813+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"441'989","reported_seq":1001,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.452066+0000","last_change":"2023-10-09T06:35:28.539445+0000","last_active":"2023-10-09T06:35:42.452066+0000","last_peered":"2023-10-09T06:35:42.452066+0000","last_clean":"2023-10-09T06:35:42.452066+0000","last_became_active":"2023-10-09T06:35:28.539141+0000","last_became_peered":"2023-10-09T06:35:28.539141+0000","last_unstale":"2023-10-09T06:35:42.452066+0000","last_undegraded":"2023-10-09T06:35:42.452066+0000","last_fullsized":"2023-10-09T06:35:42.452066+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":989,"log_dups_size":0,"ondisk_log_size":989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:26.753486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64815104,"num_objects":989,"num_object_clones":0,"num_object_copies":2967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":989,"num_write_kb":63296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":787,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708069+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:35:34.708069+0000","last_peered":"2023-10-09T06:35:34.708069+0000","last_clean":"2023-10-09T06:35:34.708069+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:35:34.708069+0000","last_undegraded":"2023-10-09T06:35:34.708069+0000","last_fullsized":"2023-10-09T06:35:34.708069+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709564+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:35:34.709564+0000","last_peered":"2023-10-09T06:35:34.709564+0000","last_clean":"2023-10-09T06:35:34.709564+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:35:34.709564+0000","last_undegraded":"2023-10-09T06:35:34.709564+0000","last_fullsized":"2023-10-09T06:35:34.709564+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":422,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.556081+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:35:34.556081+0000","last_peered":"2023-10-09T06:35:34.556081+0000","last_clean":"2023-10-09T06:35:34.556081+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:35:34.556081+0000","last_undegraded":"2023-10-09T06:35:34.556081+0000","last_fullsized":"2023-10-09T06:35:34.556081+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":410,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708832+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:35:34.708832+0000","last_peered":"2023-10-09T06:35:34.708832+0000","last_clean":"2023-10-09T06:35:34.708832+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:35:34.708832+0000","last_undegraded":"2023-10-09T06:35:34.708832+0000","last_fullsized":"2023-10-09T06:35:34.708832+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"441'1019","reported_seq":1031,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.538694+0000","last_change":"2023-10-09T06:35:28.544012+0000","last_active":"2023-10-09T06:35:42.538694+0000","last_peered":"2023-10-09T06:35:42.538694+0000","last_clean":"2023-10-09T06:35:42.538694+0000","last_became_active":"2023-10-09T06:35:28.543684+0000","last_became_peered":"2023-10-09T06:35:28.543684+0000","last_unstale":"2023-10-09T06:35:42.538694+0000","last_undegraded":"2023-10-09T06:35:42.538694+0000","last_fullsized":"2023-10-09T06:35:42.538694+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":1019,"log_dups_size":0,"ondisk_log_size":1019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:47.327388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1019,"num_write_kb":65216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":783,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709204+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:35:34.709204+0000","last_peered":"2023-10-09T06:35:34.709204+0000","last_clean":"2023-10-09T06:35:34.709204+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:35:34.709204+0000","last_undegraded":"2023-10-09T06:35:34.709204+0000","last_fullsized":"2023-10-09T06:35:34.709204+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":432,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708481+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:35:34.708481+0000","last_peered":"2023-10-09T06:35:34.708481+0000","last_clean":"2023-10-09T06:35:34.708481+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:35:34.708481+0000","last_undegraded":"2023-10-09T06:35:34.708481+0000","last_fullsized":"2023-10-09T06:35:34.708481+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":419,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707949+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:35:34.707949+0000","last_peered":"2023-10-09T06:35:34.707949+0000","last_clean":"2023-10-09T06:35:34.707949+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:35:34.707949+0000","last_undegraded":"2023-10-09T06:35:34.707949+0000","last_fullsized":"2023-10-09T06:35:34.707949+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":441,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708480+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:35:34.708480+0000","last_peered":"2023-10-09T06:35:34.708480+0000","last_clean":"2023-10-09T06:35:34.708480+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:35:34.708480+0000","last_undegraded":"2023-10-09T06:35:34.708480+0000","last_fullsized":"2023-10-09T06:35:34.708480+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"441'1003","reported_seq":1016,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.793017+0000","last_change":"2023-10-09T06:35:28.750537+0000","last_active":"2023-10-09T06:35:42.793017+0000","last_peered":"2023-10-09T06:35:42.793017+0000","last_clean":"2023-10-09T06:35:42.793017+0000","last_became_active":"2023-10-09T06:35:28.750267+0000","last_became_peered":"2023-10-09T06:35:28.750267+0000","last_unstale":"2023-10-09T06:35:42.793017+0000","last_undegraded":"2023-10-09T06:35:42.793017+0000","last_fullsized":"2023-10-09T06:35:42.793017+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":1003,"log_dups_size":0,"ondisk_log_size":1003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:47.173545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1003,"num_write_kb":64192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":128,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709344+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:35:34.709344+0000","last_peered":"2023-10-09T06:35:34.709344+0000","last_clean":"2023-10-09T06:35:34.709344+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:35:34.709344+0000","last_undegraded":"2023-10-09T06:35:34.709344+0000","last_fullsized":"2023-10-09T06:35:34.709344+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":764,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709658+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:35:34.709658+0000","last_peered":"2023-10-09T06:35:34.709658+0000","last_clean":"2023-10-09T06:35:34.709658+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:35:34.709658+0000","last_undegraded":"2023-10-09T06:35:34.709658+0000","last_fullsized":"2023-10-09T06:35:34.709658+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708228+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:35:34.708228+0000","last_peered":"2023-10-09T06:35:34.708228+0000","last_clean":"2023-10-09T06:35:34.708228+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:35:34.708228+0000","last_undegraded":"2023-10-09T06:35:34.708228+0000","last_fullsized":"2023-10-09T06:35:34.708228+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":403,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708653+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:35:34.708653+0000","last_peered":"2023-10-09T06:35:34.708653+0000","last_clean":"2023-10-09T06:35:34.708653+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:35:34.708653+0000","last_undegraded":"2023-10-09T06:35:34.708653+0000","last_fullsized":"2023-10-09T06:35:34.708653+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"441'1056","reported_seq":1068,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.533261+0000","last_change":"2023-10-09T06:35:28.750108+0000","last_active":"2023-10-09T06:35:42.533261+0000","last_peered":"2023-10-09T06:35:42.533261+0000","last_clean":"2023-10-09T06:35:42.533261+0000","last_became_active":"2023-10-09T06:35:28.749808+0000","last_became_peered":"2023-10-09T06:35:28.749808+0000","last_unstale":"2023-10-09T06:35:42.533261+0000","last_undegraded":"2023-10-09T06:35:42.533261+0000","last_fullsized":"2023-10-09T06:35:42.533261+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":1056,"log_dups_size":0,"ondisk_log_size":1056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:44.197980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1056,"num_write_kb":67584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":749,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.707940+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:35:34.707940+0000","last_peered":"2023-10-09T06:35:34.707940+0000","last_clean":"2023-10-09T06:35:34.707940+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:35:34.707940+0000","last_undegraded":"2023-10-09T06:35:34.707940+0000","last_fullsized":"2023-10-09T06:35:34.707940+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":304,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709476+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:35:34.709476+0000","last_peered":"2023-10-09T06:35:34.709476+0000","last_clean":"2023-10-09T06:35:34.709476+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:35:34.709476+0000","last_undegraded":"2023-10-09T06:35:34.709476+0000","last_fullsized":"2023-10-09T06:35:34.709476+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552777+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:35:34.552777+0000","last_peered":"2023-10-09T06:35:34.552777+0000","last_clean":"2023-10-09T06:35:34.552777+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:35:34.552777+0000","last_undegraded":"2023-10-09T06:35:34.552777+0000","last_fullsized":"2023-10-09T06:35:34.552777+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":425,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.553042+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:35:34.553042+0000","last_peered":"2023-10-09T06:35:34.553042+0000","last_clean":"2023-10-09T06:35:34.553042+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:35:34.553042+0000","last_undegraded":"2023-10-09T06:35:34.553042+0000","last_fullsized":"2023-10-09T06:35:34.553042+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"441'1020","reported_seq":1033,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.798473+0000","last_change":"2023-10-09T06:35:28.748964+0000","last_active":"2023-10-09T06:35:42.798473+0000","last_peered":"2023-10-09T06:35:42.798473+0000","last_clean":"2023-10-09T06:35:42.798473+0000","last_became_active":"2023-10-09T06:35:28.748674+0000","last_became_peered":"2023-10-09T06:35:28.748674+0000","last_unstale":"2023-10-09T06:35:42.798473+0000","last_undegraded":"2023-10-09T06:35:42.798473+0000","last_fullsized":"2023-10-09T06:35:42.798473+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":1020,"log_dups_size":0,"ondisk_log_size":1020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:02.079067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66846720,"num_objects":1020,"num_object_clones":0,"num_object_copies":3060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1020,"num_write_kb":65280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":763,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708904+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:35:34.708904+0000","last_peered":"2023-10-09T06:35:34.708904+0000","last_clean":"2023-10-09T06:35:34.708904+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:35:34.708904+0000","last_undegraded":"2023-10-09T06:35:34.708904+0000","last_fullsized":"2023-10-09T06:35:34.708904+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":304,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709137+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:35:34.709137+0000","last_peered":"2023-10-09T06:35:34.709137+0000","last_clean":"2023-10-09T06:35:34.709137+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:35:34.709137+0000","last_undegraded":"2023-10-09T06:35:34.709137+0000","last_fullsized":"2023-10-09T06:35:34.709137+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":436,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708806+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:35:34.708806+0000","last_peered":"2023-10-09T06:35:34.708806+0000","last_clean":"2023-10-09T06:35:34.708806+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:35:34.708806+0000","last_undegraded":"2023-10-09T06:35:34.708806+0000","last_fullsized":"2023-10-09T06:35:34.708806+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":424,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552816+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:35:34.552816+0000","last_peered":"2023-10-09T06:35:34.552816+0000","last_clean":"2023-10-09T06:35:34.552816+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:35:34.552816+0000","last_undegraded":"2023-10-09T06:35:34.552816+0000","last_fullsized":"2023-10-09T06:35:34.552816+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"441'1002","reported_seq":1014,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.447241+0000","last_change":"2023-10-09T06:35:28.750679+0000","last_active":"2023-10-09T06:35:42.447241+0000","last_peered":"2023-10-09T06:35:42.447241+0000","last_clean":"2023-10-09T06:35:42.447241+0000","last_became_active":"2023-10-09T06:35:28.750367+0000","last_became_peered":"2023-10-09T06:35:28.750367+0000","last_unstale":"2023-10-09T06:35:42.447241+0000","last_undegraded":"2023-10-09T06:35:42.447241+0000","last_fullsized":"2023-10-09T06:35:42.447241+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":1002,"log_dups_size":0,"ondisk_log_size":1002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.804665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1002,"num_write_kb":64128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":764,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555983+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:35:34.555983+0000","last_peered":"2023-10-09T06:35:34.555983+0000","last_clean":"2023-10-09T06:35:34.555983+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:35:34.555983+0000","last_undegraded":"2023-10-09T06:35:34.555983+0000","last_fullsized":"2023-10-09T06:35:34.555983+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":171,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708590+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:35:34.708590+0000","last_peered":"2023-10-09T06:35:34.708590+0000","last_clean":"2023-10-09T06:35:34.708590+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:35:34.708590+0000","last_undegraded":"2023-10-09T06:35:34.708590+0000","last_fullsized":"2023-10-09T06:35:34.708590+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":404,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708170+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:35:34.708170+0000","last_peered":"2023-10-09T06:35:34.708170+0000","last_clean":"2023-10-09T06:35:34.708170+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:35:34.708170+0000","last_undegraded":"2023-10-09T06:35:34.708170+0000","last_fullsized":"2023-10-09T06:35:34.708170+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":160,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:36.575450+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:35:36.575450+0000","last_peered":"2023-10-09T06:35:36.575450+0000","last_clean":"2023-10-09T06:35:36.575450+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:35:36.575450+0000","last_undegraded":"2023-10-09T06:35:36.575450+0000","last_fullsized":"2023-10-09T06:35:36.575450+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"441'975","reported_seq":987,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:42.454828+0000","last_change":"2023-10-09T06:35:28.748822+0000","last_active":"2023-10-09T06:35:42.454828+0000","last_peered":"2023-10-09T06:35:42.454828+0000","last_clean":"2023-10-09T06:35:42.454828+0000","last_became_active":"2023-10-09T06:35:28.748506+0000","last_became_peered":"2023-10-09T06:35:28.748506+0000","last_unstale":"2023-10-09T06:35:42.454828+0000","last_undegraded":"2023-10-09T06:35:42.454828+0000","last_fullsized":"2023-10-09T06:35:42.454828+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":975,"log_dups_size":0,"ondisk_log_size":975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:24.343197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":975,"num_write_kb":62400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":793,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708892+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:35:34.708892+0000","last_peered":"2023-10-09T06:35:34.708892+0000","last_clean":"2023-10-09T06:35:34.708892+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:35:34.708892+0000","last_undegraded":"2023-10-09T06:35:34.708892+0000","last_fullsized":"2023-10-09T06:35:34.708892+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":425,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.709687+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:35:34.709687+0000","last_peered":"2023-10-09T06:35:34.709687+0000","last_clean":"2023-10-09T06:35:34.709687+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:35:34.709687+0000","last_undegraded":"2023-10-09T06:35:34.709687+0000","last_fullsized":"2023-10-09T06:35:34.709687+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":406,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552969+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:35:34.552969+0000","last_peered":"2023-10-09T06:35:34.552969+0000","last_clean":"2023-10-09T06:35:34.552969+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:35:34.552969+0000","last_undegraded":"2023-10-09T06:35:34.552969+0000","last_fullsized":"2023-10-09T06:35:34.552969+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":404,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555605+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:35:34.555605+0000","last_peered":"2023-10-09T06:35:34.555605+0000","last_clean":"2023-10-09T06:35:34.555605+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:35:34.555605+0000","last_undegraded":"2023-10-09T06:35:34.555605+0000","last_fullsized":"2023-10-09T06:35:34.555605+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":720,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708339+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:35:34.708339+0000","last_peered":"2023-10-09T06:35:34.708339+0000","last_clean":"2023-10-09T06:35:34.708339+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:35:34.708339+0000","last_undegraded":"2023-10-09T06:35:34.708339+0000","last_fullsized":"2023-10-09T06:35:34.708339+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":407,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.552821+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:35:34.552821+0000","last_peered":"2023-10-09T06:35:34.552821+0000","last_clean":"2023-10-09T06:35:34.552821+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:35:34.552821+0000","last_undegraded":"2023-10-09T06:35:34.552821+0000","last_fullsized":"2023-10-09T06:35:34.552821+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":429,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708763+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:35:34.708763+0000","last_peered":"2023-10-09T06:35:34.708763+0000","last_clean":"2023-10-09T06:35:34.708763+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:35:34.708763+0000","last_undegraded":"2023-10-09T06:35:34.708763+0000","last_fullsized":"2023-10-09T06:35:34.708763+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":650,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708742+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:35:34.708742+0000","last_peered":"2023-10-09T06:35:34.708742+0000","last_clean":"2023-10-09T06:35:34.708742+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:35:34.708742+0000","last_undegraded":"2023-10-09T06:35:34.708742+0000","last_fullsized":"2023-10-09T06:35:34.708742+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":132,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.710397+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:35:34.710397+0000","last_peered":"2023-10-09T06:35:34.710397+0000","last_clean":"2023-10-09T06:35:34.710397+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:35:34.710397+0000","last_undegraded":"2023-10-09T06:35:34.710397+0000","last_fullsized":"2023-10-09T06:35:34.710397+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":448,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.556002+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:35:34.556002+0000","last_peered":"2023-10-09T06:35:34.556002+0000","last_clean":"2023-10-09T06:35:34.556002+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:35:34.556002+0000","last_undegraded":"2023-10-09T06:35:34.556002+0000","last_fullsized":"2023-10-09T06:35:34.556002+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":132,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708632+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:35:34.708632+0000","last_peered":"2023-10-09T06:35:34.708632+0000","last_clean":"2023-10-09T06:35:34.708632+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:35:34.708632+0000","last_undegraded":"2023-10-09T06:35:34.708632+0000","last_fullsized":"2023-10-09T06:35:34.708632+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":458,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555719+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:35:34.555719+0000","last_peered":"2023-10-09T06:35:34.555719+0000","last_clean":"2023-10-09T06:35:34.555719+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:35:34.555719+0000","last_undegraded":"2023-10-09T06:35:34.555719+0000","last_fullsized":"2023-10-09T06:35:34.555719+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":797,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708697+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:35:34.708697+0000","last_peered":"2023-10-09T06:35:34.708697+0000","last_clean":"2023-10-09T06:35:34.708697+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:35:34.708697+0000","last_undegraded":"2023-10-09T06:35:34.708697+0000","last_fullsized":"2023-10-09T06:35:34.708697+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":142,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-10-09T06:35:36.576597+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:35:36.576597+0000","last_peered":"2023-10-09T06:35:36.576597+0000","last_clean":"2023-10-09T06:35:36.576597+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:35:36.576597+0000","last_undegraded":"2023-10-09T06:35:36.576597+0000","last_fullsized":"2023-10-09T06:35:36.576597+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":159,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708634+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:35:34.708634+0000","last_peered":"2023-10-09T06:35:34.708634+0000","last_clean":"2023-10-09T06:35:34.708634+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:35:34.708634+0000","last_undegraded":"2023-10-09T06:35:34.708634+0000","last_fullsized":"2023-10-09T06:35:34.708634+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":562,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.708345+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:35:34.708345+0000","last_peered":"2023-10-09T06:35:34.708345+0000","last_clean":"2023-10-09T06:35:34.708345+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:35:34.708345+0000","last_undegraded":"2023-10-09T06:35:34.708345+0000","last_fullsized":"2023-10-09T06:35:34.708345+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":437,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.553031+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:35:34.553031+0000","last_peered":"2023-10-09T06:35:34.553031+0000","last_clean":"2023-10-09T06:35:34.553031+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:35:34.553031+0000","last_undegraded":"2023-10-09T06:35:34.553031+0000","last_fullsized":"2023-10-09T06:35:34.553031+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":401,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-10-09T06:35:34.555457+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:35:34.555457+0000","last_peered":"2023-10-09T06:35:34.555457+0000","last_clean":"2023-10-09T06:35:34.555457+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:35:34.555457+0000","last_undegraded":"2023-10-09T06:35:34.555457+0000","last_fullsized":"2023-10-09T06:35:34.555457+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1020657664,"num_objects":15574,"num_object_clones":0,"num_object_copies":46722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15574,"num_write_kb":996736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3006791680,"data_stored":3006791680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15574,"ondisk_log_size":15574,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234461,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622980,"kb_used_data":434364,"kb_used_omap":0,"kb_used_meta":1188608,"kb_avail":92122172,"statfs":{"total":95995035648,"available":94333104128,"internally_reserved":0,"allocated":444788736,"data_stored":443749967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1217134592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:08 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":4.6699999999999999,"15min":6.1829999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":2.984,"5min":196.28899999999999,"15min":219.41900000000001},"last":1.9530000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":4.79,"15min":6.3479999999999999},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":3.1160000000000001,"5min":196.333,"15min":219.399},"last":0.374}]},{"osd":1,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":3.8900000000000001,"15min":4.3540000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":6.3920000000000003,"5min":107.87,"15min":215.61699999999999},"last":1.151},{"interface":"front","average":{"1min":1.3400000000000001,"5min":3.3660000000000001,"15min":4.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":8.0830000000000002,"5min":107.904,"15min":216.22200000000001},"last":1.9890000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:43 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":4.2050000000000001,"15min":6.9450000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":39.747,"5min":198.94200000000001,"15min":216.34299999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":4.4509999999999996,"5min":4.2679999999999998,"15min":7.0730000000000004},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.28000000000000003},"max":{"1min":39.658999999999999,"5min":198.92099999999999,"15min":216.25200000000001},"last":2.0670000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":3.0470000000000002,"15min":4.5650000000000004},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.26100000000000001},"max":{"1min":8.375,"5min":97.441999999999993,"15min":215.922},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.1040000000000001,"5min":2.9340000000000002,"15min":4.6230000000000002},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.29799999999999999},"max":{"1min":3.1549999999999998,"5min":97.162000000000006,"15min":215.94900000000001},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.67600000000000005,"15min":2.3159999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":1.774,"5min":4.9329999999999998,"15min":218.446},"last":2.1000000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.70899999999999996,"15min":2.3050000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.23699999999999999},"max":{"1min":3.1829999999999998,"5min":3.2210000000000001,"15min":216.172},"last":0.93899999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.72399999999999998,"15min":2.129},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.22900000000000001},"max":{"1min":2.766,"5min":4.2309999999999999,"15min":216.37200000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.92100000000000004,"15min":2.1139999999999999},"min":{"1min":0.312,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":2.7240000000000002,"5min":4.8970000000000002,"15min":218.15899999999999},"last":2.0299999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:35:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.3740000000000001,"15min":2.4039999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":48.283000000000001,"5min":48.283000000000001,"15min":218.19999999999999},"last":2.2370000000000001},{"interface":"front","average":{"1min":3.0870000000000002,"5min":1.2070000000000001,"15min":2.2029999999999998},"min":{"1min":0.377,"5min":0.36099999999999999,"15min":0.33200000000000002},"max":{"1min":41.448999999999998,"5min":41.448999999999998,"15min":218.05500000000001},"last":0.40699999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496097,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1221804,"kb_used_data":322788,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92523348,"statfs":{"total":95995035648,"available":94743908352,"internally_reserved":0,"allocated":330534912,"data_stored":329519441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:22 2023","interfaces":[{"interface":"back","average":{"1min":5.4930000000000003,"5min":4.1479999999999997,"15min":3.2290000000000001},"min":{"1min":0.28799999999999998,"5min":0.218,"15min":0.20999999999999999},"max":{"1min":66.867000000000004,"5min":75.061999999999998,"15min":110.926},"last":0.94399999999999995},{"interface":"front","average":{"1min":5.625,"5min":4.2839999999999998,"15min":3.4769999999999999},"min":{"1min":0.30299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":66.856999999999999,"5min":75.021000000000001,"15min":110.911},"last":1.4379999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":3.3300000000000001,"15min":5.0759999999999996},"min":{"1min":0.23100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.7949999999999999,"5min":82.159000000000006,"15min":292.92200000000003},"last":1.5129999999999999},{"interface":"front","average":{"1min":1.853,"5min":3.5569999999999999,"15min":4.9569999999999999},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":9.2560000000000002,"5min":82.174000000000007,"15min":292.904},"last":0.29799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":3.3450000000000002,"15min":3.1800000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":10.239000000000001,"5min":74.748999999999995,"15min":75.191999999999993},"last":1.538},{"interface":"front","average":{"1min":1.8080000000000001,"5min":3.3799999999999999,"15min":2.9249999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.281000000000001,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.97399999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":4.0359999999999996,"15min":3.5680000000000001},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":13.336,"5min":88.763000000000005,"15min":110.358},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.744,"5min":3.8959999999999999,"15min":3.6150000000000002},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.311},"max":{"1min":6.7350000000000003,"5min":86.831999999999994,"15min":110.43899999999999},"last":1.0029999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.73,"15min":1.488},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":86.171999999999997,"5min":86.171999999999997,"15min":86.171999999999997},"last":1.478},{"interface":"front","average":{"1min":4.556,"5min":1.617,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.223},"max":{"1min":86.727000000000004,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.33600000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:34:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.93799999999999994,"15min":1.3540000000000001},"min":{"1min":0.33300000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.907,"5min":5.7539999999999996,"15min":22.379999999999999},"last":1.8999999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1599999999999999,"15min":1.393},"min":{"1min":0.33300000000000002,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":4.6219999999999999,"5min":8.8000000000000007,"15min":24.401},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.97999999999999998,"15min":2.516},"min":{"1min":0.46100000000000002,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":3.7599999999999998,"5min":19.295999999999999,"15min":122.937},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.76900000000000002,"5min":1.135,"15min":2.657},"min":{"1min":0.40500000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.6589999999999998,"5min":19.584,"15min":122.92100000000001},"last":0.55300000000000005}]}]},{"osd":1,"up_from":110,"seq":472446402834,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1121548,"kb_used_data":327684,"kb_used_omap":4,"kb_used_meta":793851,"kb_avail":92623604,"statfs":{"total":95995035648,"available":94846570496,"internally_reserved":0,"allocated":335548416,"data_stored":333569957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":812904334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":4.5609999999999999,"15min":5.9199999999999999},"min":{"1min":0.21099999999999999,"5min":0.20399999999999999,"15min":0.189},"max":{"1min":9.4659999999999993,"5min":143.465,"15min":269.79199999999997},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.266,"5min":4.7290000000000001,"15min":6.3899999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.17899999999999999},"max":{"1min":7.9349999999999996,"5min":143.66399999999999,"15min":271.20999999999998},"last":0.38300000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:45 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":4.0869999999999997,"15min":3.6629999999999998},"min":{"1min":0.23400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":13.217000000000001,"5min":138.452,"15min":139.75399999999999},"last":0.76800000000000002},{"interface":"front","average":{"1min":3.3069999999999999,"5min":4.21,"15min":4.4119999999999999},"min":{"1min":0.254,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.244,"5min":138.47499999999999,"15min":140.29900000000001},"last":0.73699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:35:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":3.0630000000000002,"15min":3.3919999999999999},"min":{"1min":0.27400000000000002,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":5.7720000000000002,"5min":27.064,"15min":77.331999999999994},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.9119999999999999,"5min":2.0459999999999998,"15min":2.274},"min":{"1min":0.308,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":7.46,"5min":26.626999999999999,"15min":77.197000000000003},"last":3.3690000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.3170000000000002,"15min":3.1549999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.27500000000000002},"max":{"1min":2.613,"5min":38.835999999999999,"15min":99.391000000000005},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":1.7350000000000001,"15min":2.8919999999999999},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":2.2850000000000001,"5min":38.819000000000003,"15min":99.462000000000003},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.8169999999999999,"15min":2.121},"min":{"1min":0.42099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.2829999999999999,"5min":38.776000000000003,"15min":77.436999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.89800000000000002,"5min":1.8080000000000001,"15min":2.5499999999999998},"min":{"1min":0.38100000000000001,"5min":0.371,"15min":0.32700000000000001},"max":{"1min":2.2509999999999999,"5min":38.786999999999999,"15min":77.388000000000005},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 06:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":2.7120000000000002,"15min":3.1880000000000002},"min":{"1min":0.47799999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.9980000000000002,"5min":107.27200000000001,"15min":107.27200000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.94199999999999995,"5min":3.3290000000000002,"15min":3.3650000000000002},"min":{"1min":0.41899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":2.6709999999999998,"5min":107.252,"15min":107.252},"last":0.53200000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":3.194,"15min":3.5209999999999999},"min":{"1min":0.45600000000000002,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":2.6389999999999998,"5min":104.07299999999999,"15min":104.07299999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":2.6349999999999998,"15min":2.9359999999999999},"min":{"1min":0.39000000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.27,"5min":104.056,"15min":104.056},"last":0.501}]}]},{"osd":0,"up_from":91,"seq":390842024267,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123476,"kb_used_data":159692,"kb_used_omap":5,"kb_used_meta":963770,"kb_avail":92621676,"statfs":{"total":95995035648,"available":94844596224,"internally_reserved":0,"allocated":163524608,"data_stored":161250013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":986901114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:35:40 2023","interfaces":[{"interface":"back","average":{"1min":1.738,"5min":1.9410000000000001,"15min":3.3079999999999998},"min":{"1min":0.192,"5min":0.183,"15min":0.16600000000000001},"max":{"1min":13.728,"5min":33.091999999999999,"15min":191.446},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":2.5289999999999999,"15min":3.6339999999999999},"min":{"1min":0.25700000000000001,"5min":0.187,"15min":0.187},"max":{"1min":15.541,"5min":24.838999999999999,"15min":191.393},"last":0.25700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:52 2023","interfaces":[{"interface":"back","average":{"1min":3.4199999999999999,"5min":2.8119999999999998,"15min":2.6429999999999998},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.19900000000000001},"max":{"1min":12.532,"5min":16.936,"15min":57.162999999999997},"last":0.29799999999999999},{"interface":"front","average":{"1min":4.133,"5min":3.524,"15min":3.121},"min":{"1min":0.22800000000000001,"5min":0.22600000000000001,"15min":0.20999999999999999},"max":{"1min":13.579000000000001,"5min":15.794,"15min":57.585000000000001},"last":0.5}]},{"osd":3,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":5.7140000000000004,"15min":6.0190000000000001},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":24.129000000000001,"5min":187.172,"15min":208.11199999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":5.3150000000000004,"5min":5.6699999999999999,"15min":5.9800000000000004},"min":{"1min":0.34399999999999997,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":24.797000000000001,"5min":194.851,"15min":208.25899999999999},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.649,"15min":1.5760000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":8.9800000000000004,"5min":8.9800000000000004,"15min":9.7159999999999993},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.504,"5min":2.2090000000000001,"15min":1.929},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.385999999999999,"5min":12.385999999999999,"15min":18.081},"last":0.84199999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.7669999999999999,"15min":1.873},"min":{"1min":0.309,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":9.0099999999999998,"5min":9.0099999999999998,"15min":62.984999999999999},"last":2.1659999999999999},{"interface":"front","average":{"1min":2.601,"5min":2.3380000000000001,"15min":2.3730000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.8239999999999998,"5min":13.968999999999999,"15min":64.983999999999995},"last":0.83199999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.1829999999999998,"5min":2.4550000000000001,"15min":3.036},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":22.292000000000002,"5min":22.292000000000002,"15min":120.572},"last":1.369},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9910000000000001,"15min":2.52},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.30299999999999999},"max":{"1min":22.475000000000001,"5min":22.475000000000001,"15min":120.64400000000001},"last":1.4359999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":2.472,"15min":2.734},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":22.879000000000001,"5min":22.879000000000001,"15min":122.018},"last":1.835},{"interface":"front","average":{"1min":4.1139999999999999,"5min":2.79,"15min":3.125},"min":{"1min":0.39700000000000002,"5min":0.39100000000000001,"15min":0.377},"max":{"1min":22.683,"5min":22.683,"15min":122.32299999999999},"last":0.55600000000000005}]}]},{"osd":2,"up_from":128,"seq":549755814136,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206908,"kb_used_data":455476,"kb_used_omap":7,"kb_used_meta":751416,"kb_avail":92538244,"statfs":{"total":95995035648,"available":94759161856,"internally_reserved":0,"allocated":466407424,"data_stored":464499869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":769450065},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":15.65,"5min":7.032,"15min":5.5129999999999999},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.157},"max":{"1min":156.94499999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":15.272,"5min":6.734,"15min":5.0049999999999999},"min":{"1min":0.315,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":156.922,"5min":156.922,"15min":168.49700000000001},"last":0.26600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":14.833,"5min":6.202,"15min":4.7000000000000002},"min":{"1min":0.41499999999999998,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":140.452,"5min":140.452,"15min":205.148},"last":0.45700000000000002},{"interface":"front","average":{"1min":14.69,"5min":5.9409999999999998,"15min":4.8739999999999997},"min":{"1min":0.35599999999999998,"5min":0.183,"15min":0.183},"max":{"1min":139.75299999999999,"5min":139.75299999999999,"15min":205.267},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:35:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":3.6709999999999998,"15min":3.27},"min":{"1min":0.26600000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.243,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.8260000000000001,"5min":4.0609999999999999,"15min":3.698},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":11.176,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.495}]},{"osd":4,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":2.0249999999999999,"15min":2.27},"min":{"1min":0.40200000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":3.423,"5min":44.314,"15min":90.938999999999993},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.4329999999999998,"15min":2.6019999999999999},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":4.6950000000000003,"5min":43.896000000000001,"15min":90.950000000000003},"last":1.093}]},{"osd":5,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.794,"15min":1.675},"min":{"1min":0.379,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.7250000000000001,"5min":13.102,"15min":17.800000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.958,"5min":2.0009999999999999,"15min":1.9630000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.6779999999999999,"5min":11.285,"15min":11.285},"last":1.1339999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":4.524,"15min":3.4060000000000001},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":20.445,"5min":123.47799999999999,"15min":138.291},"last":1.0509999999999999},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.3840000000000003,"15min":3.202},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":20.148,"5min":122.517,"15min":138.01599999999999},"last":0.748}]},{"osd":7,"last update":"Mon Oct 9 06:34:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":2.7839999999999998,"15min":2.3820000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":16.574999999999999,"5min":47.389000000000003,"15min":55.851999999999997},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.444,"5min":2.3570000000000002,"15min":2.1549999999999998},"min":{"1min":0.40300000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":3.3330000000000002,"5min":47.206000000000003,"15min":55.731000000000002},"last":0.70899999999999996}]}]},{"osd":3,"up_from":149,"seq":639950127334,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070632,"kb_used_data":384032,"kb_used_omap":2,"kb_used_meta":686589,"kb_avail":92674520,"statfs":{"total":95995035648,"available":94898708480,"internally_reserved":0,"allocated":393248768,"data_stored":391419815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":703067189},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":6.9050000000000002,"15min":5.8890000000000002},"min":{"1min":0.20300000000000001,"5min":0.19,"15min":0.19},"max":{"1min":7.4420000000000002,"5min":161.821,"15min":187.852},"last":0.255},{"interface":"front","average":{"1min":2.839,"5min":6.3209999999999997,"15min":5.8490000000000002},"min":{"1min":0.23200000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":11.301,"5min":163.52000000000001,"15min":187.923},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":4.3849999999999998,"15min":4.2930000000000001},"min":{"1min":0.24399999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.343999999999999,"5min":155.23099999999999,"15min":187.89099999999999},"last":1.554},{"interface":"front","average":{"1min":3.2879999999999998,"5min":5.6909999999999998,"15min":4.2519999999999998},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":12.288,"5min":155.24799999999999,"15min":155.24799999999999},"last":0.439}]},{"osd":2,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":4.5389999999999997,"15min":4.1130000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.4800000000000004,"5min":167.88499999999999,"15min":187.76400000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":5.1779999999999999,"15min":4.2709999999999999},"min":{"1min":0.29899999999999999,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":8.3179999999999996,"5min":168.16900000000001,"15min":187.79499999999999},"last":0.29599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":2.4950000000000001,"15min":2.0739999999999998},"min":{"1min":0.33100000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.8109999999999999,"5min":48.249000000000002,"15min":48.249000000000002},"last":1.736},{"interface":"front","average":{"1min":1.97,"5min":2.5270000000000001,"15min":2.6890000000000001},"min":{"1min":0.32600000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.871,"5min":48.031999999999996,"15min":187.67400000000001},"last":0.66900000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":5.8019999999999996,"5min":3.448,"15min":2.9769999999999999},"min":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":87.075000000000003,"5min":87.075000000000003,"15min":187.833},"last":0.625},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.036,"15min":2.2989999999999999},"min":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":2.504,"5min":48.106000000000002,"15min":187.952},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2989999999999999,"5min":2.7709999999999999,"15min":3.3620000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":11.994999999999999,"5min":48.270000000000003,"15min":187.78200000000001},"last":2.23},{"interface":"front","average":{"1min":0.82699999999999996,"5min":2.2250000000000001,"15min":2.8769999999999998},"min":{"1min":0.436,"5min":0.375,"15min":0.32900000000000001},"max":{"1min":2.73,"5min":48.155000000000001,"15min":187.81800000000001},"last":0.97099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:35:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.5510000000000002,"15min":3.391},"min":{"1min":0.42399999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.5519999999999996,"5min":48.292000000000002,"15min":192.20400000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":6.0110000000000001,"5min":3.6440000000000001,"15min":3.758},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":87.001999999999995,"5min":87.001999999999995,"15min":187.86500000000001},"last":1.6140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1558196,"kb_used_data":667308,"kb_used_omap":0,"kb_used_meta":890880,"kb_avail":92186956,"statfs":{"total":95995035648,"available":94399442944,"internally_reserved":0,"allocated":683323392,"data_stored":682300106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":912261120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":17.963999999999999,"5min":14.997,"15min":6.7309999999999999},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":286.30099999999999,"5min":287.58800000000002,"15min":287.58800000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":18.295000000000002,"5min":14.887,"15min":6.9299999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":286.36000000000001,"5min":287.57100000000003,"15min":287.57100000000003},"last":0.42299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":10.643000000000001,"5min":5.5620000000000003,"15min":3.4590000000000001},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":134.37100000000001,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":10.766,"5min":5.7549999999999999,"15min":3.669},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":134.21700000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":12.302,"5min":5.3170000000000002,"15min":3.7749999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":148.36099999999999,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.442},{"interface":"front","average":{"1min":12.159000000000001,"5min":5.2990000000000004,"15min":3.7559999999999998},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":145.755,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.39200000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:35:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":6.649,"15min":5.702},"min":{"1min":0.312,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":4.5529999999999999,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.258,"5min":6.5609999999999999,"15min":5.5380000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":4.3360000000000003,"5min":161.934,"15min":161.934},"last":0.94199999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:34:40 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.70699999999999996,"15min":0.98999999999999999},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80600000000000005,"5min":6.0199999999999996,"15min":10.673},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.82799999999999996,"15min":1.0029999999999999},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.548,"5min":5.7000000000000002,"15min":12.552},"last":0.46400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:35:13 2023","interfaces":[{"interface":"back","average":{"1min":8.0860000000000003,"5min":2.2149999999999999,"15min":1.8029999999999999},"min":{"1min":0.308,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":115.70999999999999,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":8.4830000000000005,"5min":2.3029999999999999,"15min":2.1150000000000002},"min":{"1min":0.35199999999999998,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":115.68600000000001,"5min":115.68600000000001,"15min":163.381},"last":1.5369999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:35:20 2023","interfaces":[{"interface":"back","average":{"1min":7.0170000000000003,"5min":2.7650000000000001,"15min":1.9119999999999999},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":106.801,"5min":106.801,"15min":106.801},"last":0.503},{"interface":"front","average":{"1min":6.7919999999999998,"5min":2.9340000000000002,"15min":1.871},"min":{"1min":0.41899999999999998,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":106.827,"5min":106.827,"15min":106.827},"last":0.70099999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757735,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1366380,"kb_used_data":476964,"kb_used_omap":0,"kb_used_meta":889408,"kb_avail":92378772,"statfs":{"total":95995035648,"available":94595862528,"internally_reserved":0,"allocated":488411136,"data_stored":487392294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910753792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,4,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":12.871,"5min":8.1099999999999994,"15min":5.742},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":149.149,"5min":294.89699999999999,"15min":294.89699999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":12.737,"5min":7.6159999999999997,"15min":5.5460000000000003},"min":{"1min":0.245,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":149.21100000000001,"5min":294.30399999999997,"15min":294.30399999999997},"last":0.58099999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":7.4349999999999996,"5min":4.0570000000000004,"15min":3.274},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":119.024,"5min":119.024,"15min":126.71899999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":7.5579999999999998,"5min":4.2779999999999996,"15min":3.4369999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":119.133,"5min":119.133,"15min":126.931},"last":0.26100000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:34:50 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":2.4609999999999999,"15min":2.4969999999999999},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.5149999999999997,"5min":64.150000000000006,"15min":81.260000000000005},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.2290000000000001,"15min":2.4289999999999998},"min":{"1min":0.312,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":4.8550000000000004,"5min":64.507000000000005,"15min":81.245000000000005},"last":0.71399999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:35:10 2023","interfaces":[{"interface":"back","average":{"1min":9.0340000000000007,"5min":4.4349999999999996,"15min":5.1299999999999999},"min":{"1min":0.35399999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":85.671000000000006,"5min":92.521000000000001,"15min":144.56399999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":8.4719999999999995,"5min":4.1699999999999999,"15min":4.9400000000000004},"min":{"1min":0.373,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":85.593999999999994,"5min":94.492000000000004,"15min":144.62799999999999},"last":0.64200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:35:40 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.64400000000000002,"15min":1.1080000000000001},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":3.121,"5min":3.6400000000000001,"15min":32.850999999999999},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.58199999999999996,"15min":0.98399999999999999},"min":{"1min":0.34599999999999997,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":3.863,"5min":3.863,"15min":32.82},"last":0.77400000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.387,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.9460000000000002,"5min":57.505000000000003,"15min":57.505000000000003},"last":0.745},{"interface":"front","average":{"1min":1.028,"5min":1.5169999999999999,"15min":1.7470000000000001},"min":{"1min":0.35099999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.173,"5min":55.040999999999997,"15min":55.040999999999997},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 06:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.879,"15min":1.331},"min":{"1min":0.33800000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.2200000000000002,"5min":6.226,"15min":30.658999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.76200000000000001,"15min":1.351},"min":{"1min":0.38400000000000001,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":30.683},"last":0.68400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":162856960,"data_stored":162856960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":332595200,"data_stored":332595200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":495190016,"data_stored":495190016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":283377664,"data_stored":283377664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":572260352,"data_stored":572260352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":508952576,"data_stored":508952576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":323878912,"data_stored":323878912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680000,"data_stored":327680000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:35:46.507 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 16 2023-10-09T06:35:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:46.508 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:46.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:46.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 23496 23480 146.712 102.75 0.00471217 0.00681009 2023-10-09T06:35:46.609 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:46.613 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:46 smithi150 ceph-mon[194178]: pgmap v1341: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:35:46.673 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:46.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:46 smithi107 ceph-mon[212750]: pgmap v1341: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:35:46.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:46 smithi107 ceph-mon[220921]: pgmap v1341: 119 pgs: 119 active+clean; 974 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-09T06:35:47.111 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:47.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:47.216 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:47.259 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:47.342 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:47.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:47.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:35:47.517 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:47.534 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:47.561 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 25633 25617 145.516 133.562 0.00794735 0.00686559 2023-10-09T06:35:47.871 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:47.881 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:35:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:47.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:35:47.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:47.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:47.889 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:47.889 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:47.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:35:47.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:35:47.988 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:48.005 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:35:48.005 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:35:48.036 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:48.128 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:48.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:48.240 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:48.295 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:48.563 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 27625 27609 143.764 124.5 0.00697113 0.00695081 2023-10-09T06:35:48.660 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:35:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:48 smithi107 ceph-mon[212750]: pgmap v1342: 119 pgs: 119 active+clean; 1010 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:35:48.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:48.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:48 smithi107 ceph-mon[220921]: pgmap v1342: 119 pgs: 119 active+clean; 1010 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:35:48.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:48.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:48 smithi150 ceph-mon[194178]: pgmap v1342: 119 pgs: 119 active+clean; 1010 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-09T06:35:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:35:49.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 29117 29101 139.879 93.25 0.00501891 0.00714321 2023-10-09T06:35:49.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[212750]: failed to encode map e442 with expected crc 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[212750]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[220921]: failed to encode map e442 with expected crc 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:49.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:49 smithi107 ceph-mon[220921]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T06:35:49.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:49 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:35:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:49 smithi150 ceph-mon[194178]: failed to encode map e442 with expected crc 2023-10-09T06:35:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1982503862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:35:49.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:49 smithi150 ceph-mon[194178]: osdmap e442: 8 total, 8 up, 8 in 2023-10-09T06:35:50.436 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:50.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:50.543 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:50.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 30966 30950 138.141 115.562 0.0049568 0.00723352 2023-10-09T06:35:50.598 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:50.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:50 smithi107 ceph-mon[212750]: pgmap v1344: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:35:50.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:50 smithi107 ceph-mon[220921]: pgmap v1344: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:35:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:50 smithi150 ceph-mon[194178]: pgmap v1344: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T06:35:51.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 32912 32896 137.04 121.625 0.00489325 0.00729164 2023-10-09T06:35:52.057 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:52.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:52.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:35:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:35:52] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T06:35:52.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:35:52.246 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:52.305 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:52.348 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:35:52.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:52.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:35:52.455 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:35:52.517 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:52.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 34572 34556 134.959 103.75 0.0139708 0.00740425 2023-10-09T06:35:52.620 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:35:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:52.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:35:52.733 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:52.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[212750]: pgmap v1345: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T06:35:52.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2936987380' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.783 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[220921]: pgmap v1345: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T06:35:52.783 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2936987380' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.784 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:52 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.796 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:52.839 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:35:52.874 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:35:52.880 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:52.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:52.904 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:35:52.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:52 smithi150 ceph-mon[194178]: pgmap v1345: 119 pgs: 119 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-09T06:35:52.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2936987380' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:52 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T06:35:52.939 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:35:52.967 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:35:52.994 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:53.004 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:35:53.037 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:35:53.064 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:53.069 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.071 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.072 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.072 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:35:53.163 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:53.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:53.270 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:53.304 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:53.510 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 13 pgp_num to 16 2023-10-09T06:35:53.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 36102 36086 132.645 95.625 0.006588 0.00750967 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2980733379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[220921]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[220921]: failed to encode map e443 with expected crc 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2980733379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:53.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:35:53.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[212750]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T06:35:53.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:53 smithi107 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:35:53.929 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:35:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2980733379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:35:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T06:35:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:53 smithi150 ceph-mon[194178]: osdmap e443: 8 total, 8 up, 8 in 2023-10-09T06:35:54.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:53 smithi150 ceph-mon[194178]: failed to encode map e443 with expected crc 2023-10-09T06:35:54.562 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 37343 37327 129.585 77.5625 0.0091593 0.007711 2023-10-09T06:35:54.833 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:54.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:54.943 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:55.002 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:54 smithi150 ceph-mon[194178]: pgmap v1346: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:35:55.002 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:54 smithi150 ceph-mon[194178]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.002 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:54 smithi150 ceph-mon[194178]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.002 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:54 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:55.008 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:55.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[212750]: pgmap v1346: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:35:55.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[212750]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:55.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[220921]: pgmap v1346: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T06:35:55.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[220921]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[220921]: failed to encode map e443 with expected crc 2023-10-09T06:35:55.016 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:54 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:35:55.123 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:55.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:55.249 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:55.298 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:55.416 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:55.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:55.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:55.519 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:55.557 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:55.563 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 38863 38847 127.757 95 0.0155592 0.00782181 2023-10-09T06:35:55.577 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:55.690 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:35:55.690 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:35:55.690 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67296 11.2 11:4fcd2a40:::benchmark_data_smithi107_418363_object67295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.852661+0000", 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 510.83451688600002, 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4124151779999998, 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:35:55.691 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67296 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852661+0000", 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852661+0000", 2023-10-09T06:35:55.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852660+0000", 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852691+0000", 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1084999999999998e-05 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852702+0000", 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0575e-05 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852710+0000", 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6799999999999993e-06 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.853088+0000", 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00037833300000000001 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.853204+0000", 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116292 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:35:55.695 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.853345+0000", 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014062600000000001 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.863922+0000", 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01057751 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.867687+0000", 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00376459 2023-10-09T06:35:55.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875194+0000", 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075072589999999996 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875206+0000", 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.219e-05 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.862114+0000", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98690718899999996 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.862127+0000", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3101e-05 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.243825+0000", 2023-10-09T06:35:55.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.381698665 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.244855+0000", 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001029379 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.264972+0000", 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.020117537000000001 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.265076+0000", 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000104029 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:67299 11.2 11:4480f7cb:::benchmark_data_smithi107_418363_object67298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:27:24.852722+0000", 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 510.83445610899997, 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4240359890000001, 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 67299 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.701 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852722+0000", 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852722+0000", 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852721+0000", 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852744+0000", 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2895999999999999e-05 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.703 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852751+0000", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4740000000000002e-06 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.852756+0000", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3020000000000002e-06 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.855631+0000", 2023-10-09T06:35:55.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028745799999999998 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.855775+0000", 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014462599999999999 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,3", 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.855932+0000", 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000157107 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.867353+0000", 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011420410000000001 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.867764+0000", 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041071499999999997 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875249+0000", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074857020000000003 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:24.875259+0000", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.893e-06 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.862841+0000", 2023-10-09T06:35:55.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98758214600000005 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:25.862853+0000", 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1955e-05 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.257930+0000", 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3950763020000001 2023-10-09T06:35:55.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.258678+0000", 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00074830299999999997 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.276661+0000", 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017983277999999998 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:27:27.276758+0000", 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6824000000000007e-05 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:76830 12.0 e382/361 12:0418eee9:::benchmark_data_smithi107_479504_object76829:head v 382'4793, mlcod=382'4791)", 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:11.742706+0000", 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.944472168, 2023-10-09T06:35:55.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20775029, 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742706+0000", 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742706+0000", 2023-10-09T06:35:55.711 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742705+0000", 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742739+0000", 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3661000000000002e-05 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742750+0000", 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1722000000000001e-05 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742758+0000", 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9869999999999999e-06 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742808+0000", 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0123000000000001e-05 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.742827+0000", 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8776000000000001e-05 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.747305+0000", 2023-10-09T06:35:55.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0044775750000000001 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.950456+0000", 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.203151428 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:76836 12.c e382/361 12:3bbd3750:::benchmark_data_smithi107_479504_object76835:head v 382'4814, mlcod=382'4810)", 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:11.745324+0000", 2023-10-09T06:35:55.715 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 223.941853721, 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.210001189, 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745324+0000", 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745324+0000", 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745324+0000", 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745347+0000", 2023-10-09T06:35:55.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3125e-05 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745357+0000", 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0722e-05 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.745364+0000", 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.759e-06 2023-10-09T06:35:55.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.950447+0000", 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20508237400000001 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.952981+0000", 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002534929 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.955301+0000", 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023190260000000001 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:11.955326+0000", 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5011000000000002e-05 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.720 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:150366 12.8 e410/390 12:1202f04a:::benchmark_data_smithi107_479504_object50755:head v 410'7569, mlcod=410'7567)", 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:55.371022+0000", 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 120.316156091, 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21897704700000001, 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371022+0000", 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371022+0000", 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371021+0000", 2023-10-09T06:35:55.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371029+0000", 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2740000000000007e-06 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371041+0000", 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1499e-05 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371048+0000", 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9800000000000001e-06 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371091+0000", 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3460000000000001e-05 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.724 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.371113+0000", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2067999999999999e-05 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.589969+0000", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21885581300000001 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.589999+0000", 2023-10-09T06:35:55.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0088999999999999e-05 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:150368 12.10 12:0b5b593f:::benchmark_data_smithi107_479504_object50757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:55.374335+0000", 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 120.312843222, 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23155305600000001, 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.726 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 150368 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.374335+0000", 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.727 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.374335+0000", 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.374333+0000", 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.374340+0000", 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6479999999999997e-06 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585806+0000", 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21146579500000001 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585820+0000", 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3685e-05 2023-10-09T06:35:55.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.588768+0000", 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029484120000000001 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.588920+0000", 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015232399999999999 2023-10-09T06:35:55.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.589331+0000", 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041067099999999999 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.592443+0000", 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031117739999999999 2023-10-09T06:35:55.731 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.595461+0000", 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030182170000000001 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.599785+0000", 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0043236489999999997 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.605360+0000", 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055750310000000003 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.605415+0000", 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5488999999999997e-05 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.605888+0000", 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00047278799999999997 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:150371 12.d e410/361 12:bfa844f1:::benchmark_data_smithi107_479504_object50760:head v 410'9557, mlcod=410'9555)", 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:55.378894+0000", 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 120.30828385700001, 2023-10-09T06:35:55.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21267947600000001, 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.378894+0000", 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.378894+0000", 2023-10-09T06:35:55.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.378893+0000", 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.378903+0000", 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0359e-05 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585787+0000", 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20688342900000001 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585800+0000", 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2818e-05 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585857+0000", 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7346999999999998e-05 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.585879+0000", 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2201999999999999e-05 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.738 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.591539+0000", 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056596199999999998 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:55.591574+0000", 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4876999999999998e-05 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.739 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153010 12.18 e410/390 12:1b91cf60:::benchmark_data_smithi107_479504_object53399:head v 410'7554, mlcod=410'7552)", 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.001580+0000", 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.685597747, 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23592447, 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001580+0000", 2023-10-09T06:35:55.740 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001580+0000", 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001580+0000", 2023-10-09T06:35:55.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001586+0000", 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.443e-06 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001596+0000", 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9820000000000003e-06 2023-10-09T06:35:55.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001602+0000", 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2090000000000002e-06 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001662+0000", 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0133000000000002e-05 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.001681+0000", 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8626000000000001e-05 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237480+0000", 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.235798964 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237505+0000", 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4763999999999999e-05 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153012 12.b e410/361 12:db04fead:::benchmark_data_smithi107_479504_object53401:head v 410'9392, mlcod=410'9389)", 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.003601+0000", 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.683577034, 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24164556400000001, 2023-10-09T06:35:55.745 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003601+0000", 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003601+0000", 2023-10-09T06:35:55.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003600+0000", 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003606+0000", 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5740000000000003e-06 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003614+0000", 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3960000000000006e-06 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.003620+0000", 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.57e-06 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237378+0000", 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23375765300000001 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237401+0000", 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3620000000000001e-05 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.245216+0000", 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0078149580000000003 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.245247+0000", 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0338e-05 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153016 12.d e410/361 12:b3683356:::benchmark_data_smithi107_479504_object53405:head v 410'9707, mlcod=410'9705)", 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.008280+0000", 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.678897905, 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.229130798, 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008280+0000", 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008280+0000", 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008279+0000", 2023-10-09T06:35:55.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008289+0000", 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3680000000000008e-06 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008301+0000", 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2488e-05 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008309+0000", 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9480000000000008e-06 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008378+0000", 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8893999999999995e-05 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008401+0000", 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2949999999999999e-05 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237385+0000", 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.228984566 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237411+0000", 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5548999999999999e-05 2023-10-09T06:35:55.755 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153022 12.0 e410/390 12:03cdd803:::benchmark_data_smithi107_479504_object53411:head v 410'7689, mlcod=410'7687)", 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.008410+0000", 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.678767816, 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22911762699999999, 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008410+0000", 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008410+0000", 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008410+0000", 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008417+0000", 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2930000000000003e-06 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008428+0000", 2023-10-09T06:35:55.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0906e-05 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008435+0000", 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.447e-06 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008492+0000", 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6663000000000003e-05 2023-10-09T06:35:55.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008586+0000", 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4050999999999998e-05 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237508+0000", 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.228921599 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237528+0000", 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0288999999999998e-05 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:153023 12.10 12:0f377afd:::benchmark_data_smithi107_479504_object53412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.008423+0000", 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.678755491, 2023-10-09T06:35:55.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24969936700000001, 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 153023 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008423+0000", 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008423+0000", 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008421+0000", 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008426+0000", 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2650000000000004e-06 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008436+0000", 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8209999999999997e-06 2023-10-09T06:35:55.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008442+0000", 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3459999999999999e-06 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.240065+0000", 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23162303100000001 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.240214+0000", 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014893099999999999 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,7", 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.240651+0000", 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043647599999999999 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.242647+0000", 2023-10-09T06:35:55.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001995808 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.243678+0000", 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0010312629999999999 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.253072+0000", 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.009393808 2023-10-09T06:35:55.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.257741+0000", 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0046692670000000004 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.257789+0000", 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8405999999999997e-05 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.258122+0000", 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000332527 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153025 12.5 e410/390 12:a0bf62b8:::benchmark_data_smithi107_479504_object53414:head v 410'7696, mlcod=410'7694)", 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.008510+0000", 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.678667605, 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22887125899999999, 2023-10-09T06:35:55.769 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008510+0000", 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008510+0000", 2023-10-09T06:35:55.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008510+0000", 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008515+0000", 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7500000000000003e-06 2023-10-09T06:35:55.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008521+0000", 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9950000000000002e-06 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008526+0000", 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8640000000000004e-06 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.008991+0000", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000464964 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009010+0000", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9140999999999999e-05 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237344+0000", 2023-10-09T06:35:55.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22833393399999999 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237382+0000", 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8049000000000001e-05 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153018 12.b e410/361 12:d41cd46f:::benchmark_data_smithi107_479504_object53407:head v 410'9393, mlcod=410'9390)", 2023-10-09T06:35:55.774 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.009357+0000", 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.67782151599999, 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23590984100000001, 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009357+0000", 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.775 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009357+0000", 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009356+0000", 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.776 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009363+0000", 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6590000000000001e-06 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009373+0000", 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0361e-05 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.009380+0000", 2023-10-09T06:35:55.777 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.951e-06 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.240052+0000", 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23067170400000001 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.240069+0000", 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7382999999999999e-05 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.778 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.245249+0000", 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005180222 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.245266+0000", 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7254000000000001e-05 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.779 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153024 12.b e410/361 12:d7bae3ea:::benchmark_data_smithi107_479504_object53413:head v 410'9394, mlcod=410'9390)", 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.010322+0000", 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.676855575, 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23987583300000001, 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.780 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.010322+0000", 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.010322+0000", 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.010322+0000", 2023-10-09T06:35:55.781 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.010328+0000", 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2450000000000003e-06 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237458+0000", 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.227129571 2023-10-09T06:35:55.782 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237465+0000", 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9650000000000002e-06 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.242567+0000", 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051023470000000001 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.783 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.242581+0000", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3760999999999999e-05 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.250163+0000", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075817879999999999 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.250198+0000", 2023-10-09T06:35:55.784 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5806000000000002e-05 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46340.0:153030 12.17 e410/390 12:ea01dd7a:::benchmark_data_smithi107_479504_object53419:head v 410'7593, mlcod=410'7591)", 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:33:59.013322+0000", 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 116.67385585700001, 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23698750699999999, 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.785 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.013322+0000", 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.013322+0000", 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.786 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.013321+0000", 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.013329+0000", 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5989999999999996e-06 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.787 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237350+0000", 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22402118200000001 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237361+0000", 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0424999999999999e-05 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.788 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237500+0000", 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000139581 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.237520+0000", 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0041e-05 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.250283+0000", 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012762542999999999 2023-10-09T06:35:55.789 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:33:59.250310+0000", 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6970000000000001e-05 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:1106 13.f e441/437 13:f90c42c6:::benchmark_data_smithi107_535588_object1105:head v 441'65, mlcod=441'62)", 2023-10-09T06:35:55.790 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:36.895551+0000", 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 18.791627131999999, 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18795642800000001, 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895551+0000", 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.791 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895551+0000", 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895550+0000", 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.792 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895585+0000", 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4921e-05 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895595+0000", 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5980000000000003e-06 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895601+0000", 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.117e-06 2023-10-09T06:35:55.793 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895632+0000", 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1347000000000003e-05 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.794 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895645+0000", 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2852e-05 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.083472+0000", 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18782697000000001 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.795 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.083507+0000", 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5126000000000002e-05 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:1111 13.8 e441/437 13:1b92a876:::benchmark_data_smithi107_535588_object1110:head v 441'59, mlcod=441'57)", 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:36.895857+0000", 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 18.791320813999999, 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.18818439200000001, 2023-10-09T06:35:55.796 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895857+0000", 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895857+0000", 2023-10-09T06:35:55.797 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895857+0000", 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:36.895917+0000", 2023-10-09T06:35:55.798 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0859999999999997e-05 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.082591+0000", 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.186674005 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.082625+0000", 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3016999999999999e-05 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.799 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.082726+0000", 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101375 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.082802+0000", 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5927999999999995e-05 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.800 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.083982+0000", 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0011798469999999999 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:37.084042+0000", 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9969000000000001e-05 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.801 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:36073 13.1 e442/437 13:8ff98c89:::benchmark_data_smithi107_535588_object36072:head v 442'2190, mlcod=442'2187)", 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:53.460455+0000", 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 2.2267232909999999, 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.211706534, 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460455+0000", 2023-10-09T06:35:55.802 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460455+0000", 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460455+0000", 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967296 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.803 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460491+0000", 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.629e-05 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460496+0000", 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.254e-06 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.804 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460500+0000", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5990000000000002e-06 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460536+0000", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6229999999999997e-05 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.460553+0000", 2023-10-09T06:35:55.805 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.747e-05 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.672126+0000", 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21157299900000001 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.672161+0000", 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4913999999999998e-05 2023-10-09T06:35:55.806 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:36082 13.1 e442/437 13:8ec7c14a:::benchmark_data_smithi107_535588_object36081:head v 442'2191, mlcod=442'2187)", 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:53.463761+0000", 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 2.223416716, 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20843077400000001, 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:35:55.807 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.463761+0000", 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.463761+0000", 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.808 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.463761+0000", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.463801+0000", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0296000000000002e-05 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.537365+0000", 2023-10-09T06:35:55.809 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.073563932999999998 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.537380+0000", 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5045000000000001e-05 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.538843+0000", 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001462496 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.810 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.538860+0000", 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6939000000000002e-05 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.672166+0000", 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.133306235 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:35:55.811 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:53.672192+0000", 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6349000000000001e-05 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:35:55.812 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:35:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:55.813 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:55.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:55.914 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:55.975 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:56.175 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:56.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:56.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:56.288 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:35:56.317 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:56.563 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:35:56.564389+0000 min lat: 0.00116741 max lat: 0.272235 avg lat: 0.00800245 2023-10-09T06:35:56.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:35:56.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 16 39990 39974 124.891 70.4375 0.00802843 0.00800245 2023-10-09T06:35:56.726 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:35:56.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:56.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:35:56.830 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:35:56.898 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:56.940 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:56 smithi107 ceph-mon[212750]: pgmap v1348: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T06:35:56.941 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:56 smithi107 ceph-mon[220921]: pgmap v1348: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T06:35:56.941 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:35:57.058 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:35:57.088 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:35:57.096 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:56 smithi150 ceph-mon[194178]: pgmap v1348: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2023-10-09T06:35:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:57.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:35:57.199 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:35:57.257 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:57.414 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:35:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:57.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:35:57.521 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:57.547 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:57.563 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 41272 41256 122.759 80.125 0.00250837 0.0080861 2023-10-09T06:35:57.858 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:57.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:57.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:57.964 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:58.123 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:58.124 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:35:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:58.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:35:58.190 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:35:58.234 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:58.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:58.440 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:35:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:35:58.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:35:58.545 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:35:58.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 42641 42625 121.069 85.5625 0.0155924 0.00825334 2023-10-09T06:35:58.610 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:35:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:58 smithi107 ceph-mon[212750]: pgmap v1349: 119 pgs: 119 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:35:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:58 smithi107 ceph-mon[220921]: pgmap v1349: 119 pgs: 119 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:35:59.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:58 smithi150 ceph-mon[194178]: pgmap v1349: 119 pgs: 119 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T06:35:59.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 44080 44064 119.715 89.9375 0.00485501 0.00834814 2023-10-09T06:35:59.703 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:36:00.017 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:00.017 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:00.017 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T06:36:00.017 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:36:00.018 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:36:00.018 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T06:36:00.026 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:35:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:00.027 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:35:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:00.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:35:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:00.261 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:00.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:00.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:00.368 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:00.419 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:00.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 45273 45257 117.833 74.5625 0.00955411 0.0084815 2023-10-09T06:36:00.662 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:00.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: pgmap v1350: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:36:00.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:00.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.931 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:00.932 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:00.933 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: pgmap v1350: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:36:00.933 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:00.934 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: pgmap v1350: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2170629267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: osdmap e444: 8 total, 8 up, 8 in 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.935 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.936 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.936 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: failed to encode map e444 with expected crc 2023-10-09T06:36:00.936 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:00.936 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:01.032 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:01.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:01.141 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:01.206 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:01.385 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:01.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:01.493 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:01.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 46649 46633 116.56 86 0.00666665 0.00857343 2023-10-09T06:36:01.568 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:01.671 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:01.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:01.711 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:01.746 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:01.782 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:01.784 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:01.830 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:01.836 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:01.836 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3983669160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:01.843 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:01.856 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:01.891 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:01.935 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:01.937 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:01.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:01.968 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:01.972 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:02.004 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:02.004 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3983669160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:02.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:02.004 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3983669160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:02.044 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:02.049 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:02.049 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:02.122 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:02] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:36:02.564 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 48114 48098 115.598 91.5625 0.0147791 0.0086458 2023-10-09T06:36:02.707 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:02 smithi107 ceph-mon[212750]: pgmap v1352: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:36:02.708 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:02 smithi107 ceph-mon[220921]: pgmap v1352: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:36:03.131 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:36:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:02 smithi150 ceph-mon[194178]: pgmap v1352: 119 pgs: 119 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-10-09T06:36:03.318 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:03.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:03.421 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:03.467 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:03.566 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 49207 49191 113.837 68.3125 0.0087608 0.00877955 2023-10-09T06:36:03.586 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:36:03.741 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:03.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:03.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:03.851 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:03.936 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:03.973 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:04.129 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:04.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:04.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:04.234 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:04.295 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:04.392 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:04.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:04.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:04.500 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:04.542 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:04.566 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 50619 50603 112.923 88.25 0.010045 0.00884981 2023-10-09T06:36:04.622 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:04.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:04.728 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:04.791 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:04.865 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:04.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[212750]: pgmap v1353: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[220921]: pgmap v1353: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:04.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:04 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:04.971 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:04.997 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:05.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:04 smithi150 ceph-mon[194178]: pgmap v1353: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:05.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:05.161 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:04 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:05.450 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:05.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:05.504 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:05.536 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:05.537 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:36:05.537 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:05.556 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:05.566 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 51633 51617 111.215 63.375 0.0149427 0.00898713 2023-10-09T06:36:05.603 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:05.709 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:05.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:05.746 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:36:05.816 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:05.877 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:36:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:36:05.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:05 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:05.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:36:05.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:36:05.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:05.979 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:36:05.979 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:36:05.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:05 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:36:06.251 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:06.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:06.355 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.384 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:06.433 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:06.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:06.537 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:06.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 52704 52688 109.723 66.9375 0.0125906 0.00910824 2023-10-09T06:36:06.587 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:06.663 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:06.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:06.759 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:36:06.768 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:06.808 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:06.900 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[220921]: pgmap v1354: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1810072115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:06.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[212750]: pgmap v1354: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1810072115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:06 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:07.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:06 smithi150 ceph-mon[194178]: pgmap v1354: 119 pgs: 119 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T06:36:07.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1810072115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:07.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:06 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:07.156 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:07.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:07.263 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:07.291 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:07.334 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:07.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:07.439 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:07.493 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:07.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 53882 53866 108.558 73.625 0.0206654 0.00920644 2023-10-09T06:36:07.592 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:07.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:07.701 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:07.764 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:07.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:36:07.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:36:07.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: failed to encode map e445 with expected crc 2023-10-09T06:36:07.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:07.765 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[212750]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T06:36:07.771 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:07.814 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:07.875 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:07.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:07.890 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:07.890 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:36:07.891 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: failed to encode map e445 with expected crc 2023-10-09T06:36:07.891 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: failed to encode map e445 with expected crc 2023-10-09T06:36:07.891 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:07.891 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:07 smithi150 ceph-mon[194178]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T06:36:07.990 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:08.067 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:08.150 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:36:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: failed to encode map e445 with expected crc 2023-10-09T06:36:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: failed to encode map e445 with expected crc 2023-10-09T06:36:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:07 smithi107 ceph-mon[220921]: osdmap e445: 8 total, 8 up, 8 in 2023-10-09T06:36:08.161 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:08.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:08.229 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:08.229 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:08.272 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:08.327 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:08.441 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:08.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:08.547 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:08.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 55262 55246 107.861 86.25 0.0128736 0.00926678 2023-10-09T06:36:08.613 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:08.697 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:08.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:08.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:08.809 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:08.840 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:09.055 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:08 smithi107 ceph-mon[212750]: pgmap v1356: 119 pgs: 119 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:36:09.055 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:08 smithi107 ceph-mon[220921]: pgmap v1356: 119 pgs: 119 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:36:09.094 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:09.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:09.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:08 smithi150 ceph-mon[194178]: pgmap v1356: 119 pgs: 119 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-09T06:36:09.200 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:09.242 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:09.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 56564 56548 107.059 81.375 0.0100076 0.00933601 2023-10-09T06:36:09.645 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:36:09.650 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:09.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:09.758 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:09.774 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:36:09.812 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:09.894 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:09.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:10.003 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:10.053 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:10.155 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:10.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:10.265 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:10.310 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:10.450 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:10.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:10.469 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:10.528 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:10.530 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:10.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:10.563 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:10.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 57833 57817 106.242 79.3125 0.0147344 0.00940714 2023-10-09T06:36:10.605 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:10.614 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:10.636 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:10.672 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:10.715 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:10.722 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:10.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:10.743 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:10.786 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:10.798 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.799 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:10.834 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:10.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:10 smithi150 ceph-mon[194178]: pgmap v1357: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:36:10.882 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1760329985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:10.884 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:10.995 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:11.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:11.063 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:10 smithi107 ceph-mon[212750]: pgmap v1357: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:36:11.063 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1760329985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:11.063 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:10 smithi107 ceph-mon[220921]: pgmap v1357: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T06:36:11.064 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1760329985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:11.104 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:11.179 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:11.571 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 59140 59124 105.541 81.6875 0.0103631 0.00946963 2023-10-09T06:36:11.976 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:11.976 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:11.976 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T06:36:11.977 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:36:11.977 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:36:11.977 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T06:36:12.064 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:12.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:12.170 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:12.198 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:12.331 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:12] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T06:36:12.564 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:12.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 60109 60093 104.292 60.5625 0.0100492 0.00958371 2023-10-09T06:36:12.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:12.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:12.968 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:13.023 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:13.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:12 smithi107 ceph-mon[212750]: pgmap v1358: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:36:13.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:12 smithi107 ceph-mon[220921]: pgmap v1358: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:36:13.113 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:13.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:13.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:12 smithi150 ceph-mon[194178]: pgmap v1358: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T06:36:13.222 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:13.294 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:13.424 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:13.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:13.534 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:13.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 61342 61326 103.556 77.0625 0.0136216 0.00965168 2023-10-09T06:36:13.588 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:13.672 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:13.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:13.777 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:13.844 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125493 11.b 11:d6c54ca9:::benchmark_data_smithi107_418363_object35057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.301556+0000", 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.53891709800001, 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2052812930000001, 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:36:13.845 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125493 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301556+0000", 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301556+0000", 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301556+0000", 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301560+0000", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8149999999999999e-06 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301566+0000", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6900000000000003e-06 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301572+0000", 2023-10-09T06:36:13.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4249999999999997e-06 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301639+0000", 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6853999999999994e-05 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301772+0000", 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013308100000000001 2023-10-09T06:36:13.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1", 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.301907+0000", 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013560499999999999 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.307861+0000", 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059538170000000001 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.311103+0000", 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032419010000000002 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.322755+0000", 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011652216 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.322768+0000", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2418999999999999e-05 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313493+0000", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99072570100000001 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313507+0000", 2023-10-09T06:36:13.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4008e-05 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.506768+0000", 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1932607310000001 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.506838+0000", 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9560000000000005e-05 2023-10-09T06:36:13.853 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125498 11.9 11:9ee9bae5:::benchmark_data_smithi107_418363_object35062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.310864+0000", 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.529609042, 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2007971670000002, 2023-10-09T06:36:13.854 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125498 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310864+0000", 2023-10-09T06:36:13.855 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310864+0000", 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310864+0000", 2023-10-09T06:36:13.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310866+0000", 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5069999999999999e-06 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310873+0000", 2023-10-09T06:36:13.857 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1809999999999998e-06 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.310878+0000", 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4659999999999998e-06 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.311973+0000", 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001094928 2023-10-09T06:36:13.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.312088+0000", 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011521 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.312194+0000", 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106254 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319413+0000", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007218342 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319422+0000", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.02e-06 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.331380+0000", 2023-10-09T06:36:13.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011958148 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.331394+0000", 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3587999999999999e-05 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313689+0000", 2023-10-09T06:36:13.861 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98229566999999995 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313701+0000", 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1548e-05 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.862 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.482102+0000", 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1684013849999999 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.482246+0000", 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014408100000000001 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511584+0000", 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.029337617999999999 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511661+0000", 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7620000000000006e-05 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.864 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125501 11.2 11:48269a44:::benchmark_data_smithi107_418363_object35065:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.319479+0000", 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.52099431900001, 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1754861440000002, 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125501 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.865 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319479+0000", 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319479+0000", 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319478+0000", 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319484+0000", 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5400000000000001e-06 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319495+0000", 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0448999999999999e-05 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319502+0000", 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7789999999999998e-06 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.325794+0000", 2023-10-09T06:36:13.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0062920429999999998 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.325928+0000", 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013442799999999999 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.326557+0000", 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062867200000000004 2023-10-09T06:36:13.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.339555+0000", 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012998572999999999 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.339567+0000", 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1769e-05 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.317722+0000", 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97815534299999995 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.317730+0000", 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9910000000000002e-06 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.480568+0000", 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.16283768 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for missing object", 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.480701+0000", 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013248599999999999 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.872 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.494881+0000", 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014180722999999999 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.494965+0000", 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3928e-05 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.873 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125503 11.b 11:d122422a:::benchmark_data_smithi107_418363_object35067:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.323775+0000", 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.51669873100002, 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.210639102, 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125503 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.874 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323775+0000", 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323775+0000", 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323773+0000", 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323780+0000", 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8530000000000003e-06 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323792+0000", 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2193e-05 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323803+0000", 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0689999999999999e-05 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323853+0000", 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9854000000000001e-05 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.878 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.878 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.323863+0000", 2023-10-09T06:36:13.878 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0611000000000001e-05 2023-10-09T06:36:13.878 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.878 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313532+0000", 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98966878899999999 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313540+0000", 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.225e-06 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.507070+0000", 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1935302080000001 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.507180+0000", 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109207 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:36:13.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.507741+0000", 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00056108999999999998 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.518087+0000", 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010346483 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.525841+0000", 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077538279999999999 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.528930+0000", 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030891109999999999 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534253+0000", 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005322561 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534304+0000", 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1476e-05 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534414+0000", 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010951300000000001 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.884 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125505 11.9 11:97089783:::benchmark_data_smithi107_418363_object35069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.334014+0000", 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.506459484, 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1951911750000002, 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.885 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125505 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334014+0000", 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334014+0000", 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334012+0000", 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334020+0000", 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6580000000000002e-06 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334033+0000", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3311e-05 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.334041+0000", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6799999999999993e-06 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.339169+0000", 2023-10-09T06:36:13.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051279200000000002 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.339179+0000", 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0599e-05 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313726+0000", 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.974546727 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313735+0000", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4859999999999997e-06 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.482478+0000", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1687434459999999 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.482623+0000", 2023-10-09T06:36:13.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000145439 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.483318+0000", 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00069474499999999998 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.501064+0000", 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017746277000000001 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511209+0000", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010144659 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511693+0000", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00048412900000000003 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529035+0000", 2023-10-09T06:36:13.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017342178 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529091+0000", 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5204999999999997e-05 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529205+0000", 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114416 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.893 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125518 11.9 11:955bd713:::benchmark_data_smithi107_418363_object35082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.369317+0000", 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.47115625700002, 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1600846800000002, 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.894 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125518 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369317+0000", 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369317+0000", 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369315+0000", 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369325+0000", 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0689e-05 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369344+0000", 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8359000000000002e-05 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369355+0000", 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1559999999999999e-05 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369409+0000", 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3659000000000002e-05 2023-10-09T06:36:13.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369418+0000", 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8349999999999993e-06 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313756+0000", 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94433848799999998 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313765+0000", 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4030000000000007e-06 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.489568+0000", 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1758035360000001 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.489718+0000", 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000149479 2023-10-09T06:36:13.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.489870+0000", 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000152705 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.504758+0000", 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014887578 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511231+0000", 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064731620000000002 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511775+0000", 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00054418000000000001 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529298+0000", 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017522638 2023-10-09T06:36:13.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529330+0000", 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2344000000000001e-05 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529402+0000", 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1546999999999998e-05 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125520 11.b 11:df0aa113:::benchmark_data_smithi107_418363_object35084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.369395+0000", 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.47107856000002, 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1651943170000001, 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125520 2023-10-09T06:36:13.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369395+0000", 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369395+0000", 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369394+0000", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369397+0000", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3760000000000002e-06 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369402+0000", 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5219999999999998e-06 2023-10-09T06:36:13.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369409+0000", 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6789999999999999e-06 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369449+0000", 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9560000000000001e-05 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.369455+0000", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8360000000000003e-06 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313563+0000", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.94410734299999999 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313571+0000", 2023-10-09T06:36:13.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2050000000000002e-06 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.514349+0000", 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.200778369 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.514485+0000", 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000135919 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.514629+0000", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014330300000000001 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.525828+0000", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011199606000000001 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.525848+0000", 2023-10-09T06:36:13.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9578000000000001e-05 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.528982+0000", 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031342050000000002 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534457+0000", 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054749179999999996 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534499+0000", 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2144999999999997e-05 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534589+0000", 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.009e-05 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125526 11.9 11:9daf46d4:::benchmark_data_smithi107_418363_object35090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.383300+0000", 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.45717294500002, 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.146223193, 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125526 2023-10-09T06:36:13.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383300+0000", 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383300+0000", 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383298+0000", 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383306+0000", 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1829999999999997e-06 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383322+0000", 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5988e-05 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383330+0000", 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3469999999999999e-06 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383386+0000", 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5689000000000002e-05 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.383398+0000", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2604e-05 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313787+0000", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.93038843000000004 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313797+0000", 2023-10-09T06:36:13.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0868e-05 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.501096+0000", 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1872986320000001 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.501272+0000", 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000176162 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.501419+0000", 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014661399999999999 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511731+0000", 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010312508 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.526062+0000", 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014330193999999999 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.526082+0000", 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0077000000000001e-05 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529429+0000", 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003347133 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529456+0000", 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7542000000000002e-05 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529524+0000", 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7235000000000003e-05 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125531 11.9 11:9de447eb:::benchmark_data_smithi107_418363_object35095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.395375+0000", 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.44509805899997, 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1343029389999999, 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125531 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.921 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395375+0000", 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395375+0000", 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395373+0000", 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395380+0000", 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6789999999999999e-06 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395396+0000", 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6339999999999999e-05 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.395405+0000", 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5660000000000007e-06 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.398326+0000", 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002921589 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.398338+0000", 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.163e-05 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313821+0000", 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.91548293599999997 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313829+0000", 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2209999999999997e-06 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.504795+0000", 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1909662029999999 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.504940+0000", 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014455300000000001 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:36:13.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.505097+0000", 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015693 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511738+0000", 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0066413319999999998 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.526099+0000", 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014361319000000001 2023-10-09T06:36:13.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529004+0000", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0029050040000000001 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529571+0000", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00056669200000000004 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529602+0000", 2023-10-09T06:36:13.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0549999999999997e-05 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529678+0000", 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6633999999999998e-05 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125540 11.9 11:95c25dc5:::benchmark_data_smithi107_418363_object35104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.929 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.423619+0000", 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.41685455200002, 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1061959579999998, 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125540 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423619+0000", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423619+0000", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423617+0000", 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:13.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423625+0000", 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3829999999999992e-06 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423643+0000", 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7215999999999999e-05 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423655+0000", 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2281e-05 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423728+0000", 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3169000000000004e-05 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.423743+0000", 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5197999999999999e-05 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313855+0000", 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.89011205699999996 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313864+0000", 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3799999999999994e-06 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.508426+0000", 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1945621909999999 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.508551+0000", 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012481999999999999 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,6", 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.508677+0000", 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012588599999999999 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.511744+0000", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030669019999999998 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.526118+0000", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014374191 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529265+0000", 2023-10-09T06:36:13.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031474559999999999 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529702+0000", 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043677100000000002 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529729+0000", 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.686e-05 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.529815+0000", 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5910999999999998e-05 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125544 11.b 11:dab1e666:::benchmark_data_smithi107_418363_object35108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.441869+0000", 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 430.39860441299999, 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.097239117, 2023-10-09T06:36:13.938 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125544 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441869+0000", 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441869+0000", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441866+0000", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441881+0000", 2023-10-09T06:36:13.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4815e-05 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441909+0000", 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8204e-05 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441923+0000", 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3244999999999999e-05 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441984+0000", 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1586000000000003e-05 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.441995+0000", 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1219999999999999e-05 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313594+0000", 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.87159859399999995 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.313602+0000", 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1319999999999994e-06 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.518114+0000", 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.204512013 2023-10-09T06:36:13.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.518248+0000", 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013366300000000001 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.518385+0000", 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000136685 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.525854+0000", 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074691820000000004 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.534651+0000", 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0087972610000000007 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.536678+0000", 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020267369999999998 2023-10-09T06:36:13.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.538988+0000", 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023107919999999999 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.539030+0000", 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.18e-05 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.539108+0000", 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7751999999999997e-05 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104316 12.12 12:4daa80d6:::benchmark_data_smithi107_479504_object4705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.206939+0000", 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.63353386899999, 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0209618310000002, 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.947 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104316 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206939+0000", 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206939+0000", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206938+0000", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206944+0000", 2023-10-09T06:36:13.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5610000000000004e-06 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206957+0000", 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2843000000000001e-05 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.206963+0000", 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0809999999999999e-06 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.207029+0000", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5559e-05 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.207043+0000", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3884e-05 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042418+0000", 2023-10-09T06:36:13.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83537493500000004 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042429+0000", 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0876000000000001e-05 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.161199+0000", 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.118770547 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.161989+0000", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00078933599999999997 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.216310+0000", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.054321537000000003 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.216430+0000", 2023-10-09T06:36:13.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000120123 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.216795+0000", 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00036496400000000001 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.223781+0000", 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069858070000000001 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.223824+0000", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2846000000000003e-05 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.226431+0000", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026071039999999998 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.226469+0000", 2023-10-09T06:36:13.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7796999999999997e-05 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.227901+0000", 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001432628 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104351 12.12 12:4c7d9407:::benchmark_data_smithi107_479504_object4740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.956 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.268514+0000", 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.57195977000001, 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9721861540000001, 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104351 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268514+0000", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268514+0000", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268512+0000", 2023-10-09T06:36:13.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268520+0000", 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8490000000000006e-06 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268538+0000", 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8331000000000001e-05 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268548+0000", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5810000000000003e-06 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268608+0000", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0123e-05 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.268619+0000", 2023-10-09T06:36:13.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1165000000000001e-05 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042450+0000", 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.77383142400000005 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042458+0000", 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4590000000000001e-06 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.194653+0000", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.152195469 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.195344+0000", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00069115400000000003 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.221664+0000", 2023-10-09T06:36:13.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.026320073999999999 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.221796+0000", 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013107900000000001 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.222191+0000", 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00039588500000000001 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.231179+0000", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0089876220000000007 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.232827+0000", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001647962 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.239573+0000", 2023-10-09T06:36:13.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0067456199999999999 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.239632+0000", 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9638000000000001e-05 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.240700+0000", 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001067464 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.965 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104366 12.12 12:4f916a32:::benchmark_data_smithi107_479504_object4755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.284282+0000", 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.55619160200001, 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.006126198, 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.966 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104366 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284282+0000", 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284282+0000", 2023-10-09T06:36:13.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284280+0000", 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284288+0000", 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.588e-06 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284306+0000", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7875000000000001e-05 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284315+0000", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0110000000000006e-06 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284423+0000", 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108797 2023-10-09T06:36:13.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.284435+0000", 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1266e-05 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042478+0000", 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.75804318900000001 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042485+0000", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.311e-06 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.223855+0000", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1813695129999999 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.224606+0000", 2023-10-09T06:36:13.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075111899999999996 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.246684+0000", 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.022078159 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.246805+0000", 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000121279 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.247186+0000", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00038098500000000003 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.258951+0000", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011764808 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.289019+0000", 2023-10-09T06:36:13.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.030068035 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.289054+0000", 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5299000000000002e-05 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.289092+0000", 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7809999999999999e-05 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.290408+0000", 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013156960000000001 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104384 12.12 12:4abeff6a:::benchmark_data_smithi107_479504_object4773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.309156+0000", 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.53131704399999, 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.982079396, 2023-10-09T06:36:13.975 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104384 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309156+0000", 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309156+0000", 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309155+0000", 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309161+0000", 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7279999999999998e-06 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309176+0000", 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4273e-05 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309183+0000", 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6620000000000005e-06 2023-10-09T06:36:13.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309261+0000", 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7309000000000001e-05 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.309274+0000", 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3665999999999999e-05 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042505+0000", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.73323038799999996 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042512+0000", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1380000000000001e-06 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.224651+0000", 2023-10-09T06:36:13.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1821390460000001 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.225360+0000", 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00070883000000000003 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.250607+0000", 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025246777000000001 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.250701+0000", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4702000000000002e-05 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.251003+0000", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030173599999999999 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.265884+0000", 2023-10-09T06:36:13.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014880526 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.290457+0000", 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024573542 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.290485+0000", 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7506000000000001e-05 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.290519+0000", 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4413999999999999e-05 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.291236+0000", 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00071668700000000003 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104428 12.12 12:49491ef6:::benchmark_data_smithi107_479504_object4817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.356404+0000", 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.48406969199999, 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.935681939, 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104428 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356404+0000", 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356404+0000", 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356402+0000", 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356410+0000", 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4039999999999995e-06 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356427+0000", 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6980999999999999e-05 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356435+0000", 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2199999999999992e-06 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356499+0000", 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3429000000000005e-05 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.356510+0000", 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1212e-05 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042531+0000", 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.68602128200000001 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042539+0000", 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2989999999999999e-06 2023-10-09T06:36:13.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.225426+0000", 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.18288722 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.226403+0000", 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00097704799999999989 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.254853+0000", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.028449957000000001 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.254960+0000", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000106606 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.255251+0000", 2023-10-09T06:36:13.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000291587 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.265891+0000", 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010640119 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.291299+0000", 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025407770999999999 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.291326+0000", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6536000000000001e-05 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.291357+0000", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1259e-05 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.292086+0000", 2023-10-09T06:36:13.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00072877499999999997 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104477 12.12 12:48df3a0f:::benchmark_data_smithi107_479504_object4866:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.432823+0000", 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.407650604, 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8602245289999999, 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:13.994 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104477 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432823+0000", 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:13.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432823+0000", 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432822+0000", 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432827+0000", 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2340000000000002e-06 2023-10-09T06:36:13.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432839+0000", 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1761e-05 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432846+0000", 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2629999999999997e-06 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432894+0000", 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8285000000000003e-05 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.432905+0000", 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0325e-05 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042559+0000", 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.60965405100000003 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042566+0000", 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.255e-06 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.227971+0000", 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1854048669999999 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:13.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.228826+0000", 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00085546699999999995 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.258996+0000", 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.030169799000000001 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.259132+0000", 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013558099999999999 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.259542+0000", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041012200000000001 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.271122+0000", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011579567000000001 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.292133+0000", 2023-10-09T06:36:14.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021011617999999999 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.292172+0000", 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8658999999999997e-05 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.292221+0000", 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8844000000000002e-05 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.293047+0000", 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00082662499999999999 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104518 12.12 12:4d8133d4:::benchmark_data_smithi107_479504_object4907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.486384+0000", 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.35408953699999, 2023-10-09T06:36:14.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8076802519999999, 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104518 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486384+0000", 2023-10-09T06:36:14.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486384+0000", 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486382+0000", 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486390+0000", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1980000000000001e-06 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486407+0000", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6643999999999999e-05 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486416+0000", 2023-10-09T06:36:14.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7609999999999996e-06 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486469+0000", 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3507000000000001e-05 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.486482+0000", 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2418e-05 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042585+0000", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.55610383299999999 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042593+0000", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2540000000000003e-06 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.228869+0000", 2023-10-09T06:36:14.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.186276704 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.229574+0000", 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00070442599999999997 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.264382+0000", 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.034807865 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.264454+0000", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1902000000000004e-05 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.264734+0000", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00028039800000000001 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.275439+0000", 2023-10-09T06:36:14.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010705475000000001 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.293118+0000", 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017678793000000002 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.293191+0000", 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2974999999999998e-05 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.293256+0000", 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4508e-05 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.294064+0000", 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00080838100000000005 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104581 12.12 12:4dc620ab:::benchmark_data_smithi107_479504_object4970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.550025+0000", 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.29044853100001, 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.744959631, 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104581 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:14.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550025+0000", 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550025+0000", 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:14.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550023+0000", 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550030+0000", 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7129999999999999e-06 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550044+0000", 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3749e-05 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550052+0000", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0450000000000001e-06 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550112+0000", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9895999999999997e-05 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.550132+0000", 2023-10-09T06:36:14.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0338000000000001e-05 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042612+0000", 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.49247988999999998 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042619+0000", 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2960000000000001e-06 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.229615+0000", 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.186995805 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.230279+0000", 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00066433599999999996 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.269575+0000", 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.039295923000000003 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.269704+0000", 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012848800000000001 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.270063+0000", 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000358897 2023-10-09T06:36:14.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.279651+0000", 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0095883640000000003 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.294134+0000", 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014483028 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.294186+0000", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2287999999999998e-05 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.294253+0000", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6779999999999994e-05 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.294984+0000", 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00073120399999999995 2023-10-09T06:36:14.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104594 12.12 12:4a31551e:::benchmark_data_smithi107_479504_object4983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e399)", 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:50.567552+0000", 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 203.27292171100001, 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.728392938, 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:36:14.022 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104594 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567552+0000", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567552+0000", 2023-10-09T06:36:14.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567551+0000", 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567555+0000", 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5719999999999999e-06 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567562+0000", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3540000000000002e-06 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567568+0000", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7180000000000001e-06 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567624+0000", 2023-10-09T06:36:14.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6597e-05 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:50.567636+0000", 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1800999999999999e-05 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042639+0000", 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.47500262100000001 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:51.042646+0000", 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1729999999999998e-06 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.230314+0000", 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.187667625 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:14.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.231117+0000", 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00080355400000000005 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.273934+0000", 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.042817409000000001 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.274030+0000", 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5466999999999999e-05 2023-10-09T06:36:14.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5", 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.274338+0000", 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030792699999999999 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.279663+0000", 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053250169999999996 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.295050+0000", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015386907999999999 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.295103+0000", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3128999999999999e-05 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.295153+0000", 2023-10-09T06:36:14.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0024000000000001e-05 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.295945+0000", 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00079161800000000001 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:36:14.031 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:36:14.032 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:14.048 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:14.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:14.156 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.193 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:14.266 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:14.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:14.371 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:14.443 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:14.538 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:14.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:14.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 62600 62584 102.9 78.625 0.0215224 0.00971339 2023-10-09T06:36:14.648 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:14.702 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:14.810 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:14.812 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:36:14.819 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:36:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:14.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:14.866 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:36:14.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:14.976 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:15.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:15.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[220921]: pgmap v1359: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:15.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/702455706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:15.061 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[212750]: pgmap v1359: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:15.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/702455706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:15.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:14 smithi150 ceph-mon[194178]: pgmap v1359: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:15.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/702455706' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.162 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:15.163 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:15.256 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:15.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 63783 63767 102.157 73.9375 0.00611636 0.00978436 2023-10-09T06:36:15.748 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:15.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:15.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[212750]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:15.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:15 smithi107 ceph-mon[220921]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.093 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: osdmap e446: 8 total, 8 up, 8 in 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.098 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:15 smithi150 ceph-mon[194178]: failed to encode map e446 with expected crc 2023-10-09T06:36:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:16.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:16.200 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:16.248 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:16.366 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:16.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:16.449 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:16.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:16.511 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:16.568 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:16.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:36:16.572945+0000 min lat: 0.00116741 max lat: 0.272235 avg lat: 0.00985059 2023-10-09T06:36:16.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:36:16.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 64980 64964 101.473 74.8125 0.0127725 0.00985059 2023-10-09T06:36:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:16.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:16.675 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:16.709 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:16.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:16 smithi107 ceph-mon[212750]: pgmap v1361: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:36:16.888 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:16.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:16 smithi107 ceph-mon[220921]: pgmap v1361: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:36:16.888 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:16.921 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:16.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:16.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:17.028 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:16 smithi150 ceph-mon[194178]: pgmap v1361: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T06:36:17.101 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:17.107 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:17.177 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:17.177 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "" 2023-10-09T06:36:17.177 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:17.224 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:17.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:17.333 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:17.384 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:17.456 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:17.467 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:17.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:17.485 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:17.514 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:17.541 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:17.571 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:17.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 66100 66084 100.706 70 0.00601957 0.00992565 2023-10-09T06:36:17.575 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:17.617 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:17.635 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:36:17.642 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:17.654 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:17.690 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:17.700 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.701 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:17.796 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:17.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:17.915 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:17.980 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:18.035 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:18.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:18.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:18.138 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1084661596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1084661596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:18.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1084661596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:18.164 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:18.205 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:18.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:18.308 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:18.359 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:18.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 66806 66790 99.3587 44.125 0.00462895 0.0100433 2023-10-09T06:36:18.672 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:18.848 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:18 smithi107 ceph-mon[220921]: pgmap v1362: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:19.083 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:19.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:18 smithi107 ceph-mon[212750]: pgmap v1362: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:18 smithi150 ceph-mon[194178]: pgmap v1362: 119 pgs: 119 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:36:19.191 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:19.270 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:19.386 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:19.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:19.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:19.489 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:19.545 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:19.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 67727 67711 98.3868 57.5625 0.0205998 0.0101585 2023-10-09T06:36:19.790 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:19.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:19.869 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:19.869 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:19.906 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:20.002 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:20.104 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:20.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:20.221 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:20.292 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:20.374 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:20.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:20.478 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:20.509 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:20.551 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:36:20.556 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:20.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:20.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:20.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 68634 68618 97.4392 56.6875 0.00813446 0.0102066 2023-10-09T06:36:20.660 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:20.719 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:20.828 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:20.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:20.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:20.882 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:36:20.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:20.983 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:21.054 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:21.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:21.066 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:36:21.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:20 smithi150 ceph-mon[194178]: pgmap v1363: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:36:21.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3602521185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.142 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:20 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[220921]: pgmap v1363: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:36:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3602521185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[212750]: pgmap v1363: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T06:36:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3602521185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:20 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:21.159 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:21.190 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:21.233 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:21.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:21.336 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:21.365 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:21.429 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:21.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:21.537 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:21.572 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 69514 69498 96.4962 55 0.00648587 0.0103576 2023-10-09T06:36:21.576 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:21.659 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:21.773 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:21.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:21.848 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:21.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:21.963 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:22.039 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:22.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:22.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:22] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[212750]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:21 smithi107 ceph-mon[220921]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:22.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: osdmap e447: 8 total, 8 up, 8 in 2023-10-09T06:36:22.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:21 smithi150 ceph-mon[194178]: failed to encode map e447 with expected crc 2023-10-09T06:36:22.157 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:22.247 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:22.479 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:22.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:22.573 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 70522 70506 95.7681 63 0.00598884 0.0104144 2023-10-09T06:36:22.582 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:22.609 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:22.653 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:22.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:22.756 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:22.781 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:22.822 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:22.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:22.925 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:22.971 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:23.062 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:23.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:22 smithi107 ceph-mon[212750]: pgmap v1365: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:36:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:22 smithi107 ceph-mon[220921]: pgmap v1365: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:36:23.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:22 smithi150 ceph-mon[194178]: pgmap v1365: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:36:23.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:23.232 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:23.400 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:23.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:23.519 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:23.569 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:23.573 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 71346 71330 94.8262 51.5 0.00905331 0.0105396 2023-10-09T06:36:23.634 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:23.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:23.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:23.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:23.669 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:23.695 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:23.729 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:23.761 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:23.762 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:23.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:36:23.795 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:23.814 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:23.841 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:23.885 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:23.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:23.891 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.891 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.891 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.892 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.892 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.892 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.892 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.892 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:23.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:23.914 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4086446932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:23.995 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:24.031 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:24.031 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:24.031 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T06:36:24.031 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:36:24.031 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:36:24.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T06:36:24.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4086446932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:24.090 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:24.126 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:24.126 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:36:24.126 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:36:24.126 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:36:24.126 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:24.367 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:24.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4086446932' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:24.472 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:24.517 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:24.573 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 72208 72192 93.973 53.875 0.00949955 0.0106373 2023-10-09T06:36:24.726 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:24.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:24.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:24.831 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:24.878 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:24.962 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:24.963 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:24.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:25.067 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:25.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:24 smithi107 ceph-mon[212750]: pgmap v1366: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:36:25.106 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:24 smithi107 ceph-mon[220921]: pgmap v1366: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:36:25.112 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:25.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:24 smithi150 ceph-mon[194178]: pgmap v1366: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T06:36:25.232 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:25.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:25.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:25.264 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:36:25.337 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:25.442 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:25.626 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:25.627 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:25.627 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 73282 73266 93.4214 67.125 0.01823 0.0106997 2023-10-09T06:36:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:25.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:25.730 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:25.882 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:26.196 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:26.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:26.303 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:26.388 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:26.505 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:26.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:26.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 74145 74129 92.6318 53.9375 0.00338697 0.0107808 2023-10-09T06:36:26.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:26.639 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:26.680 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:26.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:26.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:26.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:36:26.843 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:26.943 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:26.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:27.058 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:27.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:26 smithi107 ceph-mon[212750]: pgmap v1367: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:27.099 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:26 smithi107 ceph-mon[220921]: pgmap v1367: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:27.100 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:27.147 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:27.176 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:27.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:27.282 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:27.341 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:26 smithi150 ceph-mon[194178]: pgmap v1367: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:27.350 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:27.387 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:27.523 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:27.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:27.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 75069 75053 91.9479 57.75 0.00853337 0.010871 2023-10-09T06:36:27.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:27.670 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:27.734 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:27.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:27.839 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:27.903 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:27.923 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:27.942 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:36:28.001 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:28.001 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "" 2023-10-09T06:36:28.001 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:28.049 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:28.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:28.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:28.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:28.230 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:28.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:28.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 76005 75989 91.3047 58.5 0.0978714 0.010939 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: pgmap v1368: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[212750]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: pgmap v1368: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:28 smithi107 ceph-mon[220921]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.194 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:29.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: pgmap v1368: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T06:36:29.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:29.196 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/285963845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:29.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: osdmap e448: 8 total, 8 up, 8 in 2023-10-09T06:36:29.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.197 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:28 smithi150 ceph-mon[194178]: failed to encode map e448 with expected crc 2023-10-09T06:36:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:29.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:29.300 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:29.354 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:29.474 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:29.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:29.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:29.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 76822 76806 90.5455 51.0625 0.00954942 0.0110385 2023-10-09T06:36:29.578 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:29.673 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:29.830 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:29.843 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:29.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:29.862 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:29.890 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:29.914 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:29.947 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:29.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:29.974 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:30.013 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:30.014 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:30.061 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:30.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.062 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.063 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:30.149 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:30.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1036957087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:30.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:30.172 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:30.172 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:30.225 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1036957087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:30.226 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1036957087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:30.254 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:30.330 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:30.435 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:30.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:30.542 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:30.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 77703 77687 89.8885 55.0625 0.00420944 0.0111145 2023-10-09T06:36:30.583 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:30.627 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:30.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:30.731 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:30.759 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:30.918 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:30.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:31.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:31.127 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:31.131 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:36:31.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:30 smithi107 ceph-mon[212750]: pgmap v1370: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:36:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:30 smithi107 ceph-mon[220921]: pgmap v1370: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:36:31.298 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:31.313 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:30 smithi150 ceph-mon[194178]: pgmap v1370: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T06:36:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:31.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:31.415 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:31.420 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:36:31.451 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:31.537 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:31.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:31.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 78562 78546 89.2303 53.6875 0.016439 0.0112021 2023-10-09T06:36:31.650 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:31.697 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:31.777 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:31.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:31.807 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:31.888 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:31.932 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:32.037 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:32.063 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:32.075 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:32.075 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:32] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T06:36:32.076 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:32.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:32.177 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:32.263 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:32.268 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:32.536 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:32.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:32.575 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 79582 79566 88.7753 63.75 0.00785073 0.011259 2023-10-09T06:36:32.645 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:32.727 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:32.829 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:32.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:32.923 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:36:32.936 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:33.008 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:33.099 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:33.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:32 smithi150 ceph-mon[194178]: pgmap v1371: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T06:36:33.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:33.205 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:33.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:32 smithi107 ceph-mon[212750]: pgmap v1371: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T06:36:33.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:32 smithi107 ceph-mon[220921]: pgmap v1371: 119 pgs: 119 active+clean; 4.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T06:36:33.263 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:33.396 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:33.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:33.513 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 80622 80606 88.3582 65 0.00489395 0.0113121 2023-10-09T06:36:33.602 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:33.609 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:33.690 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:33.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:33.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:33.805 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:33.870 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:33.978 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:33.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:34.084 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:34.142 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:34.278 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:34.278 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:34.382 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:34.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:34.487 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:34.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 81614 81598 87.9037 62 0.01552 0.0113717 2023-10-09T06:36:34.584 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:34.629 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:34.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:34.640 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:34.640 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:34.640 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T06:36:34.640 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:36:34.641 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:36:34.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T06:36:34.735 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:34.774 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:34.817 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:34.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:34.974 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[212750]: pgmap v1372: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T06:36:34.974 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/491955496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:34.974 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:34.980 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:36:35.032 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:35.112 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:35.228 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[220921]: pgmap v1372: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T06:36:35.229 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/491955496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:35.229 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:34 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:35.383 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:35.394 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:34 smithi150 ceph-mon[194178]: pgmap v1372: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T06:36:35.394 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/491955496' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:35.394 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:34 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:35.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:35.407 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:36:35.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:35.538 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:35.549 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:35.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 82248 82232 87.0856 39.625 0.0235624 0.0114778 2023-10-09T06:36:35.720 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:35.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:35.826 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:35.873 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:36.051 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:36.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:36.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:36.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:36:36.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:36.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T06:36:36.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[212750]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:36:36.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:36.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T06:36:36.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:35 smithi107 ceph-mon[220921]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.156 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:36.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:36:36.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:36:36.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:36.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: osdmap e449: 8 total, 8 up, 8 in 2023-10-09T06:36:36.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.158 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:35 smithi150 ceph-mon[194178]: failed to encode map e449 with expected crc 2023-10-09T06:36:36.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.183 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:36.212 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:36.240 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:36.241 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:36.271 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:36.328 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:36.374 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:36.374 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:36.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:36.408 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.412 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.413 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.413 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:36.486 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:36.562 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:36.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:36:36.576748+0000 min lat: 0.00116741 max lat: 0.348861 avg lat: 0.0115389 2023-10-09T06:36:36.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:36:36.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 16 83204 83188 86.63 59.75 0.0309504 0.0115389 2023-10-09T06:36:36.620 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:36.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:36.724 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:36.753 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:36.825 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:36.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:36.932 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:36.978 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:37.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:36 smithi107 ceph-mon[212750]: pgmap v1374: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:36:37.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/22767732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:36 smithi107 ceph-mon[220921]: pgmap v1374: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:36:37.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/22767732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:37.196 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:37.241 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:37.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:37.347 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:37.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:36 smithi150 ceph-mon[194178]: pgmap v1374: 119 pgs: 119 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:36:37.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/22767732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:37.413 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:37.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 84036 84020 86.0623 52 0.0114702 0.0116138 2023-10-09T06:36:37.662 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:37.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:37.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:37.729 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:36:37.770 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:37.837 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:37.936 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:37.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:38.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:38.052 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:38.052 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T06:36:38.052 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:38.117 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:38.221 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:38.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 84885 84869 85.53 53.0625 0.00710016 0.0116768 2023-10-09T06:36:38.985 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:38.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:39.097 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:39.122 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:39 smithi107 ceph-mon[212750]: pgmap v1375: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s 2023-10-09T06:36:39.123 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:39 smithi107 ceph-mon[220921]: pgmap v1375: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s 2023-10-09T06:36:39.126 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:39.179 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:39.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:39.242 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:36:39.283 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:39.316 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:39 smithi150 ceph-mon[194178]: pgmap v1375: 119 pgs: 119 active+clean; 4.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s 2023-10-09T06:36:39.350 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:39.549 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:39.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:39.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 85768 85752 85.0484 55.1875 0.0253693 0.0117521 2023-10-09T06:36:39.658 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:39.735 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:39.818 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:39.830 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:39.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:39.941 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:40.030 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:40.068 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:40.069 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:36:40.069 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:36:40.069 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:36:40.069 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:40.310 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:40.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:40.416 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:40.484 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:40.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 86717 86701 84.6463 59.3125 0.0102656 0.0118092 2023-10-09T06:36:40.597 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:40.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:40.707 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:40.737 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:40.781 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:40.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:40.887 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:40.947 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:36:40.976 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:41.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:41 smithi107 ceph-mon[212750]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:36:41.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:41.050 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:36:41.148 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:41.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:41.263 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:41.295 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:41 smithi107 ceph-mon[220921]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:36:41.295 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:41.296 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:41.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:41 smithi150 ceph-mon[194178]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-10-09T06:36:41.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:36:41.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 87443 87427 84.0422 45.375 0.0187548 0.011893 2023-10-09T06:36:41.638 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:41.855 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:41.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:41.970 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:42.034 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T06:36:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[212750]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:42 smithi107 ceph-mon[220921]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.155 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:42] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-10-09T06:36:42.177 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:42.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:42.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:42.281 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:42.285 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:42.313 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:42.339 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:42.366 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2802572744' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: osdmap e450: 8 total, 8 up, 8 in 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.367 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.368 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.368 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.368 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.368 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.368 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:42 smithi150 ceph-mon[194178]: failed to encode map e450 with expected crc 2023-10-09T06:36:42.383 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:42.393 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:42.425 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:42.462 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:42.493 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.496 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.497 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.497 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:42.509 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:42.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:42.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:42.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 15 88404 88389 83.6797 60.125 0.0180538 0.0119452 2023-10-09T06:36:42.622 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:42.675 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:42.745 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:42.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:42.852 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:42.878 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:42.919 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:42.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:42.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:43.023 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:43.065 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:43.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:43 smithi107 ceph-mon[220921]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-10-09T06:36:43.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2654005278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:43.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:43 smithi107 ceph-mon[212750]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-10-09T06:36:43.157 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2654005278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:43.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:43 smithi150 ceph-mon[194178]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 590 op/s 2023-10-09T06:36:43.332 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2654005278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:43.390 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:43.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:43.501 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:43.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 89201 89185 83.1733 49.75 0.0318342 0.0120176 2023-10-09T06:36:43.593 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:43.779 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:43.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:43.870 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:36:43.886 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:43.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:44.071 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:44.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:44.112 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:44.113 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:44.186 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:44.255 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:44.327 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:44.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:44.433 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:44.448 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:44.475 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:44.572 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:44.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:44.576 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 90119 90103 82.7939 57.375 0.00798496 0.0120744 2023-10-09T06:36:44.581 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:36:44.581 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:36:44.581 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-10-09T06:36:44.581 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:36:44.677 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:36:44.733 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:45.006 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:36:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:45.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:36:45.108 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:36:45.114 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:45.178 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:45 smithi107 ceph-mon[212750]: pgmap v1379: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:45.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:45 smithi107 ceph-mon[220921]: pgmap v1379: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:45.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:45 smithi150 ceph-mon[194178]: pgmap v1379: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T06:36:45.406 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:36:45.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 90873 90857 82.277 47.125 0.00963985 0.0121488 2023-10-09T06:36:45.630 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:45.741 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:45.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:45.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:45.926 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:46.007 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:46.026 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:46.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:46.119 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:46.137 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:46.195 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:46.292 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:46.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:46.398 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:46.486 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:46.575 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:46.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 91781 91765 81.9123 56.75 0.0292782 0.0122034 2023-10-09T06:36:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:46.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:46.681 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:46.707 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:46.748 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:46.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:46.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:46.926 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:47.085 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:47 smithi150 ceph-mon[194178]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T06:36:47.086 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:47.086 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:47.103 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:36:47.139 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:47.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:47.256 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:47.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:47.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[212750]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T06:36:47.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:47.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[220921]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2023-10-09T06:36:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:36:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:36:47.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 92536 92520 81.4233 47.1875 0.00523741 0.012277 2023-10-09T06:36:47.900 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:47.900 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:48.095 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:48.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:48.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T06:36:48.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[212750]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T06:36:48.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:48 smithi107 ceph-mon[220921]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.134 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:48.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:48.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:48.242 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2570859208' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: osdmap e451: 8 total, 8 up, 8 in 2023-10-09T06:36:48.292 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:48 smithi150 ceph-mon[194178]: failed to encode map e451 with expected crc 2023-10-09T06:36:48.297 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:48.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:48.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:48.480 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:48.506 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:48.548 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:48.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:48.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 93234 93218 80.8983 43.625 0.0159582 0.0123475 2023-10-09T06:36:48.651 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:48.719 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:48.884 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:48.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:48.930 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:36:49.007 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:49.096 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:49 smithi150 ceph-mon[194178]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 576 op/s 2023-10-09T06:36:49.221 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:49.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:49.288 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:49 smithi107 ceph-mon[212750]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 576 op/s 2023-10-09T06:36:49.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:49 smithi107 ceph-mon[220921]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 576 op/s 2023-10-09T06:36:49.340 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:49.425 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:49.518 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:49.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:49.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:49.578 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:36:49.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 94100 94084 80.5312 54.125 0.0185111 0.0124121 2023-10-09T06:36:49.588 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:36:49.589 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:49.623 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:49.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:49.648 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:49.677 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:49.680 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:49.685 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:49.734 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:49.776 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:49.805 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:49.820 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:49.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:49.844 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:49.849 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.849 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.849 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.849 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.849 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.850 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.850 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.850 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:36:49.925 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:49.956 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:50.003 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:50.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:50.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:50.107 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:50.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/33428124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/33428124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:50.195 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:50.362 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/33428124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:50.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:50.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:50.529 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:50.569 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:50.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 94830 94814 80.0595 45.625 0.010799 0.0124855 2023-10-09T06:36:50.647 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:36:50.790 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:50.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:50.895 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:50.911 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1383,"stamp":"2023-10-09T06:36:49.452065+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6059530144,"num_objects":92476,"num_object_clones":0,"num_object_copies":277428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92513,"num_write_kb":5916957,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97498,"ondisk_log_size":97498,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24913108,"kb_used_data":17688724,"kb_used_omap":20,"kb_used_meta":7224299,"kb_avail":725048108,"statfs":{"total":767960285184,"available":742449262592,"internally_reserved":0,"allocated":18113253376,"data_stored":18101066415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7397682830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":558694400,"num_objects":8525,"num_object_clones":0,"num_object_copies":25575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8525,"num_write_kb":545600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.416776"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":825,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.051395+0000","last_change":"2023-10-09T06:35:30.160523+0000","last_active":"2023-10-09T06:36:41.051395+0000","last_peered":"2023-10-09T06:36:41.051395+0000","last_clean":"2023-10-09T06:36:41.051395+0000","last_became_active":"2023-10-09T06:32:19.752697+0000","last_became_peered":"2023-10-09T06:32:19.752697+0000","last_unstale":"2023-10-09T06:36:41.051395+0000","last_undegraded":"2023-10-09T06:36:41.051395+0000","last_fullsized":"2023-10-09T06:36:41.051395+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:30.160439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:30.160439+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:36.130795+0000","scrub_duration":0.002097706,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":430,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.054299+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:36:41.054299+0000","last_peered":"2023-10-09T06:36:41.054299+0000","last_clean":"2023-10-09T06:36:41.054299+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:36:41.054299+0000","last_undegraded":"2023-10-09T06:36:41.054299+0000","last_fullsized":"2023-10-09T06:36:41.054299+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045666+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:36:41.045666+0000","last_peered":"2023-10-09T06:36:41.045666+0000","last_clean":"2023-10-09T06:36:41.045666+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:36:41.045666+0000","last_undegraded":"2023-10-09T06:36:41.045666+0000","last_fullsized":"2023-10-09T06:36:41.045666+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":782,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.056724+0000","last_change":"2023-10-09T06:35:03.005446+0000","last_active":"2023-10-09T06:36:41.056724+0000","last_peered":"2023-10-09T06:36:41.056724+0000","last_clean":"2023-10-09T06:36:41.056724+0000","last_became_active":"2023-10-09T06:32:19.756882+0000","last_became_peered":"2023-10-09T06:32:19.756882+0000","last_unstale":"2023-10-09T06:36:41.056724+0000","last_undegraded":"2023-10-09T06:36:41.056724+0000","last_fullsized":"2023-10-09T06:36:41.056724+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:33:06.799059+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":445,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.038660+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:36:41.038660+0000","last_peered":"2023-10-09T06:36:41.038660+0000","last_clean":"2023-10-09T06:36:41.038660+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:36:41.038660+0000","last_undegraded":"2023-10-09T06:36:41.038660+0000","last_fullsized":"2023-10-09T06:36:41.038660+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":449,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972318+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:36:34.972318+0000","last_peered":"2023-10-09T06:36:34.972318+0000","last_clean":"2023-10-09T06:36:34.972318+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:36:34.972318+0000","last_undegraded":"2023-10-09T06:36:34.972318+0000","last_fullsized":"2023-10-09T06:36:34.972318+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":698,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.054214+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:36:41.054214+0000","last_peered":"2023-10-09T06:36:41.054214+0000","last_clean":"2023-10-09T06:36:41.054214+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:36:41.054214+0000","last_undegraded":"2023-10-09T06:36:41.054214+0000","last_fullsized":"2023-10-09T06:36:41.054214+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":169,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044163+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:36:41.044163+0000","last_peered":"2023-10-09T06:36:41.044163+0000","last_clean":"2023-10-09T06:36:41.044163+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:36:41.044163+0000","last_undegraded":"2023-10-09T06:36:41.044163+0000","last_fullsized":"2023-10-09T06:36:41.044163+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":440,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.062366+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:36:41.062366+0000","last_peered":"2023-10-09T06:36:41.062366+0000","last_clean":"2023-10-09T06:36:41.062366+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:36:41.062366+0000","last_undegraded":"2023-10-09T06:36:41.062366+0000","last_fullsized":"2023-10-09T06:36:41.062366+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":559,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.065410+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:36:41.065410+0000","last_peered":"2023-10-09T06:36:41.065410+0000","last_clean":"2023-10-09T06:36:41.065410+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:36:41.065410+0000","last_undegraded":"2023-10-09T06:36:41.065410+0000","last_fullsized":"2023-10-09T06:36:41.065410+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":124,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972691+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:36:34.972691+0000","last_peered":"2023-10-09T06:36:34.972691+0000","last_clean":"2023-10-09T06:36:34.972691+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:36:34.972691+0000","last_undegraded":"2023-10-09T06:36:34.972691+0000","last_fullsized":"2023-10-09T06:36:34.972691+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044658+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:36:41.044658+0000","last_peered":"2023-10-09T06:36:41.044658+0000","last_clean":"2023-10-09T06:36:41.044658+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:36:41.044658+0000","last_undegraded":"2023-10-09T06:36:41.044658+0000","last_fullsized":"2023-10-09T06:36:41.044658+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":596,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.066274+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:36:41.066274+0000","last_peered":"2023-10-09T06:36:41.066274+0000","last_clean":"2023-10-09T06:36:41.066274+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:36:41.066274+0000","last_undegraded":"2023-10-09T06:36:41.066274+0000","last_fullsized":"2023-10-09T06:36:41.066274+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.058082+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:36:41.058082+0000","last_peered":"2023-10-09T06:36:41.058082+0000","last_clean":"2023-10-09T06:36:41.058082+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:36:41.058082+0000","last_undegraded":"2023-10-09T06:36:41.058082+0000","last_fullsized":"2023-10-09T06:36:41.058082+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":151,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044717+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:36:41.044717+0000","last_peered":"2023-10-09T06:36:41.044717+0000","last_clean":"2023-10-09T06:36:41.044717+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:36:41.044717+0000","last_undegraded":"2023-10-09T06:36:41.044717+0000","last_fullsized":"2023-10-09T06:36:41.044717+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":659,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.056536+0000","last_change":"2023-10-09T06:35:31.210236+0000","last_active":"2023-10-09T06:36:41.056536+0000","last_peered":"2023-10-09T06:36:41.056536+0000","last_clean":"2023-10-09T06:36:41.056536+0000","last_became_active":"2023-10-09T06:32:19.750847+0000","last_became_peered":"2023-10-09T06:32:19.750847+0000","last_unstale":"2023-10-09T06:36:41.056536+0000","last_undegraded":"2023-10-09T06:36:41.056536+0000","last_fullsized":"2023-10-09T06:36:41.056536+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:31.210137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:31.210137+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:04.153669+0000","scrub_duration":0.0017668409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":332,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.067021+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:36:41.067021+0000","last_peered":"2023-10-09T06:36:41.067021+0000","last_clean":"2023-10-09T06:36:41.067021+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:36:41.067021+0000","last_undegraded":"2023-10-09T06:36:41.067021+0000","last_fullsized":"2023-10-09T06:36:41.067021+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":387,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039928+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:36:41.039928+0000","last_peered":"2023-10-09T06:36:41.039928+0000","last_clean":"2023-10-09T06:36:41.039928+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:36:41.039928+0000","last_undegraded":"2023-10-09T06:36:41.039928+0000","last_fullsized":"2023-10-09T06:36:41.039928+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":558,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039084+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:36:41.039084+0000","last_peered":"2023-10-09T06:36:41.039084+0000","last_clean":"2023-10-09T06:36:41.039084+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:36:41.039084+0000","last_undegraded":"2023-10-09T06:36:41.039084+0000","last_fullsized":"2023-10-09T06:36:41.039084+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":457,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053203+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:36:41.053203+0000","last_peered":"2023-10-09T06:36:41.053203+0000","last_clean":"2023-10-09T06:36:41.053203+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:36:41.053203+0000","last_undegraded":"2023-10-09T06:36:41.053203+0000","last_fullsized":"2023-10-09T06:36:41.053203+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":421,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.054123+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:36:41.054123+0000","last_peered":"2023-10-09T06:36:41.054123+0000","last_clean":"2023-10-09T06:36:41.054123+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:36:41.054123+0000","last_undegraded":"2023-10-09T06:36:41.054123+0000","last_fullsized":"2023-10-09T06:36:41.054123+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":722,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039814+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:36:41.039814+0000","last_peered":"2023-10-09T06:36:41.039814+0000","last_clean":"2023-10-09T06:36:41.039814+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:36:41.039814+0000","last_undegraded":"2023-10-09T06:36:41.039814+0000","last_fullsized":"2023-10-09T06:36:41.039814+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":300,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.065887+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:36:41.065887+0000","last_peered":"2023-10-09T06:36:41.065887+0000","last_clean":"2023-10-09T06:36:41.065887+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:36:41.065887+0000","last_undegraded":"2023-10-09T06:36:41.065887+0000","last_fullsized":"2023-10-09T06:36:41.065887+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045261+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:36:41.045261+0000","last_peered":"2023-10-09T06:36:41.045261+0000","last_clean":"2023-10-09T06:36:41.045261+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:36:41.045261+0000","last_undegraded":"2023-10-09T06:36:41.045261+0000","last_fullsized":"2023-10-09T06:36:41.045261+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":565,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.051395+0000","last_change":"2023-10-09T06:32:16.207118+0000","last_active":"2023-10-09T06:36:41.051395+0000","last_peered":"2023-10-09T06:36:41.051395+0000","last_clean":"2023-10-09T06:36:41.051395+0000","last_became_active":"2023-10-09T06:32:03.407412+0000","last_became_peered":"2023-10-09T06:32:03.407412+0000","last_unstale":"2023-10-09T06:36:41.051395+0000","last_undegraded":"2023-10-09T06:36:41.051395+0000","last_fullsized":"2023-10-09T06:36:41.051395+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.666290+0000","scrub_duration":0.0014177409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.971950+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:36:34.971950+0000","last_peered":"2023-10-09T06:36:34.971950+0000","last_clean":"2023-10-09T06:36:34.971950+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:36:34.971950+0000","last_undegraded":"2023-10-09T06:36:34.971950+0000","last_fullsized":"2023-10-09T06:36:34.971950+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057275+0000","last_change":"2023-10-09T06:28:11.704672+0000","last_active":"2023-10-09T06:36:41.057275+0000","last_peered":"2023-10-09T06:36:41.057275+0000","last_clean":"2023-10-09T06:36:41.057275+0000","last_became_active":"2023-10-09T06:28:11.704354+0000","last_became_peered":"2023-10-09T06:28:11.704354+0000","last_unstale":"2023-10-09T06:36:41.057275+0000","last_undegraded":"2023-10-09T06:36:41.057275+0000","last_fullsized":"2023-10-09T06:36:41.057275+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:56.783885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":777,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.038766+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:36:41.038766+0000","last_peered":"2023-10-09T06:36:41.038766+0000","last_clean":"2023-10-09T06:36:41.038766+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:36:41.038766+0000","last_undegraded":"2023-10-09T06:36:41.038766+0000","last_fullsized":"2023-10-09T06:36:41.038766+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":294,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.068942+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:36:41.068942+0000","last_peered":"2023-10-09T06:36:41.068942+0000","last_clean":"2023-10-09T06:36:41.068942+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:36:41.068942+0000","last_undegraded":"2023-10-09T06:36:41.068942+0000","last_fullsized":"2023-10-09T06:36:41.068942+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":124,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972098+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:36:34.972098+0000","last_peered":"2023-10-09T06:36:34.972098+0000","last_clean":"2023-10-09T06:36:34.972098+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:36:34.972098+0000","last_undegraded":"2023-10-09T06:36:34.972098+0000","last_fullsized":"2023-10-09T06:36:34.972098+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":565,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057848+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:36:41.057848+0000","last_peered":"2023-10-09T06:36:41.057848+0000","last_clean":"2023-10-09T06:36:41.057848+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:36:41.057848+0000","last_undegraded":"2023-10-09T06:36:41.057848+0000","last_fullsized":"2023-10-09T06:36:41.057848+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":529,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.059375+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:36:41.059375+0000","last_peered":"2023-10-09T06:36:41.059375+0000","last_clean":"2023-10-09T06:36:41.059375+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:36:41.059375+0000","last_undegraded":"2023-10-09T06:36:41.059375+0000","last_fullsized":"2023-10-09T06:36:41.059375+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045835+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:36:41.045835+0000","last_peered":"2023-10-09T06:36:41.045835+0000","last_clean":"2023-10-09T06:36:41.045835+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:36:41.045835+0000","last_undegraded":"2023-10-09T06:36:41.045835+0000","last_fullsized":"2023-10-09T06:36:41.045835+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"451'5763","reported_seq":5785,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.565543+0000","last_change":"2023-10-09T06:35:28.749051+0000","last_active":"2023-10-09T06:36:47.565543+0000","last_peered":"2023-10-09T06:36:47.565543+0000","last_clean":"2023-10-09T06:36:47.565543+0000","last_became_active":"2023-10-09T06:35:28.748772+0000","last_became_peered":"2023-10-09T06:35:28.748772+0000","last_unstale":"2023-10-09T06:36:47.565543+0000","last_undegraded":"2023-10-09T06:36:47.565543+0000","last_fullsized":"2023-10-09T06:36:47.565543+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5763,"log_dups_size":0,"ondisk_log_size":5763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:18:58.923271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377683968,"num_objects":5763,"num_object_clones":0,"num_object_copies":17289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5763,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":578,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.066640+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:36:41.066640+0000","last_peered":"2023-10-09T06:36:41.066640+0000","last_clean":"2023-10-09T06:36:41.066640+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:36:41.066640+0000","last_undegraded":"2023-10-09T06:36:41.066640+0000","last_fullsized":"2023-10-09T06:36:41.066640+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":336,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.067361+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:36:41.067361+0000","last_peered":"2023-10-09T06:36:41.067361+0000","last_clean":"2023-10-09T06:36:41.067361+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:36:41.067361+0000","last_undegraded":"2023-10-09T06:36:41.067361+0000","last_fullsized":"2023-10-09T06:36:41.067361+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.051576+0000","last_change":"2023-10-09T06:28:11.700289+0000","last_active":"2023-10-09T06:36:41.051576+0000","last_peered":"2023-10-09T06:36:41.051576+0000","last_clean":"2023-10-09T06:36:41.051576+0000","last_became_active":"2023-10-09T06:28:11.699646+0000","last_became_peered":"2023-10-09T06:28:11.699646+0000","last_unstale":"2023-10-09T06:36:41.051576+0000","last_undegraded":"2023-10-09T06:36:41.051576+0000","last_fullsized":"2023-10-09T06:36:41.051576+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:55:35.657875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"451'5798","reported_seq":5820,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.454240+0000","last_change":"2023-10-09T06:35:28.749790+0000","last_active":"2023-10-09T06:36:47.454240+0000","last_peered":"2023-10-09T06:36:47.454240+0000","last_clean":"2023-10-09T06:36:47.454240+0000","last_became_active":"2023-10-09T06:35:28.748466+0000","last_became_peered":"2023-10-09T06:35:28.748466+0000","last_unstale":"2023-10-09T06:36:47.454240+0000","last_undegraded":"2023-10-09T06:36:47.454240+0000","last_fullsized":"2023-10-09T06:36:47.454240+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:34.659448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379977728,"num_objects":5798,"num_object_clones":0,"num_object_copies":17394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5798,"num_write_kb":371072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":836,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045045+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:36:41.045045+0000","last_peered":"2023-10-09T06:36:41.045045+0000","last_clean":"2023-10-09T06:36:41.045045+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:36:41.045045+0000","last_undegraded":"2023-10-09T06:36:41.045045+0000","last_fullsized":"2023-10-09T06:36:41.045045+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":298,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.075790+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:36:41.075790+0000","last_peered":"2023-10-09T06:36:41.075790+0000","last_clean":"2023-10-09T06:36:41.075790+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:36:41.075790+0000","last_undegraded":"2023-10-09T06:36:41.075790+0000","last_fullsized":"2023-10-09T06:36:41.075790+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":448,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053350+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:36:41.053350+0000","last_peered":"2023-10-09T06:36:41.053350+0000","last_clean":"2023-10-09T06:36:41.053350+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:36:41.053350+0000","last_undegraded":"2023-10-09T06:36:41.053350+0000","last_fullsized":"2023-10-09T06:36:41.053350+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"450'5760","reported_seq":5782,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:46.021469+0000","last_change":"2023-10-09T06:35:28.532035+0000","last_active":"2023-10-09T06:36:46.021469+0000","last_peered":"2023-10-09T06:36:46.021469+0000","last_clean":"2023-10-09T06:36:46.021469+0000","last_became_active":"2023-10-09T06:35:28.531723+0000","last_became_peered":"2023-10-09T06:35:28.531723+0000","last_unstale":"2023-10-09T06:36:46.021469+0000","last_undegraded":"2023-10-09T06:36:46.021469+0000","last_fullsized":"2023-10-09T06:36:46.021469+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:19.173492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377487360,"num_objects":5760,"num_object_clones":0,"num_object_copies":17280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5760,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":752,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.062209+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:36:41.062209+0000","last_peered":"2023-10-09T06:36:41.062209+0000","last_clean":"2023-10-09T06:36:41.062209+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:36:41.062209+0000","last_undegraded":"2023-10-09T06:36:41.062209+0000","last_fullsized":"2023-10-09T06:36:41.062209+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":354,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.066801+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:36:41.066801+0000","last_peered":"2023-10-09T06:36:41.066801+0000","last_clean":"2023-10-09T06:36:41.066801+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:36:41.066801+0000","last_undegraded":"2023-10-09T06:36:41.066801+0000","last_fullsized":"2023-10-09T06:36:41.066801+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":427,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.059504+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:36:41.059504+0000","last_peered":"2023-10-09T06:36:41.059504+0000","last_clean":"2023-10-09T06:36:41.059504+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:36:41.059504+0000","last_undegraded":"2023-10-09T06:36:41.059504+0000","last_fullsized":"2023-10-09T06:36:41.059504+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"451'5770","reported_seq":5792,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.440043+0000","last_change":"2023-10-09T06:35:28.543526+0000","last_active":"2023-10-09T06:36:47.440043+0000","last_peered":"2023-10-09T06:36:47.440043+0000","last_clean":"2023-10-09T06:36:47.440043+0000","last_became_active":"2023-10-09T06:35:28.543240+0000","last_became_peered":"2023-10-09T06:35:28.543240+0000","last_unstale":"2023-10-09T06:36:47.440043+0000","last_undegraded":"2023-10-09T06:36:47.440043+0000","last_fullsized":"2023-10-09T06:36:47.440043+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:03:16.584504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378142720,"num_objects":5770,"num_object_clones":0,"num_object_copies":17310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5770,"num_write_kb":369280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":575,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.058608+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:36:41.058608+0000","last_peered":"2023-10-09T06:36:41.058608+0000","last_clean":"2023-10-09T06:36:41.058608+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:36:41.058608+0000","last_undegraded":"2023-10-09T06:36:41.058608+0000","last_fullsized":"2023-10-09T06:36:41.058608+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.056226+0000","last_change":"2023-10-09T06:28:11.700192+0000","last_active":"2023-10-09T06:36:41.056226+0000","last_peered":"2023-10-09T06:36:41.056226+0000","last_clean":"2023-10-09T06:36:41.056226+0000","last_became_active":"2023-10-09T06:28:11.699497+0000","last_became_peered":"2023-10-09T06:28:11.699497+0000","last_unstale":"2023-10-09T06:36:41.056226+0000","last_undegraded":"2023-10-09T06:36:41.056226+0000","last_fullsized":"2023-10-09T06:36:41.056226+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:46:38.625255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":411,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972036+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:36:34.972036+0000","last_peered":"2023-10-09T06:36:34.972036+0000","last_clean":"2023-10-09T06:36:34.972036+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:36:34.972036+0000","last_undegraded":"2023-10-09T06:36:34.972036+0000","last_fullsized":"2023-10-09T06:36:34.972036+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"451'5825","reported_seq":5847,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.510034+0000","last_change":"2023-10-09T06:35:28.543354+0000","last_active":"2023-10-09T06:36:47.510034+0000","last_peered":"2023-10-09T06:36:47.510034+0000","last_clean":"2023-10-09T06:36:47.510034+0000","last_became_active":"2023-10-09T06:35:28.543001+0000","last_became_peered":"2023-10-09T06:35:28.543001+0000","last_unstale":"2023-10-09T06:36:47.510034+0000","last_undegraded":"2023-10-09T06:36:47.510034+0000","last_fullsized":"2023-10-09T06:36:47.510034+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5825,"log_dups_size":0,"ondisk_log_size":5825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:40:38.235566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381747200,"num_objects":5825,"num_object_clones":0,"num_object_copies":17475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5825,"num_write_kb":372800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":803,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.069874+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:36:41.069874+0000","last_peered":"2023-10-09T06:36:41.069874+0000","last_clean":"2023-10-09T06:36:41.069874+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:36:41.069874+0000","last_undegraded":"2023-10-09T06:36:41.069874+0000","last_fullsized":"2023-10-09T06:36:41.069874+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":178,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045083+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:36:41.045083+0000","last_peered":"2023-10-09T06:36:41.045083+0000","last_clean":"2023-10-09T06:36:41.045083+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:36:41.045083+0000","last_undegraded":"2023-10-09T06:36:41.045083+0000","last_fullsized":"2023-10-09T06:36:41.045083+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":431,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053053+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:36:41.053053+0000","last_peered":"2023-10-09T06:36:41.053053+0000","last_clean":"2023-10-09T06:36:41.053053+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:36:41.053053+0000","last_undegraded":"2023-10-09T06:36:41.053053+0000","last_fullsized":"2023-10-09T06:36:41.053053+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"451'5820","reported_seq":5842,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.506985+0000","last_change":"2023-10-09T06:35:28.750085+0000","last_active":"2023-10-09T06:36:47.506985+0000","last_peered":"2023-10-09T06:36:47.506985+0000","last_clean":"2023-10-09T06:36:47.506985+0000","last_became_active":"2023-10-09T06:35:28.749727+0000","last_became_peered":"2023-10-09T06:35:28.749727+0000","last_unstale":"2023-10-09T06:36:47.506985+0000","last_undegraded":"2023-10-09T06:36:47.506985+0000","last_fullsized":"2023-10-09T06:36:47.506985+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:00.559675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381419520,"num_objects":5820,"num_object_clones":0,"num_object_copies":17460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5820,"num_write_kb":372480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":783,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.051493+0000","last_change":"2023-10-09T06:32:26.271453+0000","last_active":"2023-10-09T06:36:41.051493+0000","last_peered":"2023-10-09T06:36:41.051493+0000","last_clean":"2023-10-09T06:36:41.051493+0000","last_became_active":"2023-10-09T06:32:10.599616+0000","last_became_peered":"2023-10-09T06:32:10.599616+0000","last_unstale":"2023-10-09T06:36:41.051493+0000","last_undegraded":"2023-10-09T06:36:41.051493+0000","last_fullsized":"2023-10-09T06:36:41.051493+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:23.726956+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":408,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.059368+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:36:41.059368+0000","last_peered":"2023-10-09T06:36:41.059368+0000","last_clean":"2023-10-09T06:36:41.059368+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:36:41.059368+0000","last_undegraded":"2023-10-09T06:36:41.059368+0000","last_fullsized":"2023-10-09T06:36:41.059368+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":428,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053398+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:36:41.053398+0000","last_peered":"2023-10-09T06:36:41.053398+0000","last_clean":"2023-10-09T06:36:41.053398+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:36:41.053398+0000","last_undegraded":"2023-10-09T06:36:41.053398+0000","last_fullsized":"2023-10-09T06:36:41.053398+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"451'5689","reported_seq":5711,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.564252+0000","last_change":"2023-10-09T06:35:28.543670+0000","last_active":"2023-10-09T06:36:47.564252+0000","last_peered":"2023-10-09T06:36:47.564252+0000","last_clean":"2023-10-09T06:36:47.564252+0000","last_became_active":"2023-10-09T06:35:28.543089+0000","last_became_peered":"2023-10-09T06:35:28.543089+0000","last_unstale":"2023-10-09T06:36:47.564252+0000","last_undegraded":"2023-10-09T06:36:47.564252+0000","last_fullsized":"2023-10-09T06:36:47.564252+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:02.123068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372834304,"num_objects":5689,"num_object_clones":0,"num_object_copies":17067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":364096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":786,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.038850+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:36:41.038850+0000","last_peered":"2023-10-09T06:36:41.038850+0000","last_clean":"2023-10-09T06:36:41.038850+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:36:41.038850+0000","last_undegraded":"2023-10-09T06:36:41.038850+0000","last_fullsized":"2023-10-09T06:36:41.038850+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":432,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053139+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:36:41.053139+0000","last_peered":"2023-10-09T06:36:41.053139+0000","last_clean":"2023-10-09T06:36:41.053139+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:36:41.053139+0000","last_undegraded":"2023-10-09T06:36:41.053139+0000","last_fullsized":"2023-10-09T06:36:41.053139+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":411,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.058806+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:36:41.058806+0000","last_peered":"2023-10-09T06:36:41.058806+0000","last_clean":"2023-10-09T06:36:41.058806+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:36:41.058806+0000","last_undegraded":"2023-10-09T06:36:41.058806+0000","last_fullsized":"2023-10-09T06:36:41.058806+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"451'5706","reported_seq":5726,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.502524+0000","last_change":"2023-10-09T06:35:28.540580+0000","last_active":"2023-10-09T06:36:47.502524+0000","last_peered":"2023-10-09T06:36:47.502524+0000","last_clean":"2023-10-09T06:36:47.502524+0000","last_became_active":"2023-10-09T06:35:28.540221+0000","last_became_peered":"2023-10-09T06:35:28.540221+0000","last_unstale":"2023-10-09T06:36:47.502524+0000","last_undegraded":"2023-10-09T06:36:47.502524+0000","last_fullsized":"2023-10-09T06:36:47.502524+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5706,"log_dups_size":0,"ondisk_log_size":5706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:31.385902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373948416,"num_objects":5706,"num_object_clones":0,"num_object_copies":17118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5706,"num_write_kb":365184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":761,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057720+0000","last_change":"2023-10-09T06:32:25.246311+0000","last_active":"2023-10-09T06:36:41.057720+0000","last_peered":"2023-10-09T06:36:41.057720+0000","last_clean":"2023-10-09T06:36:41.057720+0000","last_became_active":"2023-10-09T06:32:10.606523+0000","last_became_peered":"2023-10-09T06:32:10.606523+0000","last_unstale":"2023-10-09T06:36:41.057720+0000","last_undegraded":"2023-10-09T06:36:41.057720+0000","last_fullsized":"2023-10-09T06:36:41.057720+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:00.085234+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044590+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:36:41.044590+0000","last_peered":"2023-10-09T06:36:41.044590+0000","last_clean":"2023-10-09T06:36:41.044590+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:36:41.044590+0000","last_undegraded":"2023-10-09T06:36:41.044590+0000","last_fullsized":"2023-10-09T06:36:41.044590+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":405,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039519+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:36:41.039519+0000","last_peered":"2023-10-09T06:36:41.039519+0000","last_clean":"2023-10-09T06:36:41.039519+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:36:41.039519+0000","last_undegraded":"2023-10-09T06:36:41.039519+0000","last_fullsized":"2023-10-09T06:36:41.039519+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"451'5742","reported_seq":5764,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.438323+0000","last_change":"2023-10-09T06:35:28.750966+0000","last_active":"2023-10-09T06:36:47.438323+0000","last_peered":"2023-10-09T06:36:47.438323+0000","last_clean":"2023-10-09T06:36:47.438323+0000","last_became_active":"2023-10-09T06:35:28.750668+0000","last_became_peered":"2023-10-09T06:35:28.750668+0000","last_unstale":"2023-10-09T06:36:47.438323+0000","last_undegraded":"2023-10-09T06:36:47.438323+0000","last_fullsized":"2023-10-09T06:36:47.438323+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5742,"log_dups_size":0,"ondisk_log_size":5742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:47.331236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376307712,"num_objects":5742,"num_object_clones":0,"num_object_copies":17226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5742,"num_write_kb":367488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":743,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.062436+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:36:41.062436+0000","last_peered":"2023-10-09T06:36:41.062436+0000","last_clean":"2023-10-09T06:36:41.062436+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:36:41.062436+0000","last_undegraded":"2023-10-09T06:36:41.062436+0000","last_fullsized":"2023-10-09T06:36:41.062436+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":457,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.040558+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:36:41.040558+0000","last_peered":"2023-10-09T06:36:41.040558+0000","last_clean":"2023-10-09T06:36:41.040558+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:36:41.040558+0000","last_undegraded":"2023-10-09T06:36:41.040558+0000","last_fullsized":"2023-10-09T06:36:41.040558+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057201+0000","last_change":"2023-10-09T06:28:11.698923+0000","last_active":"2023-10-09T06:36:41.057201+0000","last_peered":"2023-10-09T06:36:41.057201+0000","last_clean":"2023-10-09T06:36:41.057201+0000","last_became_active":"2023-10-09T06:28:11.698512+0000","last_became_peered":"2023-10-09T06:28:11.698512+0000","last_unstale":"2023-10-09T06:36:41.057201+0000","last_undegraded":"2023-10-09T06:36:41.057201+0000","last_fullsized":"2023-10-09T06:36:41.057201+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:23:44.930592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":391,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039766+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:36:41.039766+0000","last_peered":"2023-10-09T06:36:41.039766+0000","last_clean":"2023-10-09T06:36:41.039766+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:36:41.039766+0000","last_undegraded":"2023-10-09T06:36:41.039766+0000","last_fullsized":"2023-10-09T06:36:41.039766+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"451'5797","reported_seq":5819,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.480494+0000","last_change":"2023-10-09T06:35:28.539445+0000","last_active":"2023-10-09T06:36:47.480494+0000","last_peered":"2023-10-09T06:36:47.480494+0000","last_clean":"2023-10-09T06:36:47.480494+0000","last_became_active":"2023-10-09T06:35:28.539141+0000","last_became_peered":"2023-10-09T06:35:28.539141+0000","last_unstale":"2023-10-09T06:36:47.480494+0000","last_undegraded":"2023-10-09T06:36:47.480494+0000","last_fullsized":"2023-10-09T06:36:47.480494+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5797,"log_dups_size":0,"ondisk_log_size":5797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:26.753486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379912192,"num_objects":5797,"num_object_clones":0,"num_object_copies":17391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5797,"num_write_kb":371008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":797,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053055+0000","last_change":"2023-10-09T06:32:12.169958+0000","last_active":"2023-10-09T06:36:41.053055+0000","last_peered":"2023-10-09T06:36:41.053055+0000","last_clean":"2023-10-09T06:36:41.053055+0000","last_became_active":"2023-10-09T06:31:52.452586+0000","last_became_peered":"2023-10-09T06:31:52.452586+0000","last_unstale":"2023-10-09T06:36:41.053055+0000","last_undegraded":"2023-10-09T06:36:41.053055+0000","last_fullsized":"2023-10-09T06:36:41.053055+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:36.821696+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057587+0000","last_change":"2023-10-09T06:28:11.700323+0000","last_active":"2023-10-09T06:36:41.057587+0000","last_peered":"2023-10-09T06:36:41.057587+0000","last_clean":"2023-10-09T06:36:41.057587+0000","last_became_active":"2023-10-09T06:28:11.699856+0000","last_became_peered":"2023-10-09T06:28:11.699856+0000","last_unstale":"2023-10-09T06:36:41.057587+0000","last_undegraded":"2023-10-09T06:36:41.057587+0000","last_fullsized":"2023-10-09T06:36:41.057587+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:20.875225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":432,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053997+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:36:41.053997+0000","last_peered":"2023-10-09T06:36:41.053997+0000","last_clean":"2023-10-09T06:36:41.053997+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:36:41.053997+0000","last_undegraded":"2023-10-09T06:36:41.053997+0000","last_fullsized":"2023-10-09T06:36:41.053997+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":421,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.040487+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:36:41.040487+0000","last_peered":"2023-10-09T06:36:41.040487+0000","last_clean":"2023-10-09T06:36:41.040487+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:36:41.040487+0000","last_undegraded":"2023-10-09T06:36:41.040487+0000","last_fullsized":"2023-10-09T06:36:41.040487+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"451'5849","reported_seq":5871,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.507462+0000","last_change":"2023-10-09T06:35:28.544012+0000","last_active":"2023-10-09T06:36:47.507462+0000","last_peered":"2023-10-09T06:36:47.507462+0000","last_clean":"2023-10-09T06:36:47.507462+0000","last_became_active":"2023-10-09T06:35:28.543684+0000","last_became_peered":"2023-10-09T06:35:28.543684+0000","last_unstale":"2023-10-09T06:36:47.507462+0000","last_undegraded":"2023-10-09T06:36:47.507462+0000","last_fullsized":"2023-10-09T06:36:47.507462+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:47.327388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383320064,"num_objects":5849,"num_object_clones":0,"num_object_copies":17547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5849,"num_write_kb":374336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":793,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.065604+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:36:41.065604+0000","last_peered":"2023-10-09T06:36:41.065604+0000","last_clean":"2023-10-09T06:36:41.065604+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:36:41.065604+0000","last_undegraded":"2023-10-09T06:36:41.065604+0000","last_fullsized":"2023-10-09T06:36:41.065604+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":442,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044769+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:36:41.044769+0000","last_peered":"2023-10-09T06:36:41.044769+0000","last_clean":"2023-10-09T06:36:41.044769+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:36:41.044769+0000","last_undegraded":"2023-10-09T06:36:41.044769+0000","last_fullsized":"2023-10-09T06:36:41.044769+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":429,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.042016+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:36:41.042016+0000","last_peered":"2023-10-09T06:36:41.042016+0000","last_clean":"2023-10-09T06:36:41.042016+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:36:41.042016+0000","last_undegraded":"2023-10-09T06:36:41.042016+0000","last_fullsized":"2023-10-09T06:36:41.042016+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":451,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039581+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:36:41.039581+0000","last_peered":"2023-10-09T06:36:41.039581+0000","last_clean":"2023-10-09T06:36:41.039581+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:36:41.039581+0000","last_undegraded":"2023-10-09T06:36:41.039581+0000","last_fullsized":"2023-10-09T06:36:41.039581+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"451'5803","reported_seq":5824,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.945714+0000","last_change":"2023-10-09T06:35:28.750537+0000","last_active":"2023-10-09T06:36:47.945714+0000","last_peered":"2023-10-09T06:36:47.945714+0000","last_clean":"2023-10-09T06:36:47.945714+0000","last_became_active":"2023-10-09T06:35:28.750267+0000","last_became_peered":"2023-10-09T06:35:28.750267+0000","last_unstale":"2023-10-09T06:36:47.945714+0000","last_undegraded":"2023-10-09T06:36:47.945714+0000","last_fullsized":"2023-10-09T06:36:47.945714+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:47.173545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380305408,"num_objects":5803,"num_object_clones":0,"num_object_copies":17409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5803,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":138,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.056878+0000","last_change":"2023-10-09T06:28:11.704892+0000","last_active":"2023-10-09T06:36:41.056878+0000","last_peered":"2023-10-09T06:36:41.056878+0000","last_clean":"2023-10-09T06:36:41.056878+0000","last_became_active":"2023-10-09T06:28:11.704550+0000","last_became_peered":"2023-10-09T06:28:11.704550+0000","last_unstale":"2023-10-09T06:36:41.056878+0000","last_undegraded":"2023-10-09T06:36:41.056878+0000","last_fullsized":"2023-10-09T06:36:41.056878+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:14.680994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":774,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.072293+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:36:41.072293+0000","last_peered":"2023-10-09T06:36:41.072293+0000","last_clean":"2023-10-09T06:36:41.072293+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:36:41.072293+0000","last_undegraded":"2023-10-09T06:36:41.072293+0000","last_fullsized":"2023-10-09T06:36:41.072293+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":161,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044962+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:36:41.044962+0000","last_peered":"2023-10-09T06:36:41.044962+0000","last_clean":"2023-10-09T06:36:41.044962+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:36:41.044962+0000","last_undegraded":"2023-10-09T06:36:41.044962+0000","last_fullsized":"2023-10-09T06:36:41.044962+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":413,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.059570+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:36:41.059570+0000","last_peered":"2023-10-09T06:36:41.059570+0000","last_clean":"2023-10-09T06:36:41.059570+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:36:41.059570+0000","last_undegraded":"2023-10-09T06:36:41.059570+0000","last_fullsized":"2023-10-09T06:36:41.059570+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"451'5809","reported_seq":5830,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.571639+0000","last_change":"2023-10-09T06:35:28.750108+0000","last_active":"2023-10-09T06:36:47.571639+0000","last_peered":"2023-10-09T06:36:47.571639+0000","last_clean":"2023-10-09T06:36:47.571639+0000","last_became_active":"2023-10-09T06:35:28.749808+0000","last_became_peered":"2023-10-09T06:35:28.749808+0000","last_unstale":"2023-10-09T06:36:47.571639+0000","last_undegraded":"2023-10-09T06:36:47.571639+0000","last_fullsized":"2023-10-09T06:36:47.571639+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:44.197980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380698624,"num_objects":5809,"num_object_clones":0,"num_object_copies":17427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5809,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":760,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.038933+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:36:41.038933+0000","last_peered":"2023-10-09T06:36:41.038933+0000","last_clean":"2023-10-09T06:36:41.038933+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:36:41.038933+0000","last_undegraded":"2023-10-09T06:36:41.038933+0000","last_fullsized":"2023-10-09T06:36:41.038933+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":314,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.072130+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:36:41.072130+0000","last_peered":"2023-10-09T06:36:41.072130+0000","last_clean":"2023-10-09T06:36:41.072130+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:36:41.072130+0000","last_undegraded":"2023-10-09T06:36:41.072130+0000","last_fullsized":"2023-10-09T06:36:41.072130+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":124,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.971914+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:36:34.971914+0000","last_peered":"2023-10-09T06:36:34.971914+0000","last_clean":"2023-10-09T06:36:34.971914+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:36:34.971914+0000","last_undegraded":"2023-10-09T06:36:34.971914+0000","last_fullsized":"2023-10-09T06:36:34.971914+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":435,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972378+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:36:34.972378+0000","last_peered":"2023-10-09T06:36:34.972378+0000","last_clean":"2023-10-09T06:36:34.972378+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:36:34.972378+0000","last_undegraded":"2023-10-09T06:36:34.972378+0000","last_fullsized":"2023-10-09T06:36:34.972378+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"451'5769","reported_seq":5792,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.866987+0000","last_change":"2023-10-09T06:35:28.748964+0000","last_active":"2023-10-09T06:36:47.866987+0000","last_peered":"2023-10-09T06:36:47.866987+0000","last_clean":"2023-10-09T06:36:47.866987+0000","last_became_active":"2023-10-09T06:35:28.748674+0000","last_became_peered":"2023-10-09T06:35:28.748674+0000","last_unstale":"2023-10-09T06:36:47.866987+0000","last_undegraded":"2023-10-09T06:36:47.866987+0000","last_fullsized":"2023-10-09T06:36:47.866987+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5769,"log_dups_size":0,"ondisk_log_size":5769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:02.079067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378077184,"num_objects":5769,"num_object_clones":0,"num_object_copies":17307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":773,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.059718+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:36:41.059718+0000","last_peered":"2023-10-09T06:36:41.059718+0000","last_clean":"2023-10-09T06:36:41.059718+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:36:41.059718+0000","last_undegraded":"2023-10-09T06:36:41.059718+0000","last_fullsized":"2023-10-09T06:36:41.059718+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":314,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.066949+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:36:41.066949+0000","last_peered":"2023-10-09T06:36:41.066949+0000","last_clean":"2023-10-09T06:36:41.066949+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:36:41.066949+0000","last_undegraded":"2023-10-09T06:36:41.066949+0000","last_fullsized":"2023-10-09T06:36:41.066949+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":447,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.040417+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:36:41.040417+0000","last_peered":"2023-10-09T06:36:41.040417+0000","last_clean":"2023-10-09T06:36:41.040417+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:36:41.040417+0000","last_undegraded":"2023-10-09T06:36:41.040417+0000","last_fullsized":"2023-10-09T06:36:41.040417+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":433,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972258+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:36:34.972258+0000","last_peered":"2023-10-09T06:36:34.972258+0000","last_clean":"2023-10-09T06:36:34.972258+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:36:34.972258+0000","last_undegraded":"2023-10-09T06:36:34.972258+0000","last_fullsized":"2023-10-09T06:36:34.972258+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"451'5747","reported_seq":5769,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.404269+0000","last_change":"2023-10-09T06:35:28.750679+0000","last_active":"2023-10-09T06:36:47.404269+0000","last_peered":"2023-10-09T06:36:47.404269+0000","last_clean":"2023-10-09T06:36:47.404269+0000","last_became_active":"2023-10-09T06:35:28.750367+0000","last_became_peered":"2023-10-09T06:35:28.750367+0000","last_unstale":"2023-10-09T06:36:47.404269+0000","last_undegraded":"2023-10-09T06:36:47.404269+0000","last_fullsized":"2023-10-09T06:36:47.404269+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5747,"log_dups_size":0,"ondisk_log_size":5747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.804665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376635392,"num_objects":5747,"num_object_clones":0,"num_object_copies":17241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5747,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":774,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053277+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:36:41.053277+0000","last_peered":"2023-10-09T06:36:41.053277+0000","last_clean":"2023-10-09T06:36:41.053277+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:36:41.053277+0000","last_undegraded":"2023-10-09T06:36:41.053277+0000","last_fullsized":"2023-10-09T06:36:41.053277+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":182,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045955+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:36:41.045955+0000","last_peered":"2023-10-09T06:36:41.045955+0000","last_clean":"2023-10-09T06:36:41.045955+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:36:41.045955+0000","last_undegraded":"2023-10-09T06:36:41.045955+0000","last_fullsized":"2023-10-09T06:36:41.045955+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":414,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039285+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:36:41.039285+0000","last_peered":"2023-10-09T06:36:41.039285+0000","last_clean":"2023-10-09T06:36:41.039285+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:36:41.039285+0000","last_undegraded":"2023-10-09T06:36:41.039285+0000","last_fullsized":"2023-10-09T06:36:41.039285+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":170,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044516+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:36:41.044516+0000","last_peered":"2023-10-09T06:36:41.044516+0000","last_clean":"2023-10-09T06:36:41.044516+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:36:41.044516+0000","last_undegraded":"2023-10-09T06:36:41.044516+0000","last_fullsized":"2023-10-09T06:36:41.044516+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"451'5805","reported_seq":5827,"reported_epoch":451,"state":"active+clean","last_fresh":"2023-10-09T06:36:47.466434+0000","last_change":"2023-10-09T06:35:28.748822+0000","last_active":"2023-10-09T06:36:47.466434+0000","last_peered":"2023-10-09T06:36:47.466434+0000","last_clean":"2023-10-09T06:36:47.466434+0000","last_became_active":"2023-10-09T06:35:28.748506+0000","last_became_peered":"2023-10-09T06:35:28.748506+0000","last_unstale":"2023-10-09T06:36:47.466434+0000","last_undegraded":"2023-10-09T06:36:47.466434+0000","last_fullsized":"2023-10-09T06:36:47.466434+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:40:24.343197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380436480,"num_objects":5805,"num_object_clones":0,"num_object_copies":17415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5805,"num_write_kb":371520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":804,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045386+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:36:41.045386+0000","last_peered":"2023-10-09T06:36:41.045386+0000","last_clean":"2023-10-09T06:36:41.045386+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:36:41.045386+0000","last_undegraded":"2023-10-09T06:36:41.045386+0000","last_fullsized":"2023-10-09T06:36:41.045386+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":435,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.062511+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:36:41.062511+0000","last_peered":"2023-10-09T06:36:41.062511+0000","last_clean":"2023-10-09T06:36:41.062511+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:36:41.062511+0000","last_undegraded":"2023-10-09T06:36:41.062511+0000","last_fullsized":"2023-10-09T06:36:41.062511+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":416,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972229+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:36:34.972229+0000","last_peered":"2023-10-09T06:36:34.972229+0000","last_clean":"2023-10-09T06:36:34.972229+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:36:34.972229+0000","last_undegraded":"2023-10-09T06:36:34.972229+0000","last_fullsized":"2023-10-09T06:36:34.972229+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":414,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053279+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:36:41.053279+0000","last_peered":"2023-10-09T06:36:41.053279+0000","last_clean":"2023-10-09T06:36:41.053279+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:36:41.053279+0000","last_undegraded":"2023-10-09T06:36:41.053279+0000","last_fullsized":"2023-10-09T06:36:41.053279+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":730,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044354+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:36:41.044354+0000","last_peered":"2023-10-09T06:36:41.044354+0000","last_clean":"2023-10-09T06:36:41.044354+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:36:41.044354+0000","last_undegraded":"2023-10-09T06:36:41.044354+0000","last_fullsized":"2023-10-09T06:36:41.044354+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":417,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972177+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:36:34.972177+0000","last_peered":"2023-10-09T06:36:34.972177+0000","last_clean":"2023-10-09T06:36:34.972177+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:36:34.972177+0000","last_undegraded":"2023-10-09T06:36:34.972177+0000","last_fullsized":"2023-10-09T06:36:34.972177+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":440,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.040341+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:36:41.040341+0000","last_peered":"2023-10-09T06:36:41.040341+0000","last_clean":"2023-10-09T06:36:41.040341+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:36:41.040341+0000","last_undegraded":"2023-10-09T06:36:41.040341+0000","last_fullsized":"2023-10-09T06:36:41.040341+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":661,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045099+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:36:41.045099+0000","last_peered":"2023-10-09T06:36:41.045099+0000","last_clean":"2023-10-09T06:36:41.045099+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:36:41.045099+0000","last_undegraded":"2023-10-09T06:36:41.045099+0000","last_fullsized":"2023-10-09T06:36:41.045099+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":142,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.056315+0000","last_change":"2023-10-09T06:28:11.709370+0000","last_active":"2023-10-09T06:36:41.056315+0000","last_peered":"2023-10-09T06:36:41.056315+0000","last_clean":"2023-10-09T06:36:41.056315+0000","last_became_active":"2023-10-09T06:28:11.700387+0000","last_became_peered":"2023-10-09T06:28:11.700387+0000","last_unstale":"2023-10-09T06:36:41.056315+0000","last_undegraded":"2023-10-09T06:36:41.056315+0000","last_fullsized":"2023-10-09T06:36:41.056315+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:29.872547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":458,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.053920+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:36:41.053920+0000","last_peered":"2023-10-09T06:36:41.053920+0000","last_clean":"2023-10-09T06:36:41.053920+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:36:41.053920+0000","last_undegraded":"2023-10-09T06:36:41.053920+0000","last_fullsized":"2023-10-09T06:36:41.053920+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":142,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.057047+0000","last_change":"2023-10-09T06:28:11.711563+0000","last_active":"2023-10-09T06:36:41.057047+0000","last_peered":"2023-10-09T06:36:41.057047+0000","last_clean":"2023-10-09T06:36:41.057047+0000","last_became_active":"2023-10-09T06:28:11.702026+0000","last_became_peered":"2023-10-09T06:28:11.702026+0000","last_unstale":"2023-10-09T06:36:41.057047+0000","last_undegraded":"2023-10-09T06:36:41.057047+0000","last_fullsized":"2023-10-09T06:36:41.057047+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:49.967756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":468,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.054426+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:36:41.054426+0000","last_peered":"2023-10-09T06:36:41.054426+0000","last_clean":"2023-10-09T06:36:41.054426+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:36:41.054426+0000","last_undegraded":"2023-10-09T06:36:41.054426+0000","last_fullsized":"2023-10-09T06:36:41.054426+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":807,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.066364+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:36:41.066364+0000","last_peered":"2023-10-09T06:36:41.066364+0000","last_clean":"2023-10-09T06:36:41.066364+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:36:41.066364+0000","last_undegraded":"2023-10-09T06:36:41.066364+0000","last_fullsized":"2023-10-09T06:36:41.066364+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":152,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.045289+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:36:41.045289+0000","last_peered":"2023-10-09T06:36:41.045289+0000","last_clean":"2023-10-09T06:36:41.045289+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:36:41.045289+0000","last_undegraded":"2023-10-09T06:36:41.045289+0000","last_fullsized":"2023-10-09T06:36:41.045289+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":169,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.044644+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:36:41.044644+0000","last_peered":"2023-10-09T06:36:41.044644+0000","last_clean":"2023-10-09T06:36:41.044644+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:36:41.044644+0000","last_undegraded":"2023-10-09T06:36:41.044644+0000","last_fullsized":"2023-10-09T06:36:41.044644+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":572,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.039380+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:36:41.039380+0000","last_peered":"2023-10-09T06:36:41.039380+0000","last_clean":"2023-10-09T06:36:41.039380+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:36:41.039380+0000","last_undegraded":"2023-10-09T06:36:41.039380+0000","last_fullsized":"2023-10-09T06:36:41.039380+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":446,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-10-09T06:36:34.972628+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:36:34.972628+0000","last_peered":"2023-10-09T06:36:34.972628+0000","last_clean":"2023-10-09T06:36:34.972628+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:36:34.972628+0000","last_undegraded":"2023-10-09T06:36:34.972628+0000","last_fullsized":"2023-10-09T06:36:34.972628+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":411,"reported_epoch":450,"state":"active+clean","last_fresh":"2023-10-09T06:36:41.052805+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:36:41.052805+0000","last_peered":"2023-10-09T06:36:41.052805+0000","last_clean":"2023-10-09T06:36:41.052805+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:36:41.052805+0000","last_undegraded":"2023-10-09T06:36:41.052805+0000","last_fullsized":"2023-10-09T06:36:41.052805+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":6058934272,"num_objects":92452,"num_object_clones":0,"num_object_copies":277356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92452,"num_write_kb":5916928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18150064128,"data_stored":18150064128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92452,"ondisk_log_size":92452,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234474,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3047756,"kb_used_data":1846852,"kb_used_omap":0,"kb_used_meta":1200896,"kb_avail":90697396,"statfs":{"total":95995035648,"available":92874133504,"internally_reserved":0,"allocated":1891176448,"data_stored":1890137130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1229717504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":4.6120000000000001,"15min":6.1369999999999996},"min":{"1min":0.25,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":2.3199999999999998,"5min":196.28899999999999,"15min":219.41900000000001},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.77900000000000003,"5min":4.7329999999999997,"15min":6.3010000000000002},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.4209999999999998,"5min":196.333,"15min":219.399},"last":1.8440000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:35:48 2023","interfaces":[{"interface":"back","average":{"1min":11.321999999999999,"5min":4.5990000000000002,"15min":4.8099999999999996},"min":{"1min":0.35899999999999999,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":208.626,"5min":208.626,"15min":215.61699999999999},"last":3.0369999999999999},{"interface":"front","average":{"1min":1.004,"5min":2.3940000000000001,"15min":3.8650000000000002},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":3.7229999999999999,"5min":107.904,"15min":216.22200000000001},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 06:35:48 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":3.9300000000000002,"15min":6.9029999999999996},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":4.0250000000000004,"5min":198.94200000000001,"15min":216.34299999999999},"last":3.077},{"interface":"front","average":{"1min":1.319,"5min":3.9079999999999999,"15min":7.048},"min":{"1min":0.44,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":4.1219999999999999,"5min":198.92099999999999,"15min":216.25200000000001},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.105,"15min":4.3200000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.2149999999999999,"5min":97.441999999999993,"15min":215.922},"last":2.0049999999999999},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.0049999999999999,"15min":4.3710000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":5.2839999999999998,"5min":97.162000000000006,"15min":215.94900000000001},"last":2.1190000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":2.649,"5min":1.0269999999999999,"15min":2.391},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":11.367000000000001,"5min":11.367000000000001,"15min":218.446},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.599,"5min":0.878,"15min":2.2919999999999998},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.23699999999999999},"max":{"1min":5.7889999999999997,"5min":5.7889999999999997,"15min":216.172},"last":8.8070000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":0.99299999999999999,"15min":2.129},"min":{"1min":0.32300000000000001,"5min":0.27600000000000002,"15min":0.22900000000000001},"max":{"1min":13.378,"5min":13.378,"15min":216.37200000000001},"last":9.1310000000000002},{"interface":"front","average":{"1min":2.5880000000000001,"5min":1.202,"15min":2.169},"min":{"1min":0.41099999999999998,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":17.087,"5min":17.087,"15min":218.15899999999999},"last":0.45200000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.5529999999999999,"15min":2.4329999999999998},"min":{"1min":0.42799999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":11.321999999999999,"5min":48.283000000000001,"15min":218.19999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.4550000000000001,"15min":2.21},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.33200000000000002},"max":{"1min":7.3639999999999999,"5min":41.448999999999998,"15min":218.05500000000001},"last":1.782}]}]},{"osd":6,"up_from":46,"seq":197568496110,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2750196,"kb_used_data":1838892,"kb_used_omap":0,"kb_used_meta":911296,"kb_avail":90994956,"statfs":{"total":95995035648,"available":93178834944,"internally_reserved":0,"allocated":1883025408,"data_stored":1882009388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933167104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":4.1349999999999998,"15min":3.1680000000000001},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":4.1449999999999996,"5min":75.061999999999998,"15min":110.926},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.657,"5min":4.3639999999999999,"15min":3.4119999999999999},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.23599999999999999},"max":{"1min":7.6520000000000001,"5min":75.021000000000001,"15min":110.911},"last":5.0679999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:35:53 2023","interfaces":[{"interface":"back","average":{"1min":4.1269999999999998,"5min":2.8809999999999998,"15min":4.2380000000000004},"min":{"1min":0.29599999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":65.153000000000006,"5min":65.153000000000006,"15min":292.92200000000003},"last":5.1050000000000004},{"interface":"front","average":{"1min":4.0579999999999998,"5min":2.9300000000000002,"15min":4.1840000000000002},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.27900000000000003},"max":{"1min":65.132000000000005,"5min":65.132000000000005,"15min":292.904},"last":2.0630000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":3.1080000000000001,"15min":3.1019999999999999},"min":{"1min":0.26200000000000001,"5min":0.251,"15min":0.251},"max":{"1min":2.9300000000000002,"5min":74.748999999999995,"15min":75.191999999999993},"last":5.0250000000000004},{"interface":"front","average":{"1min":0.93500000000000005,"5min":3.1000000000000001,"15min":2.9199999999999999},"min":{"1min":0.36299999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2290000000000001,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.80700000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":3.2210000000000001,"15min":3.5289999999999999},"min":{"1min":0.34100000000000003,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.8580000000000005,"5min":62.223999999999997,"15min":110.358},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.282,"5min":2.9460000000000002,"15min":3.4740000000000002},"min":{"1min":0.42999999999999999,"5min":0.34599999999999997,"15min":0.311},"max":{"1min":5.5780000000000003,"5min":61.637,"15min":110.43899999999999},"last":0.82599999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":1.5960000000000001,"15min":1.3620000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.5379999999999998,"5min":86.171999999999997,"15min":86.171999999999997},"last":5.2300000000000004},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.635,"15min":1.361},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":4.0659999999999998,"5min":86.727000000000004,"15min":86.727000000000004},"last":5.3300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.91300000000000003,"15min":1.2290000000000001},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.4489999999999998,"5min":5.7539999999999996,"15min":22.379999999999999},"last":5.2539999999999996},{"interface":"front","average":{"1min":1.014,"5min":1.073,"15min":1.3129999999999999},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":4.835,"5min":6.9349999999999996,"15min":24.401},"last":3.9510000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:36:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.028,"15min":2.5179999999999998},"min":{"1min":0.37,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":6.0069999999999997,"5min":7.4029999999999996,"15min":122.937},"last":4.0010000000000003},{"interface":"front","average":{"1min":2.5640000000000001,"5min":1.319,"15min":2.702},"min":{"1min":0.45100000000000001,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":5.3810000000000002,"5min":7.4729999999999999,"15min":122.92100000000001},"last":3.9100000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402847,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2691376,"kb_used_data":1857320,"kb_used_omap":4,"kb_used_meta":834043,"kb_avail":91053776,"statfs":{"total":95995035648,"available":93239066624,"internally_reserved":0,"allocated":1901895680,"data_stored":1899897931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":854060942},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.7040000000000002,"15min":5.8140000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.189},"max":{"1min":5.0460000000000003,"5min":40.969999999999999,"15min":269.79199999999997},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.7629999999999999,"15min":6.2519999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.1890000000000001,"5min":40.880000000000003,"15min":271.20999999999998},"last":1.4730000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:36:47 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":2.3220000000000001,"15min":3.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":10.284000000000001,"5min":22.614999999999998,"15min":139.75399999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.2669999999999999,"15min":4.2039999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.577,"5min":24.372,"15min":140.29900000000001},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":3.0449999999999999,"15min":3.2069999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.215},"max":{"1min":4.8959999999999999,"5min":27.064,"15min":77.331999999999994},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.081,"5min":2.1749999999999998,"15min":2.2610000000000001},"min":{"1min":0.251,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":10.228999999999999,"5min":26.626999999999999,"15min":77.197000000000003},"last":1.282}]},{"osd":4,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":9.3109999999999999,"5min":4.0359999999999996,"15min":3.4849999999999999},"min":{"1min":0.32300000000000001,"5min":0.317,"15min":0.27500000000000002},"max":{"1min":120.309,"5min":120.309,"15min":120.309},"last":0.39000000000000001},{"interface":"front","average":{"1min":9.6270000000000007,"5min":3.4990000000000001,"15min":3.3109999999999999},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":120.297,"5min":120.297,"15min":120.297},"last":0.35099999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":12.443,"5min":4.1639999999999997,"15min":2.7989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":119.408,"5min":119.408,"15min":119.408},"last":0.32200000000000001},{"interface":"front","average":{"1min":12.667999999999999,"5min":4.1890000000000001,"15min":3.1930000000000001},"min":{"1min":0.38300000000000001,"5min":0.371,"15min":0.32700000000000001},"max":{"1min":119.483,"5min":119.483,"15min":119.483},"last":0.38}]},{"osd":6,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":6.6100000000000003,"5min":3.8719999999999999,"15min":3.3700000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":121.60899999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":12.175000000000001},{"interface":"front","average":{"1min":6.3579999999999997,"5min":4.4619999999999997,"15min":3.5350000000000001},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":121.559,"5min":121.559,"15min":121.559},"last":12.191000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8859999999999992,"5min":4.8499999999999996,"15min":3.8860000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":123.20699999999999,"5min":123.20699999999999,"15min":123.20699999999999},"last":1.7689999999999999},{"interface":"front","average":{"1min":8.7650000000000006,"5min":4.2770000000000001,"15min":3.359},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":123.48999999999999,"5min":123.48999999999999,"15min":123.48999999999999},"last":1.752}]}]},{"osd":0,"up_from":91,"seq":390842024280,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096512,"kb_used_data":1106488,"kb_used_omap":5,"kb_used_meta":990010,"kb_avail":91648640,"statfs":{"total":95995035648,"available":93848207360,"internally_reserved":0,"allocated":1133043712,"data_stored":1130735284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1013770874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:36:40 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.7759999999999998,"15min":2.6899999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":73.135000000000005,"5min":73.135000000000005,"15min":191.446},"last":0.57599999999999996},{"interface":"front","average":{"1min":5.1619999999999999,"5min":3.4329999999999998,"15min":3.0710000000000002},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.187},"max":{"1min":73.254999999999995,"5min":73.254999999999995,"15min":191.393},"last":0.873}]},{"osd":2,"last update":"Mon Oct 9 06:35:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6319999999999997,"5min":3.2309999999999999,"15min":2.8079999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":82.218000000000004,"5min":82.218000000000004,"15min":82.218000000000004},"last":3.552},{"interface":"front","average":{"1min":4.3959999999999999,"5min":3.6749999999999998,"15min":3.2989999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":81.864000000000004,"5min":81.864000000000004,"15min":81.864000000000004},"last":3.0209999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:13 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":3.4039999999999999,"15min":4.8819999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.22500000000000001},"max":{"1min":3.0499999999999998,"5min":24.129000000000001,"15min":208.11199999999999},"last":4.1879999999999997},{"interface":"front","average":{"1min":1.075,"5min":3.4260000000000002,"15min":4.7830000000000004},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25600000000000001},"max":{"1min":9.0220000000000002,"5min":24.797000000000001,"15min":208.25899999999999},"last":4.2249999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.5469999999999999,"15min":1.55},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":8.8010000000000002,"5min":8.9800000000000004,"15min":9.7159999999999993},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.986,"15min":1.855},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.306,"5min":12.385999999999999,"15min":18.081},"last":0.90500000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.722,"15min":1.845},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.27800000000000002},"max":{"1min":8.8569999999999993,"5min":9.0099999999999998,"15min":62.984999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.76600000000000001,"5min":2.0489999999999999,"15min":2.2509999999999999},"min":{"1min":0.378,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.6349999999999998,"5min":13.968999999999999,"15min":64.983999999999995},"last":2.9430000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.2400000000000002,"15min":2.9279999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":4.6299999999999999,"5min":22.292000000000002,"15min":120.572},"last":2.9809999999999999},{"interface":"front","average":{"1min":1.349,"5min":1.909,"15min":2.456},"min":{"1min":0.38400000000000001,"5min":0.32800000000000001,"15min":0.30299999999999999},"max":{"1min":8.891,"5min":22.475000000000001,"15min":120.64400000000001},"last":0.64200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":2.2719999999999998,"15min":2.6549999999999998},"min":{"1min":0.40600000000000003,"5min":0.376,"15min":0.32800000000000001},"max":{"1min":4.2610000000000001,"5min":22.879000000000001,"15min":122.018},"last":3.52},{"interface":"front","average":{"1min":1.2210000000000001,"5min":2.5510000000000002,"15min":3.0649999999999999},"min":{"1min":0.44600000000000001,"5min":0.39400000000000002,"15min":0.377},"max":{"1min":6.4420000000000002,"5min":22.683,"15min":122.32299999999999},"last":2.9079999999999999}]}]},{"osd":2,"up_from":128,"seq":549755814149,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4095260,"kb_used_data":3289684,"kb_used_omap":7,"kb_used_meta":805560,"kb_avail":89649892,"statfs":{"total":95995035648,"available":91801489408,"internally_reserved":0,"allocated":3368636416,"data_stored":3366706236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":824893521},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":12.358000000000001,"5min":8.9239999999999995,"15min":5.9299999999999997},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.157},"max":{"1min":121.84999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":11.401,"5min":8.3309999999999995,"15min":5.3730000000000002},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":121.675,"5min":156.922,"15min":168.49700000000001},"last":0.46000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":6.8570000000000002,"5min":7.0209999999999999,"15min":5.0839999999999996},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":73.515000000000001,"5min":140.452,"15min":205.148},"last":0.316},{"interface":"front","average":{"1min":7.1779999999999999,"5min":6.6340000000000003,"15min":5.2949999999999999},"min":{"1min":0.28699999999999998,"5min":0.183,"15min":0.183},"max":{"1min":73.876999999999995,"5min":139.75299999999999,"15min":205.267},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":4.2919999999999998,"15min":3.5019999999999998},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.20599999999999999},"max":{"1min":73.953000000000003,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.48799999999999999},{"interface":"front","average":{"1min":4.5830000000000002,"5min":4.609,"15min":3.843},"min":{"1min":0.312,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":73.911000000000001,"5min":78.575999999999993,"15min":91.685000000000002},"last":2.9729999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":5.5030000000000001,"5min":2.7109999999999999,"15min":2.4740000000000002},"min":{"1min":0.46100000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":73.932000000000002,"5min":73.932000000000002,"15min":90.938999999999993},"last":0.51900000000000002},{"interface":"front","average":{"1min":5.3200000000000003,"5min":2.879,"15min":2.7730000000000001},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":74.349999999999994,"5min":74.349999999999994,"15min":90.950000000000003},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":5.2069999999999999,"5min":2.2610000000000001,"15min":1.9219999999999999},"min":{"1min":0.374,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":74.204999999999998,"5min":74.204999999999998,"15min":74.204999999999998},"last":0.70899999999999996},{"interface":"front","average":{"1min":5.3650000000000002,"5min":2.4159999999999999,"15min":2.1469999999999998},"min":{"1min":0.378,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":74.683000000000007,"5min":74.683000000000007,"15min":74.683000000000007},"last":0.87}]},{"osd":6,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":5.1440000000000001,"5min":4.8360000000000003,"15min":3.661},"min":{"1min":0.41699999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":74.653999999999996,"5min":123.47799999999999,"15min":138.291},"last":0.91000000000000003},{"interface":"front","average":{"1min":5.8079999999999998,"5min":5.218,"15min":3.452},"min":{"1min":0.436,"5min":0.313,"15min":0.313},"max":{"1min":73.837000000000003,"5min":122.517,"15min":138.01599999999999},"last":0.81000000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":5.774,"5min":3.4689999999999999,"15min":2.597},"min":{"1min":0.39000000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":73.863,"5min":73.863,"15min":73.863},"last":0.60699999999999998},{"interface":"front","average":{"1min":5.2039999999999997,"5min":2.79,"15min":2.3639999999999999},"min":{"1min":0.44500000000000001,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":74.171999999999997,"5min":74.171999999999997,"15min":74.171999999999997},"last":1.03}]}]},{"osd":3,"up_from":149,"seq":639950127347,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2198800,"kb_used_data":1488392,"kb_used_omap":2,"kb_used_meta":710397,"kb_avail":91546352,"statfs":{"total":95995035648,"available":93743464448,"internally_reserved":0,"allocated":1524113408,"data_stored":1522261429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":727446581},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":6.069,"15min":5.7510000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.19},"max":{"1min":2.0150000000000001,"5min":161.821,"15min":187.852},"last":0.24299999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":5.7850000000000001,"15min":5.766},"min":{"1min":0.20599999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":12.298,"5min":163.52000000000001,"15min":187.923},"last":3.2480000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":12.403,"5min":6.0599999999999996,"15min":4.9459999999999997},"min":{"1min":0.22800000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":184.012,"5min":184.012,"15min":187.89099999999999},"last":3.2810000000000001},{"interface":"front","average":{"1min":12.103999999999999,"5min":7.0019999999999998,"15min":4.9169999999999998},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":184.703,"5min":184.703,"15min":184.703},"last":0.307}]},{"osd":2,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.893000000000001,"5min":6.0300000000000002,"15min":4.7149999999999999},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":182.44900000000001,"5min":182.44900000000001,"15min":187.76400000000001},"last":3.2000000000000002},{"interface":"front","average":{"1min":11.491,"5min":6.4260000000000002,"15min":4.8929999999999998},"min":{"1min":0.29599999999999999,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":182.56399999999999,"5min":182.56399999999999,"15min":187.79499999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.643000000000001,"5min":3.9580000000000002,"15min":2.7189999999999999},"min":{"1min":0.41299999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":178.22200000000001,"5min":178.22200000000001,"15min":178.22200000000001},"last":3.6190000000000002},{"interface":"front","average":{"1min":12.106999999999999,"5min":4.0609999999999999,"15min":3.331},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":180.43600000000001,"5min":180.43600000000001,"15min":187.67400000000001},"last":0.68500000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.898,"5min":4.9109999999999996,"15min":3.5379999999999998},"min":{"1min":0.441,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":177.56299999999999,"5min":177.56299999999999,"15min":187.833},"last":0.76300000000000001},{"interface":"front","average":{"1min":11.882999999999999,"5min":3.6309999999999998,"15min":2.9119999999999999},"min":{"1min":0.40899999999999997,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":177.59399999999999,"5min":177.59399999999999,"15min":187.952},"last":3.9790000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.1379999999999999,"15min":3.2829999999999999},"min":{"1min":0.41799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1970000000000001,"5min":16.462,"15min":187.78200000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.363,"5min":1.7310000000000001,"15min":2.843},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":8.3350000000000009,"5min":22.260000000000002,"15min":187.81800000000001},"last":4.008}]},{"osd":7,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":2.0070000000000001,"15min":3.3319999999999999},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.195,"5min":13.695,"15min":192.20400000000001},"last":3.306},{"interface":"front","average":{"1min":1.607,"5min":2.8849999999999998,"15min":3.6840000000000002},"min":{"1min":0.39100000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2619999999999996,"5min":87.001999999999995,"15min":187.86500000000001},"last":3.3239999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019383,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4200772,"kb_used_data":3318076,"kb_used_omap":0,"kb_used_meta":882688,"kb_avail":89544380,"statfs":{"total":95995035648,"available":91693445120,"internally_reserved":0,"allocated":3397709824,"data_stored":3396685928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":903872512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:14 2023","interfaces":[{"interface":"back","average":{"1min":2.5840000000000001,"5min":15.028,"15min":6.7519999999999998},"min":{"1min":0.249,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":31.366,"5min":287.58800000000002,"15min":287.58800000000002},"last":4.6429999999999998},{"interface":"front","average":{"1min":2.6389999999999998,"5min":15.023,"15min":6.9009999999999998},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":31.373999999999999,"5min":287.57100000000003,"15min":287.57100000000003},"last":1.468}]},{"osd":1,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":4.2270000000000003,"15min":3.1699999999999999},"min":{"1min":0.246,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":34.548999999999999,"5min":134.37100000000001,"15min":134.37100000000001},"last":4.7140000000000004},{"interface":"front","average":{"1min":3.262,"5min":4.327,"15min":3.335},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":31.024999999999999,"5min":134.21700000000001,"15min":134.21700000000001},"last":4.7380000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.423,"5min":5.8159999999999998,"15min":3.8050000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":33.085000000000001,"5min":151.66399999999999,"15min":151.66399999999999},"last":4.0019999999999998},{"interface":"front","average":{"1min":3.569,"5min":5.6989999999999998,"15min":3.7519999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":32.299999999999997,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":6.1849999999999996,"15min":5.3040000000000003},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":9.5359999999999996,"5min":161.19300000000001,"15min":161.19300000000001},"last":3.0539999999999998},{"interface":"front","average":{"1min":1.3360000000000001,"5min":6.1909999999999998,"15min":5.0860000000000003},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":5.4329999999999998,"5min":161.934,"15min":161.934},"last":4.8070000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":0.94299999999999995,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.252},"max":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":9.1720000000000006},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.3359999999999999,"5min":0.98699999999999999,"15min":0.99199999999999999},"min":{"1min":0.309,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":9.5009999999999994,"5min":9.5009999999999994,"15min":9.6379999999999999},"last":0.40100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":2.1920000000000002,"15min":1.831},"min":{"1min":0.315,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.8159999999999998,"5min":115.70999999999999,"15min":163.36699999999999},"last":9.4239999999999995},{"interface":"front","average":{"1min":1.1259999999999999,"5min":2.3170000000000002,"15min":2.101},"min":{"1min":0.33900000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":6.1890000000000001,"5min":115.68600000000001,"15min":163.381},"last":1.085}]},{"osd":7,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.8029999999999999,"15min":1.863},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":3.3940000000000001,"5min":106.801,"15min":106.801},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.724,"5min":2.9319999999999999,"15min":1.877},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":16.713000000000001,"5min":106.827,"15min":106.827},"last":9.4749999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757748,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3832436,"kb_used_data":2943020,"kb_used_omap":0,"kb_used_meta":889408,"kb_avail":89912716,"statfs":{"total":95995035648,"available":92070621184,"internally_reserved":0,"allocated":3013652480,"data_stored":3012633089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910753792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:07 2023","interfaces":[{"interface":"back","average":{"1min":3.6699999999999999,"5min":8.673,"15min":5.8280000000000003},"min":{"1min":0.34200000000000003,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":62.180999999999997,"5min":294.89699999999999,"15min":294.89699999999999},"last":5.8179999999999996},{"interface":"front","average":{"1min":3.6240000000000001,"5min":8.1720000000000006,"15min":5.6600000000000001},"min":{"1min":0.27400000000000002,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":59.850000000000001,"5min":294.30399999999997,"15min":294.30399999999997},"last":4.9489999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":2.4700000000000002,"15min":3.1920000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":3.2010000000000001,"5min":119.024,"15min":126.71899999999999},"last":5.9249999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.3940000000000001,"15min":3.2850000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.7709999999999999,"5min":119.133,"15min":126.931},"last":2.3610000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:35:52 2023","interfaces":[{"interface":"back","average":{"1min":6.641,"5min":3.117,"15min":2.7879999999999998},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":120.613,"5min":120.613,"15min":120.613},"last":2.4049999999999998},{"interface":"front","average":{"1min":6.5510000000000002,"5min":3.0819999999999999,"15min":2.7360000000000002},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":120.962,"5min":120.962,"15min":120.962},"last":5.0149999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:36:13 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":3.5670000000000002,"15min":4.9790000000000001},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.8809999999999998,"5min":85.671000000000006,"15min":144.56399999999999},"last":5.859},{"interface":"front","average":{"1min":0.96299999999999997,"5min":3.2690000000000001,"15min":4.7999999999999998},"min":{"1min":0.33000000000000002,"5min":0.31900000000000001,"15min":0.28699999999999998},"max":{"1min":3.8679999999999999,"5min":85.593999999999994,"15min":144.62799999999999},"last":4.9909999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4359999999999999,"5min":2.0270000000000001,"15min":1.444},"min":{"1min":0.35499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":79.043000000000006,"5min":79.043000000000006,"15min":79.043000000000006},"last":9.9290000000000003},{"interface":"front","average":{"1min":7.7809999999999997,"5min":2.0419999999999998,"15min":1.385},"min":{"1min":0.35599999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":79.165999999999997,"5min":79.165999999999997,"15min":79.165999999999997},"last":6.9630000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":1.345,"15min":1.468},"min":{"1min":0.371,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.8959999999999999,"5min":57.505000000000003,"15min":57.505000000000003},"last":4.8360000000000003},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.506,"15min":1.5900000000000001},"min":{"1min":0.34399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.7010000000000001,"5min":55.040999999999997,"15min":55.040999999999997},"last":2.4860000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0369999999999999,"5min":1.901,"15min":1.583},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":78.890000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":9.9659999999999993},{"interface":"front","average":{"1min":5.3070000000000004,"5min":1.6839999999999999,"15min":1.587},"min":{"1min":0.40300000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":79.138999999999996,"5min":79.138999999999996,"15min":79.138999999999996},"last":5.5919999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1127415808,"data_stored":1127415808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1905065984,"data_stored":1905065984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3369140224,"data_stored":3369140224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1522728960,"data_stored":1522728960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3406954496,"data_stored":3406954496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3027828736,"data_stored":3027828736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1893597184,"data_stored":1893597184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1897332736,"data_stored":1897332736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:36:50.922 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:36:50.923 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pg_num 26 2023-10-09T06:36:50.962 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:51.106 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:51.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:51.215 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:51.238 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:51.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:51 smithi107 ceph-mon[212750]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:36:51.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:51 smithi107 ceph-mon[212750]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:51.297 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:51 smithi107 ceph-mon[220921]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:36:51.297 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:51 smithi107 ceph-mon[220921]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:51.299 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:51 smithi150 ceph-mon[194178]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:36:51.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:51 smithi150 ceph-mon[194178]: from='client.36864 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:36:51.448 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:51.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:51.554 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:51.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 95567 95551 79.6063 46.0625 0.0165091 0.0125574 2023-10-09T06:36:51.647 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:51.823 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:51.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:51.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:51.912 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:51.912 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:36:51.912 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:36:51.912 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:36:51.912 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:51.935 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:52.005 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:52.119 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:52.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:52.224 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:52.258 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:36:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:36:52] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T06:36:52.261 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:52.577 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 96145 96129 79.0342 36.125 0.0117443 0.012648 2023-10-09T06:36:52.588 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:52.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:52.614 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:52.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:52.769 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:52.836 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:36:52.889 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:52.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:52.986 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:36:53.019 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:53.093 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:53.144 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:53 smithi107 ceph-mon[212750]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:36:53.144 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:53 smithi150 ceph-mon[194178]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:36:53.148 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:53 smithi107 ceph-mon[220921]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2023-10-09T06:36:53.500 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:53.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 96890 96874 78.6125 46.5625 0.00415904 0.0126946 2023-10-09T06:36:53.665 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:53.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:53.771 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:53.836 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:54.054 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:54.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:54.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:54.128 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:54.160 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:54.182 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:36:54.210 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:54.354 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2692369687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.354 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.355 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2692369687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.355 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2692369687' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:36:54.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 97678 97662 78.2361 49.25 0.007843 0.0127605 2023-10-09T06:36:54.744 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:54.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:54.849 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:54.877 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:55.084 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:55.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:36:55.168 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 13 pg_num to 26 2023-10-09T06:36:55.202 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:55.298 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:55.489 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: pgmap v1385: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-10-09T06:36:55.489 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/469814564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.490 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.521 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: pgmap v1385: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/469814564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.530 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: pgmap v1385: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: osdmap e452: 8 total, 8 up, 8 in 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: failed to encode map e452 with expected crc 2023-10-09T06:36:55.531 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/469814564' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.532 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:36:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:55.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:55.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 98353 98337 77.7798 42.1875 0.00717571 0.0128512 2023-10-09T06:36:55.638 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:55.692 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:55.817 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:36:55.817 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:36:55.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67291 11.9 11:969e6fb0:::benchmark_data_smithi107_418363_object67290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 570.92268404499998, 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2541491790000001, 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.818 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67291 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875232+0000", 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875230+0000", 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875284+0000", 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3637999999999997e-05 2023-10-09T06:36:55.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875295+0000", 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1396000000000001e-05 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875303+0000", 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1629999999999996e-06 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875356+0000", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2541999999999999e-05 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875364+0000", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6119999999999999e-06 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872682+0000", 2023-10-09T06:36:55.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99731777899999996 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872691+0000", 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.619e-06 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.128229+0000", 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.255537956 2023-10-09T06:36:55.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.129381+0000", 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011520840000000001 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67302 11.9 11:93267d49:::benchmark_data_smithi107_418363_object67301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:36:55.824 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 570.92259908400001, 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2727547110000001, 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67302 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875316+0000", 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:55.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875345+0000", 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9431e-05 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875351+0000", 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5110000000000003e-06 2023-10-09T06:36:55.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875356+0000", 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875401+0000", 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4462999999999998e-05 2023-10-09T06:36:55.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875409+0000", 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3569999999999998e-06 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872710+0000", 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99730035399999994 2023-10-09T06:36:55.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.872716+0000", 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1979999999999998e-06 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.129415+0000", 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2566990149999999 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for degraded object", 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.130278+0000", 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00086301399999999995 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.147994+0000", 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.017716686999999998 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.148071+0000", 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6717000000000004e-05 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67308 11.a 11.16ff13ca (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 570.92255096999997, 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 11.74595132, 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.832 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67308 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875365+0000", 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875364+0000", 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:55.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875439+0000", 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4506999999999997e-05 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875445+0000", 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2570000000000004e-06 2023-10-09T06:36:55.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875448+0000", 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4589999999999998e-06 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875483+0000", 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4714e-05 2023-10-09T06:36:55.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.875489+0000", 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4060000000000004e-06 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871373+0000", 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99588437299999999 2023-10-09T06:36:55.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871384+0000", 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1302e-05 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109463+0000", 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 10.238078988 2023-10-09T06:36:55.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109475+0000", 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2072000000000001e-05 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621303+0000", 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51182799400000001 2023-10-09T06:36:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621316+0000", 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2662e-05 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67311 11.1 11:8e63c8cd:::benchmark_data_smithi107_418363_object67310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 570.90975177400003, 2023-10-09T06:36:55.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2810258819999998, 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67311 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.842 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888164+0000", 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888162+0000", 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888216+0000", 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3372000000000003e-05 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888243+0000", 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7721000000000001e-05 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888266+0000", 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2503000000000001e-05 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888334+0000", 2023-10-09T06:36:55.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8200000000000004e-05 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.888343+0000", 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5529999999999993e-06 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.874624+0000", 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98628109200000003 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.874633+0000", 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6120000000000005e-06 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158060+0000", 2023-10-09T06:36:55.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2834271070000001 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158218+0000", 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015740799999999999 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2", 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.158398+0000", 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000179925 2023-10-09T06:36:55.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166665+0000", 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0082676780000000005 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166676+0000", 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0584000000000001e-05 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166718+0000", 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2141000000000002e-05 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.166800+0000", 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1827000000000001e-05 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.169190+0000", 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023897020000000001 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67312 11.a 11.25d90f4a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 570.90890223400004, 2023-10-09T06:36:55.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 11.732321325000001, 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67312 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889013+0000", 2023-10-09T06:36:55.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889078+0000", 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5513999999999996e-05 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889088+0000", 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9129999999999998e-06 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889092+0000", 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7459999999999998e-06 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889132+0000", 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9773e-05 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.889139+0000", 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6850000000000001e-06 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.873695+0000", 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98455533500000003 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.873705+0000", 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0027000000000001e-05 2023-10-09T06:36:55.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109514+0000", 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 10.235808864999999 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109522+0000", 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0639999999999994e-06 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621331+0000", 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51180884800000004 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621335+0000", 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0990000000000001e-06 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67318 11.a 11.c6b30b6a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.74355497500005, 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5669876509999998, 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.859 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67318 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054361+0000", 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054359+0000", 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054498+0000", 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013886700000000001 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054512+0000", 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4354e-05 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054522+0000", 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9839999999999996e-06 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109546+0000", 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0550239999999995 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109550+0000", 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.569e-06 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621345+0000", 2023-10-09T06:36:55.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51179456199999995 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621348+0000", 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.704e-06 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67327 11.a 11.c2bd3d8a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.63877976100002, 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4622259500000006, 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67327 2023-10-09T06:36:55.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159136+0000", 2023-10-09T06:36:55.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159135+0000", 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159191+0000", 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6713999999999999e-05 2023-10-09T06:36:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159202+0000", 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0988000000000001e-05 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.159207+0000", 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5809999999999998e-06 2023-10-09T06:36:55.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109572+0000", 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9503646000000003 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109575+0000", 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4240000000000002e-06 2023-10-09T06:36:55.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621358+0000", 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51178295900000004 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621362+0000", 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7419999999999999e-06 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.870 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.871 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67328 11.a 11.cf83d70a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.63536396200004, 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4588306979999999, 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.872 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67328 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162552+0000", 2023-10-09T06:36:55.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162550+0000", 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162624+0000", 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3619000000000001e-05 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162635+0000", 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0716999999999999e-05 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.162642+0000", 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2030000000000003e-06 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109597+0000", 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9469555560000007 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109601+0000", 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9090000000000002e-06 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621376+0000", 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51177449799999997 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621382+0000", 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4010000000000002e-06 2023-10-09T06:36:55.877 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67353 11.a 11.6050658a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.54480601199998, 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3682885060000007, 2023-10-09T06:36:55.878 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67353 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253110+0000", 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253108+0000", 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253152+0000", 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3838999999999998e-05 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253166+0000", 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4202999999999999e-05 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.253176+0000", 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4520000000000003e-06 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109622+0000", 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8564464330000003 2023-10-09T06:36:55.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109625+0000", 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000002e-06 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621394+0000", 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51176914200000001 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.883 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621398+0000", 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6100000000000002e-06 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67375 11.a 11.25a7be1a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.48893100400005, 2023-10-09T06:36:55.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3124281040000003, 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67375 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:36:55.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308985+0000", 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.308983+0000", 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309029+0000", 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6217999999999997e-05 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309045+0000", 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6220000000000001e-05 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.309052+0000", 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7970000000000003e-06 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109646+0000", 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8005935120000007 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109649+0000", 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9550000000000001e-06 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621408+0000", 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51175970999999998 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621413+0000", 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2930000000000002e-06 2023-10-09T06:36:55.889 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67383 11.a 11.e2ca7dea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.481069069, 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3045890759999992, 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.890 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67383 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:36:55.891 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316846+0000", 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316845+0000", 2023-10-09T06:36:55.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316884+0000", 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8518e-05 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316897+0000", 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3210000000000001e-05 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.316904+0000", 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0280000000000003e-06 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109669+0000", 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7927653879999994 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109672+0000", 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0910000000000001e-06 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621429+0000", 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51175660300000003 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621436+0000", 2023-10-09T06:36:55.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5640000000000002e-06 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67400 11.a 11.aa0c4bfa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.45721102200002, 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2807484280000008, 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.896 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67400 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340705+0000", 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340703+0000", 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.898 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340786+0000", 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2757999999999996e-05 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340804+0000", 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7740999999999998e-05 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.340812+0000", 2023-10-09T06:36:55.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2530000000000004e-06 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109692+0000", 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7688797350000005 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109695+0000", 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.631e-06 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621449+0000", 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.511753817 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621453+0000", 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6569999999999999e-06 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67408 11.a 11.11553a3a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.44582836799998, 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2693790020000009, 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.902 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67408 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352087+0000", 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352141+0000", 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4262999999999999e-05 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352153+0000", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2035000000000001e-05 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.352158+0000", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9289999999999997e-06 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109715+0000", 2023-10-09T06:36:55.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.757557061 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109718+0000", 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1810000000000001e-06 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621463+0000", 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.511744529 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621466+0000", 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5140000000000002e-06 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67417 11.a 11.b16a0e7a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:36:55.907 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.42693024899995, 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2504945739999993, 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67417 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370985+0000", 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.370984+0000", 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371054+0000", 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9986000000000004e-05 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371068+0000", 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3971999999999999e-05 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.371075+0000", 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3050000000000004e-06 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109738+0000", 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7386627840000006 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109742+0000", 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7689999999999998e-06 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621476+0000", 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51173471999999998 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621480+0000", 2023-10-09T06:36:55.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5659999999999998e-06 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67437 11.a 11.396f3c4a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.38782732200002, 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2114045979999997, 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.913 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67437 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410088+0000", 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410086+0000", 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410124+0000", 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7249999999999997e-05 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410138+0000", 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4321000000000001e-05 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.916 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.410145+0000", 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5030000000000002e-06 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109761+0000", 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6996166180000003 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109764+0000", 2023-10-09T06:36:55.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.932e-06 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621489+0000", 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51172516300000004 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621493+0000", 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6030000000000001e-06 2023-10-09T06:36:55.918 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67497 11.a 11.30b451da (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.26811265599997, 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0917029520000003, 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.919 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67497 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529803+0000", 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529802+0000", 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529851+0000", 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9054999999999997e-05 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529862+0000", 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1141000000000001e-05 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.529866+0000", 2023-10-09T06:36:55.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1500000000000001e-06 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109784+0000", 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5799179349999992 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109787+0000", 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9170000000000002e-06 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.923 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621502+0000", 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51171517700000002 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621506+0000", 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4879999999999999e-06 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67524 11.a 11.b059deda (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.20416185299996, 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0277651169999995, 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.925 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67524 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593754+0000", 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593752+0000", 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593814+0000", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2033999999999996e-05 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593825+0000", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1216999999999999e-05 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.593830+0000", 2023-10-09T06:36:55.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1309999999999997e-06 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109806+0000", 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5159759810000004 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109809+0000", 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1259999999999998e-06 2023-10-09T06:36:55.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621515+0000", 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51170579199999999 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621519+0000", 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.692e-06 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.930 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67531 11.a 11.e295152a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 568.18062788199995, 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0042557280000004, 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.931 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67531 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.932 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617288+0000", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617286+0000", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617336+0000", 2023-10-09T06:36:55.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9450000000000003e-05 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617354+0000", 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8726e-05 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.617362+0000", 2023-10-09T06:36:55.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4410000000000004e-06 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109829+0000", 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4924671380000003 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.109833+0000", 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6780000000000002e-06 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621535+0000", 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.51170254999999998 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:36.621543+0000", 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3489999999999992e-06 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:125509 11.1 11:80899c97:::benchmark_data_smithi107_418363_object35073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 472.45454770700002, 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.167635185, 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.937 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 125509 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343368+0000", 2023-10-09T06:36:55.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343366+0000", 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343374+0000", 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5190000000000003e-06 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343387+0000", 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2782000000000001e-05 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343394+0000", 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2559999999999996e-06 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343702+0000", 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00030755499999999999 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.343709+0000", 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4909999999999999e-06 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322829+0000", 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97911982099999995 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322840+0000", 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.128e-05 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.483725+0000", 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1608853180000001 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.483843+0000", 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117111 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2,3", 2023-10-09T06:36:55.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.484379+0000", 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00053623600000000005 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493120+0000", 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0087413509999999996 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493251+0000", 2023-10-09T06:36:55.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000130691 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.508110+0000", 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014858949 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.510802+0000", 2023-10-09T06:36:55.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002692046 2023-10-09T06:36:55.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:36:55.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.510869+0000", 2023-10-09T06:36:55.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7124000000000003e-05 2023-10-09T06:36:55.950 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.950 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.950 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.950 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.511003+0000", 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000134101 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.951 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:125522 11.1 11:8d066771:::benchmark_data_smithi107_418363_object35086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 472.41985384600002, 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1353189160000001, 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:36:55.952 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 125522 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378062+0000", 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378060+0000", 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378069+0000", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1239999999999994e-06 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378090+0000", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1047e-05 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378098+0000", 2023-10-09T06:36:55.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6480000000000003e-06 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378180+0000", 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1672e-05 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:03.378192+0000", 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.224e-05 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322866+0000", 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94467452600000001 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:04.322873+0000", 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.165e-06 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488489+0000", 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.165615474 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488641+0000", 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015251699999999999 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,2,3", 2023-10-09T06:36:55.958 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.488840+0000", 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000198171 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.493124+0000", 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0042848590000000002 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.497914+0000", 2023-10-09T06:36:55.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047894360000000002 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.511045+0000", 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.013131267 2023-10-09T06:36:55.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513239+0000", 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021935280000000001 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513284+0000", 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5090999999999999e-05 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:36:55.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:29:05.513381+0000", 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6842999999999994e-05 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:36:55.962 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:36:56.309 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:56.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:56.415 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:56.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:36:56.578840+0000 min lat: 0.00116741 max lat: 0.348861 avg lat: 0.0129061 2023-10-09T06:36:56.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:36:56.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 98803 98787 77.1592 28.125 0.216191 0.0129061 2023-10-09T06:36:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.604 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:36:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T06:36:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.605 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:56 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:36:56.645 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:56.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:36:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: osdmap e453: 8 total, 8 up, 8 in 2023-10-09T06:36:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: failed to encode map e453 with expected crc 2023-10-09T06:36:56.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:56 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:36:56.697 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:56.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:56.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:56.710 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:36:56.804 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:56.832 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:56.873 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:56.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:56.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:56.980 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:57.008 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:57.114 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:36:57.153 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:57.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:57.259 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:36:57.336 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:57.375 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:36:57.396 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:36:57.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: pgmap v1388: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: pgmap v1388: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T06:36:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:36:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T06:36:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:57 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.405 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:36:57.420 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: pgmap v1388: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: osdmap e454: 8 total, 8 up, 8 in 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.421 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:57 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:57.429 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:36:57.435 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:36:57.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:57.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:36:57.468 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:36:57.530 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:36:57.539 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:57.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 99193 99177 76.5076 24.375 0.00836408 0.0130667 2023-10-09T06:36:57.587 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:36:57.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:57.640 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:36:57.680 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.688 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:36:57.693 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:57.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:57.800 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:57.850 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:57.938 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:57.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:58.045 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:36:58.070 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:58.535 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.542 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1690134864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:58.543 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[220921]: 13.4 scrub starts 2023-10-09T06:36:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:58.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:36:58.544 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.544 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.544 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.544 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1690134864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:58.545 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:58 smithi107 ceph-mon[212750]: 13.4 scrub starts 2023-10-09T06:36:58.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 100041 100025 76.2208 53 0.0128691 0.013115 2023-10-09T06:36:58.644 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e454 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: osdmap e455: 8 total, 8 up, 8 in 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: failed to encode map e455 with expected crc 2023-10-09T06:36:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1690134864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:36:58.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:58 smithi150 ceph-mon[194178]: 13.4 scrub starts 2023-10-09T06:36:58.722 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:58.784 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:36:58.828 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:36:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:58.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:36:58.937 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:36:58.984 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:59.077 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:36:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:59.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:36:59.176 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:36:59.176 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:36:59.182 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:36:59.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:59 smithi107 ceph-mon[212750]: pgmap v1391: 129 pgs: 10 unknown, 119 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s 2023-10-09T06:36:59.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:36:59 smithi107 ceph-mon[212750]: 13.3 scrub starts 2023-10-09T06:36:59.234 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:59 smithi107 ceph-mon[220921]: pgmap v1391: 129 pgs: 10 unknown, 119 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s 2023-10-09T06:36:59.234 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:36:59 smithi107 ceph-mon[220921]: 13.3 scrub starts 2023-10-09T06:36:59.242 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:59.481 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:36:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:59.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:36:59.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 101116 101100 76.1119 67.1875 0.0214638 0.0131342 2023-10-09T06:36:59.593 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:36:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:59 smithi150 ceph-mon[194178]: pgmap v1391: 129 pgs: 10 unknown, 119 active+clean; 5.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 3.4 MiB/s wr, 54 op/s 2023-10-09T06:36:59.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:36:59 smithi150 ceph-mon[194178]: 13.3 scrub starts 2023-10-09T06:36:59.655 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:59.724 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:36:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:59.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:36:59.831 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:36:59.903 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:36:59.960 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:36:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:36:59.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:00.064 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.092 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:00.147 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:00.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:00.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:00.300 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:00.377 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.3 scrub ok 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.13 scrub starts 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.4 scrub ok 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.9 scrub starts 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.0 scrub starts 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:37:00.380 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[212750]: 13.0 scrub ok 2023-10-09T06:37:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:00.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:00.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.3 scrub ok 2023-10-09T06:37:00.381 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.13 scrub starts 2023-10-09T06:37:00.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.4 scrub ok 2023-10-09T06:37:00.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.9 scrub starts 2023-10-09T06:37:00.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.0 scrub starts 2023-10-09T06:37:00.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:37:00.382 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:00 smithi107 ceph-mon[220921]: 13.0 scrub ok 2023-10-09T06:37:00.482 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:00.523 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:00.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 101980 101964 75.8487 54 0.013846 0.0131795 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.3 scrub ok 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.13 scrub starts 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.4 scrub ok 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.9 scrub starts 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.0 scrub starts 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:37:00.603 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:00 smithi150 ceph-mon[194178]: 13.0 scrub ok 2023-10-09T06:37:00.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:37:00.645 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:37:00.715 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:00.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:00.822 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:00.884 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:00.981 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:00.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:01.087 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:01.159 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:01.259 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:01.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:01.342 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:01.354 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:37:01.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[212750]: pgmap v1392: 129 pgs: 1 unknown, 128 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 583 op/s; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[212750]: 13.13 scrub ok 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[212750]: 13.5 scrub starts 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[212750]: 13.7 scrub starts 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[220921]: pgmap v1392: 129 pgs: 1 unknown, 128 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 583 op/s; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[220921]: 13.13 scrub ok 2023-10-09T06:37:01.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[220921]: 13.5 scrub starts 2023-10-09T06:37:01.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:01 smithi107 ceph-mon[220921]: 13.7 scrub starts 2023-10-09T06:37:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:01 smithi150 ceph-mon[194178]: pgmap v1392: 129 pgs: 1 unknown, 128 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 36 MiB/s wr, 583 op/s; 12 KiB/s, 0 objects/s recovering 2023-10-09T06:37:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:01 smithi150 ceph-mon[194178]: 13.13 scrub ok 2023-10-09T06:37:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:01 smithi150 ceph-mon[194178]: 13.5 scrub starts 2023-10-09T06:37:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:01 smithi150 ceph-mon[194178]: 13.7 scrub starts 2023-10-09T06:37:01.424 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:01.558 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:01.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:01.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 103044 103028 75.7386 66.5 0.0102993 0.013199 2023-10-09T06:37:01.670 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:01.736 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:01.786 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:37:01.786 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:37:01.787 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:37:01.787 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.167250 2023-10-09T06:37:01.787 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 0.16724953001780318 2023-10-09T06:37:02.093 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:02.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:02.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:02.196 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:37:02.202 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:02.229 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:02] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[212750]: 13.14 scrub starts 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3235748963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[220921]: 13.14 scrub starts 2023-10-09T06:37:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3235748963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:37:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:02 smithi150 ceph-mon[194178]: 13.14 scrub starts 2023-10-09T06:37:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3235748963' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:02.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 103825 103809 75.4255 48.8125 0.0128298 0.0132463 2023-10-09T06:37:02.714 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:02.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:02.819 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:02.847 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:02.889 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:02.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:02.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:02.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:03.031 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: pgmap v1393: 129 pgs: 129 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: 13.5 scrub ok 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: pgmap v1393: 129 pgs: 129 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: 13.5 scrub ok 2023-10-09T06:37:03.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:03 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.427 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:03.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:03.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:03.531 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:03.562 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:03.569 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: pgmap v1393: 129 pgs: 129 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:03.569 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: 13.5 scrub ok 2023-10-09T06:37:03.569 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: osdmap e456: 8 total, 8 up, 8 in 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e456 with expected crc 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: osdmap e457: 8 total, 8 up, 8 in 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.570 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.571 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.571 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.571 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:03 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:03.578 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 104164 104148 74.8022 21.1875 0.00404742 0.0132399 2023-10-09T06:37:03.667 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:03.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:03.769 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:03.770 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:03.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:03.799 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:03.810 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:03.850 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:03.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:03.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:03.911 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:37:03.929 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:37:03.948 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:37:03.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:03.966 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:37:03.983 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:03.986 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:37:04.007 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:37:04.025 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:37:04.029 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:04.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:04.045 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.047 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:37:04.133 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:04.159 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:04.201 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:04.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:04.305 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:04.332 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:04.374 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:04.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:04.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: 13.14 scrub ok 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: 13.8 scrub starts 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: 13.8 scrub ok 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1556221093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:04.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: 13.14 scrub ok 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: 13.8 scrub starts 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: 13.8 scrub ok 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: failed to encode map e458 with expected crc 2023-10-09T06:37:04.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1556221093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:04.478 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: failed to encode map e457 with expected crc 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: 13.14 scrub ok 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: 13.8 scrub starts 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: 13.8 scrub ok 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: osdmap e458: 8 total, 8 up, 8 in 2023-10-09T06:37:04.509 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: failed to encode map e458 with expected crc 2023-10-09T06:37:04.510 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1556221093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:04.511 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:04.555 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:04.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:04.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:04.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 104245 104229 74.0098 5.0625 0.00931035 0.0135071 2023-10-09T06:37:04.664 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:04.699 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:04.902 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:37:05.315 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:05.318 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[212750]: pgmap v1396: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[212750]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[220921]: pgmap v1396: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[220921]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[220921]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.319 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:05 smithi107 ceph-mon[220921]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:05.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:05.421 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:05.478 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:05 smithi150 ceph-mon[194178]: pgmap v1396: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 10 KiB/s, 0 objects/s recovering 2023-10-09T06:37:05.478 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:05 smithi150 ceph-mon[194178]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.478 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:05 smithi150 ceph-mon[194178]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.478 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:05 smithi150 ceph-mon[194178]: failed to encode map e458 with expected crc 2023-10-09T06:37:05.485 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:05.509 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:37:05.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 105174 105158 73.8306 58.0625 0.0149149 0.0135397 2023-10-09T06:37:05.604 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:05.608 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:37:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:05.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:05.723 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:05.776 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:05.986 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:05.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:06.096 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:06.171 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:06.276 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:06.336 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:06.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:06.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:06.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:37:06.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:37:06.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:37:06.443 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:06.538 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:06.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 90 12 106104 106092 73.6588 58.375 0.00564758 0.0135682 2023-10-09T06:37:06.600 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:06.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:06.704 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:06.732 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:06.775 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:06.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:06.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:06.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:06.904 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:07.211 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:07.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:07.234 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.5 primary-affinity to 0.16725 (8109602) 2023-10-09T06:37:07.321 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:07.348 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[220921]: pgmap v1398: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[220921]: 13.11 scrub starts 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/645944903' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[212750]: pgmap v1398: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[212750]: 13.11 scrub starts 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/645944903' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.447 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:07.449 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:07 smithi150 ceph-mon[194178]: pgmap v1398: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:07.449 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:07 smithi150 ceph-mon[194178]: 13.11 scrub starts 2023-10-09T06:37:07.449 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/645944903' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.450 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]: dispatch 2023-10-09T06:37:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:07.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:07.549 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:37:07.551 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0614 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 106104 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 73.6331 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 35.64 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 207.938 2023-10-09T06:37:07.579 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 5.0625 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1178 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 570.24 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 3327 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 81 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0135752 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0299285 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.21513 2023-10-09T06:37:07.580 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.00116741 2023-10-09T06:37:07.580 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:07.584 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:37:07.661 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:07.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67292 11.5 11:ae93ee2a:::benchmark_data_smithi107_418363_object67291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:37:07.738 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 583.00161199199999, 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3007185510000001, 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67292 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.739 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733750+0000", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733749+0000", 2023-10-09T06:37:07.740 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733812+0000", 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3002000000000004e-05 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733823+0000", 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0909e-05 2023-10-09T06:37:07.741 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733828+0000", 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6720000000000003e-06 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733869+0000", 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1508999999999998e-05 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.742 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.733970+0000", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100907 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.734066+0000", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5404999999999999e-05 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.735966+0000", 2023-10-09T06:37:07.743 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001899916 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.736151+0000", 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018566700000000001 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872581+0000", 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13643008600000001 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.744 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872592+0000", 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0746e-05 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871235+0000", 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99864300699999997 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.745 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871251+0000", 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5563e-05 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034370+0000", 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631196100000001 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034469+0000", 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8395999999999998e-05 2023-10-09T06:37:07.746 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67301 11.5 11:a1b1512d:::benchmark_data_smithi107_418363_object67300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 582.88710882999999, 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1863352649999999, 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.747 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67301 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848253+0000", 2023-10-09T06:37:07.748 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848252+0000", 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848308+0000", 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5899999999999997e-05 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.749 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848320+0000", 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1922e-05 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848326+0000", 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2099999999999998e-06 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.750 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848378+0000", 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1793999999999999e-05 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848521+0000", 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142769 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.848677+0000", 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000155828 2023-10-09T06:37:07.751 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.850815+0000", 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021378920000000002 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852117+0000", 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00130197 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.752 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872639+0000", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.020522744999999998 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872646+0000", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7649999999999996e-06 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871333+0000", 2023-10-09T06:37:07.753 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99868642699999999 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871343+0000", 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0808999999999999e-05 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034505+0000", 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1631621480000001 2023-10-09T06:37:07.754 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034589+0000", 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3185999999999999e-05 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67305 11.5 11:aa961c7b:::benchmark_data_smithi107_418363_object67304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:37:07.755 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 582.88304203300004, 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.182389261, 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67305 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.756 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852320+0000", 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.757 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852319+0000", 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852365+0000", 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6390999999999997e-05 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852381+0000", 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5128e-05 2023-10-09T06:37:07.758 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.852388+0000", 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2640000000000002e-06 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853387+0000", 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00099880900000000002 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.759 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853487+0000", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100614 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.853585+0000", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7887999999999995e-05 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.855632+0000", 2023-10-09T06:37:07.760 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020469289999999998 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.857531+0000", 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0018985919999999999 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872679+0000", 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015148790000000001 2023-10-09T06:37:07.761 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872686+0000", 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.922e-06 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871386+0000", 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99869956000000004 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.762 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871393+0000", 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2060000000000001e-06 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034627+0000", 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1632337100000001 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.763 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034709+0000", 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2608000000000006e-05 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67306 11.5 11:aea7a0bf:::benchmark_data_smithi107_418363_object67305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 582.87081638500001, 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.17028496, 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.764 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67306 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.765 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864546+0000", 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864545+0000", 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.766 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864595+0000", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0657e-05 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864605+0000", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9380000000000004e-06 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864610+0000", 2023-10-09T06:37:07.767 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0130000000000001e-06 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.864660+0000", 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0006000000000002e-05 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865068+0000", 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00040736199999999998 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.768 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,6", 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.865233+0000", 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000165496 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872957+0000", 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0077238020000000001 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.769 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.872962+0000", 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2599999999999996e-06 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871429+0000", 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99846622200000001 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871438+0000", 2023-10-09T06:37:07.770 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8109999999999997e-06 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034745+0000", 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1633062599999999 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034831+0000", 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6227e-05 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.771 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.45989.0:67309 11.5 11:af8d0774:::benchmark_data_smithi107_418363_object67308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 582.85600259800003, 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.174921791, 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.45989", 2023-10-09T06:37:07.772 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 67309 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879360+0000", 2023-10-09T06:37:07.773 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879358+0000", 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879394+0000", 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5299000000000002e-05 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.774 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879411+0000", 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7544999999999999e-05 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879417+0000", 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.784e-06 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.775 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879464+0000", 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6631999999999997e-05 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:24.879484+0000", 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0475999999999998e-05 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871466+0000", 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.99198237899999997 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.776 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:25.871472+0000", 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6790000000000002e-06 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.034876+0000", 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1634033669999999 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.777 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035033+0000", 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000157217 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,6", 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.035259+0000", 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022594499999999999 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041596+0000", 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0063371349999999998 2023-10-09T06:37:07.778 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.041946+0000", 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00035007099999999998 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051163+0000", 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0092172039999999997 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.779 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.051212+0000", 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8730999999999998e-05 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:27:27.054281+0000", 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030695850000000001 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.780 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6275 12.d 12:b8b44611:::benchmark_data_smithi107_479504_object6274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 353.70997000599999, 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267782297, 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6275 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.781 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.782 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025391+0000", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025438+0000", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7531999999999998e-05 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025454+0000", 2023-10-09T06:37:07.783 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5452e-05 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025461+0000", 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1380000000000001e-06 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025514+0000", 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2778999999999997e-05 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.784 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025621+0000", 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010712 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025721+0000", 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100147 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.785 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027384+0000", 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0016627790000000001 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027404+0000", 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0047999999999998e-05 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293102+0000", 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265697933 2023-10-09T06:37:07.786 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293139+0000", 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7493000000000002e-05 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293175+0000", 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5145999999999999e-05 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.787 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6278 12.d 12:be6c2ec3:::benchmark_data_smithi107_479504_object6277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 353.709867616, 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267768063, 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.788 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6278 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:37:07.789 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025494+0000", 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025557+0000", 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2922000000000005e-05 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.790 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025575+0000", 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7739000000000001e-05 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025579+0000", 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0400000000000003e-06 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.791 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026133+0000", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055439499999999995 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026261+0000", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127372 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026428+0000", 2023-10-09T06:37:07.792 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016770500000000001 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027443+0000", 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001015099 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027452+0000", 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6920000000000008e-06 2023-10-09T06:37:07.793 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293213+0000", 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265760789 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293242+0000", 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9085e-05 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.794 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293263+0000", 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0744999999999999e-05 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6279 12.d 12:bf6b6632:::benchmark_data_smithi107_479504_object6278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:37:07.795 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 353.70977593700002, 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26777375199999998, 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6279 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.796 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:37:07.797 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025640+0000", 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4079999999999997e-05 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025645+0000", 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0640000000000001e-06 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.798 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025648+0000", 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0460000000000001e-06 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026860+0000", 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001212252 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.799 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026986+0000", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012579999999999999 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027075+0000", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8715999999999996e-05 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027532+0000", 2023-10-09T06:37:07.800 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045738199999999999 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027923+0000", 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00039128200000000001 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293289+0000", 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26536579300000002 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.801 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293328+0000", 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8287000000000001e-05 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293360+0000", 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2518000000000003e-05 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.802 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:37240 12.d 12:bb21ad69:::benchmark_data_smithi107_479504_object37239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 335.88731792599998, 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25148841100000002, 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 37240 2023-10-09T06:37:07.803 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.804 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848043+0000", 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848103+0000", 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9973999999999999e-05 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.805 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848112+0000", 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7090000000000008e-06 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848117+0000", 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3640000000000001e-06 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848144+0000", 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7644000000000002e-05 2023-10-09T06:37:07.806 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848210+0000", 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5368000000000002e-05 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848283+0000", 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3858999999999999e-05 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.807 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849143+0000", 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000859705 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097694+0000", 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24855127599999999 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.808 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099429+0000", 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001734759 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099492+0000", 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2785000000000003e-05 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099533+0000", 2023-10-09T06:37:07.809 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0816999999999999e-05 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46340.0:37240 12.d e367/361 ondisk, result = 0)", 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 335.88559440099999, 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24792936400000001, 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.810 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.811 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849767+0000", 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849771+0000", 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.523e-06 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.812 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849788+0000", 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6719e-05 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849792+0000", 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9400000000000004e-06 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097677+0000", 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24788569099999999 2023-10-09T06:37:07.813 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097687+0000", 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7059999999999999e-06 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097697+0000", 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0288e-05 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.814 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:76840 12.d 12:be36faa0:::benchmark_data_smithi107_479504_object76839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 295.98633479199998, 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.232798802, 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.815 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 76840 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:37:07.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749026+0000", 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749082+0000", 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5448000000000002e-05 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749094+0000", 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749100+0000", 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4060000000000004e-06 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749151+0000", 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1034e-05 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749306+0000", 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015525700000000001 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749503+0000", 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019740799999999999 2023-10-09T06:37:07.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.756545+0000", 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0070414650000000002 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.955708+0000", 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.199163272 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981744+0000", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026035632999999999 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981787+0000", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3210000000000001e-05 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981826+0000", 2023-10-09T06:37:07.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9178999999999999e-05 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:76844 12.d 12:b571eb92:::benchmark_data_smithi107_479504_object76843:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 295.98611664999999, 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23556622999999999, 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.822 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 76844 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749245+0000", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749318+0000", 2023-10-09T06:37:07.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2859000000000002e-05 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749325+0000", 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9809999999999997e-06 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749328+0000", 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5949999999999999e-06 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754650+0000", 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053215629999999996 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754748+0000", 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8474999999999995e-05 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754858+0000", 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000109648 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.756692+0000", 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001834086 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.957451+0000", 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20075878699999999 2023-10-09T06:37:07.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984727+0000", 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.027275995000000001 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984764+0000", 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7248e-05 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984812+0000", 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7599999999999998e-05 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:153016 12.d 12:b3683356:::benchmark_data_smithi107_479504_object53405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 188.73046194700001, 2023-10-09T06:37:07.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23512572800000001, 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 153016 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:37:07.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004899+0000", 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004906+0000", 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3010000000000001e-06 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004921+0000", 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5142e-05 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.832 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004928+0000", 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0589999999999997e-06 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004982+0000", 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3878999999999997e-05 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005090+0000", 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010767799999999999 2023-10-09T06:37:07.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005531+0000", 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044087999999999999 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.009097+0000", 2023-10-09T06:37:07.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0035658479999999999 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.012729+0000", 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036324439999999999 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.239962+0000", 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22723316399999999 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.239999+0000", 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6426e-05 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.240026+0000", 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7291999999999999e-05 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180927 12.d 12:b6ff457f:::benchmark_data_smithi107_479504_object81316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 148.986950654, 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.35203043499999997, 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180927 2023-10-09T06:37:07.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748411+0000", 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748415+0000", 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6779999999999999e-06 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748424+0000", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748429+0000", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3270000000000002e-06 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748471+0000", 2023-10-09T06:37:07.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2400000000000001e-05 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748563+0000", 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2088000000000003e-05 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748643+0000", 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9326000000000006e-05 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.750567+0000", 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019239070000000001 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.751954+0000", 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013873869999999999 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100346+0000", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.34839218799999999 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100401+0000", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4484999999999998e-05 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100442+0000", 2023-10-09T06:37:07.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1430000000000001e-05 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180945 12.d 12:b29197fc:::benchmark_data_smithi107_479504_object81334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 148.956002103, 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32532718199999999, 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.844 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180945 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779359+0000", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779366+0000", 2023-10-09T06:37:07.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8349999999999998e-06 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779380+0000", 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3978999999999999e-05 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779386+0000", 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2960000000000004e-06 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779426+0000", 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0438000000000001e-05 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779510+0000", 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3070000000000003e-05 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779582+0000", 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2076999999999994e-05 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.782438+0000", 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002856408 2023-10-09T06:37:07.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.786572+0000", 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041337539999999999 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104626+0000", 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31805433399999999 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104660+0000", 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3751000000000003e-05 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104687+0000", 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7487999999999998e-05 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98783 13.e 13:7e44f156:::benchmark_data_smithi107_535588_object98782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 11.596453233, 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23612880999999999, 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98783 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.853 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138907+0000", 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138985+0000", 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8372000000000001e-05 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138998+0000", 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.254e-05 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139004+0000", 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1290000000000001e-06 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139052+0000", 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8241000000000001e-05 2023-10-09T06:37:07.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139170+0000", 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117379 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139302+0000", 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013261800000000001 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145027+0000", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057249759999999997 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148343+0000", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033158129999999999 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374933+0000", 2023-10-09T06:37:07.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22659017100000001 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374992+0000", 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8931000000000002e-05 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.375038+0000", 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5528000000000002e-05 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.859 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 11.596339116999999, 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30779935400000002, 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:37:07.860 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:37:07.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139054+0000", 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000001e-05 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139062+0000", 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139066+0000", 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140814+0000", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001748793 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140929+0000", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.141055+0000", 2023-10-09T06:37:07.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126563 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.146811+0000", 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005755418 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446811+0000", 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29999972899999999 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446822+0000", 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1856e-05 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 11.480896791999999, 2023-10-09T06:37:07.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93419914299999995, 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254463+0000", 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254519+0000", 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6208999999999997e-05 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254547+0000", 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8109000000000001e-05 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.869 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254562+0000", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5073e-05 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254724+0000", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161974 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254729+0000", 2023-10-09T06:37:07.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497899+0000", 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24316990899999999 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497910+0000", 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0874e-05 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188593+0000", 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69068296699999998 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188665+0000", 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1802000000000001e-05 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98796 13.7 13:e44dc813:::benchmark_data_smithi107_535588_object98795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 11.379247185000001, 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84800129700000004, 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98796 2023-10-09T06:37:07.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356113+0000", 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4719000000000002e-05 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356184+0000", 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6277e-05 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356192+0000", 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9789999999999993e-06 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498051+0000", 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14185953700000001 2023-10-09T06:37:07.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498058+0000", 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2799999999999998e-06 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188683+0000", 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.690624601 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188792+0000", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108497 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188939+0000", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014728699999999999 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.191074+0000", 2023-10-09T06:37:07.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021353840000000002 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.203068+0000", 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011993634 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204019+0000", 2023-10-09T06:37:07.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000951625 2023-10-09T06:37:07.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204076+0000", 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6569e-05 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204116+0000", 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0346999999999998e-05 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.882 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98805 13.0 13:054831b9:::benchmark_data_smithi107_535588_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 11.049143240999999, 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.528813278, 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:37:07.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98805 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:37:07.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686214+0000", 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999952 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686248+0000", 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4001000000000002e-05 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686269+0000", 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0539000000000001e-05 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686278+0000", 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2469999999999997e-06 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686339+0000", 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0735e-05 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686348+0000", 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181836+0000", 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49548772400000002 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181965+0000", 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000129448 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.182343+0000", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037732500000000002 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.184651+0000", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023081030000000002 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.199956+0000", 2023-10-09T06:37:07.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015305375 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214937+0000", 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014981213 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214993+0000", 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5884999999999998e-05 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:37:07.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.215032+0000", 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9060999999999998e-05 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:37:07.891 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:37:07.892 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:07.918 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:08.231 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]': finished 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.232 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]': finished 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.233 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.234 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.234 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:08.234 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[220921]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:37:08.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:08.502 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:08 smithi107 ceph-mon[212750]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:37:08.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.16724953001780318}]': finished 2023-10-09T06:37:08.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: osdmap e459: 8 total, 8 up, 8 in 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: failed to encode map e459 with expected crc 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:08.658 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:08 smithi150 ceph-mon[194178]: osdmap e460: 8 total, 8 up, 8 in 2023-10-09T06:37:08.772 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:37:09.107 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:09.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:09.220 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:09.320 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: pgmap v1400: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[212750]: 13.17 scrub starts 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: pgmap v1400: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:09 smithi107 ceph-mon[220921]: 13.17 scrub starts 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: pgmap v1400: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 127 active+clean; 6.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: failed to encode map e460 with expected crc 2023-10-09T06:37:09.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:09 smithi150 ceph-mon[194178]: 13.17 scrub starts 2023-10-09T06:37:09.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:09.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:09.813 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:09.873 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:09.933 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:09.967 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:09.968 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:09.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:10.073 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:10.142 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:10.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:10.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:10.345 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:10.353 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:10 smithi107 ceph-mon[212750]: 13.17 scrub ok 2023-10-09T06:37:10.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:10 smithi107 ceph-mon[220921]: 13.17 scrub ok 2023-10-09T06:37:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:10 smithi150 ceph-mon[194178]: 13.17 scrub ok 2023-10-09T06:37:10.438 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:10.849 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:10.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:10.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:11.068 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:11.253 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: pgmap v1402: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/318270 objects misplaced (0.953%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:11.283 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:37:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2023-10-09T06:37:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: 13.12 scrub starts 2023-10-09T06:37:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:37:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: 13.18 scrub starts 2023-10-09T06:37:11.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: pgmap v1402: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/318270 objects misplaced (0.953%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: 13.12 scrub starts 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: 13.18 scrub starts 2023-10-09T06:37:11.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:37:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: pgmap v1402: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/318270 objects misplaced (0.953%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2023-10-09T06:37:11.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: 13.12 scrub starts 2023-10-09T06:37:11.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:37:11.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: 13.18 scrub starts 2023-10-09T06:37:11.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:12] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:37:12.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[212750]: 13.2 scrub starts 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:12 smithi107 ceph-mon[220921]: 13.2 scrub starts 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/394552418' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: osdmap e461: 8 total, 8 up, 8 in 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: failed to encode map e461 with expected crc 2023-10-09T06:37:12.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:12 smithi150 ceph-mon[194178]: 13.2 scrub starts 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[212750]: pgmap v1404: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/317832 objects misplaced (0.954%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[220921]: pgmap v1404: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/317832 objects misplaced (0.954%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:13 smithi150 ceph-mon[194178]: pgmap v1404: 129 pgs: 1 active, 1 activating, 1 active+recovering+undersized+remapped, 12 peering, 114 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 14 KiB/s rd, 7 op/s; 3033/317832 objects misplaced (0.954%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:37:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:13.884 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:13.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:13.998 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:14.068 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:14.163 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:14.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:14.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:14.273 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:14.328 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:37:14.329 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.329 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:14 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.333 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:14.437 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:14.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:14.546 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:14.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:37:14.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.614 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:37:14.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.617 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:14 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:14.620 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:14.731 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:37:14.733 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:14.735 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:37:14.735 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:37:14.736 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-10-09T06:37:14.736 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:37:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:14.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:14.743 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:14.743 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:14.775 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:37:14.809 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:37:14.839 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:37:14.844 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:14.872 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:37:14.900 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:14.903 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:37:14.948 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:37:14.981 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:37:14.986 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.986 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.986 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.986 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.987 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.987 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.987 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:37:14.987 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: pgmap v1405: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 13 KiB/s rd, 7 op/s; 2719/304839 objects misplaced (0.892%); 5.8 MiB/s, 91 objects/s recovering 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:37:15.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/630116444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: pgmap v1405: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 13 KiB/s rd, 7 op/s; 2719/304839 objects misplaced (0.892%); 5.8 MiB/s, 91 objects/s recovering 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:37:15.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/630116444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: pgmap v1405: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 13 KiB/s rd, 7 op/s; 2719/304839 objects misplaced (0.892%); 5.8 MiB/s, 91 objects/s recovering 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: failed to encode map e462 with expected crc 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: osdmap e462: 8 total, 8 up, 8 in 2023-10-09T06:37:15.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/630116444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:15.869 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:15.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:15.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:15.909 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:37:15.972 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:16.036 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[212750]: failed to encode map e463 with expected crc 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[212750]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[212750]: pgmap v1408: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 199 op/s; 2719/304839 objects misplaced (0.892%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[220921]: failed to encode map e463 with expected crc 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[220921]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[220921]: pgmap v1408: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 199 op/s; 2719/304839 objects misplaced (0.892%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:37:16.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:37:16.653 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:16 smithi150 ceph-mon[194178]: failed to encode map e463 with expected crc 2023-10-09T06:37:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:16 smithi150 ceph-mon[194178]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T06:37:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:16 smithi150 ceph-mon[194178]: pgmap v1408: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 199 op/s; 2719/304839 objects misplaced (0.892%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:37:16.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:37:16.677 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:16.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:16.782 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:16.828 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:16.896 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:16.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:16.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:17.004 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:17.035 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:37:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:37:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:17.423 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:17.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:17.527 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:17.558 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.569 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: failed to encode map e464 with expected crc 2023-10-09T06:37:17.570 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:17.596 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:37:17.606 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:17.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:17.709 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.743 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:17.744 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:17.782 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:17.815 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:17.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:17.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:17.950 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:18.058 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:18.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:18.162 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:18.191 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:18.237 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:18.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:18.340 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:18.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:37:18.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: pgmap v1411: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 2719/303858 objects misplaced (0.895%) 2023-10-09T06:37:18.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[212750]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: pgmap v1411: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 2719/303858 objects misplaced (0.895%) 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:18 smithi107 ceph-mon[220921]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.379 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: pgmap v1411: 129 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 127 active+clean; 6.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 2719/303858 objects misplaced (0.895%) 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: failed to encode map e465 with expected crc 2023-10-09T06:37:18.447 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:18 smithi150 ceph-mon[194178]: failed to encode map e465 with expected crc 2023-10-09T06:37:19.322 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:19.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:19 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:19.421 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:37:19.428 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:19.531 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:19.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T06:37:19.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: failed to encode map e466 with expected crc 2023-10-09T06:37:19.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:19.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:19 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.367 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:20.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:20.492 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:20.564 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[220921]: failed to encode map e466 with expected crc 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[220921]: pgmap v1413: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3342/296622 objects degraded (1.127%); 5526/296622 objects misplaced (1.863%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[212750]: failed to encode map e466 with expected crc 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[212750]: pgmap v1413: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3342/296622 objects degraded (1.127%); 5526/296622 objects misplaced (1.863%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T06:37:20.565 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.566 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:20 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.590 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:20 smithi150 ceph-mon[194178]: failed to encode map e466 with expected crc 2023-10-09T06:37:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:20 smithi150 ceph-mon[194178]: pgmap v1413: 129 pgs: 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 3342/296622 objects degraded (1.127%); 5526/296622 objects misplaced (1.863%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T06:37:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:20 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:20.752 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:20.798 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:20.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:20.902 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:20.983 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:21.357 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:21.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:21.466 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.520 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:21.604 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:21.618 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:21.618 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:21.618 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:21 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:21.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:21.638 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:37:21.649 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:37:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:21.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:21.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:37:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:37:21.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:21 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:21.720 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:21.781 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:21.883 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:21.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:21.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:22.000 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:22.018 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:22] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T06:37:22.067 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:22.135 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:22.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:22.244 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:22.291 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:22.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:22 smithi107 ceph-mon[220921]: pgmap v1414: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6611/295791 objects degraded (2.235%); 8795/295791 objects misplaced (2.973%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:37:22.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:22 smithi107 ceph-mon[220921]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:22.347 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:22.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:22.453 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:22.534 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:22 smithi107 ceph-mon[212750]: pgmap v1414: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6611/295791 objects degraded (2.235%); 8795/295791 objects misplaced (2.973%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:37:22.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:22 smithi107 ceph-mon[212750]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:22.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:22 smithi150 ceph-mon[194178]: pgmap v1414: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+undersized+remapped, 126 active+clean; 6.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 6611/295791 objects degraded (2.235%); 8795/295791 objects misplaced (2.973%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T06:37:22.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:22 smithi150 ceph-mon[194178]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:22.922 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:22.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:22.984 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:23.018 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1414,"stamp":"2023-10-09T06:37:21.466456+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6460532813,"num_objects":98597,"num_object_clones":0,"num_object_copies":295791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6611,"num_objects_misplaced":8795,"num_objects_unfound":0,"num_objects_dirty":98597,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":88009,"num_write_kb":5147960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1162,"num_bytes_recovered":72904562,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171917,"ondisk_log_size":171917,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":390,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":26245184,"kb_used_data":18968896,"kb_used_omap":20,"kb_used_meta":7276011,"kb_avail":723716032,"statfs":{"total":767960285184,"available":741085216768,"internally_reserved":0,"allocated":19424149504,"data_stored":19411762047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":7450635918},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-197656576,"num_objects":-3016,"num_object_clones":0,"num_object_copies":-9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6611,"num_objects_misplaced":6076,"num_objects_unfound":0,"num_objects_dirty":-3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3016,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":535,"num_bytes_recovered":35061760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001445"},"pg_stats":[{"pgid":"13.14","version":"464'6676","reported_seq":6863,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.090241+0000","last_change":"2023-10-09T06:37:02.769628+0000","last_active":"2023-10-09T06:37:17.090241+0000","last_peered":"2023-10-09T06:37:17.090241+0000","last_clean":"2023-10-09T06:37:17.090241+0000","last_became_active":"2023-10-09T06:36:57.240495+0000","last_became_peered":"2023-10-09T06:36:57.240495+0000","last_unstale":"2023-10-09T06:37:17.090241+0000","last_undegraded":"2023-10-09T06:37:17.090241+0000","last_fullsized":"2023-10-09T06:37:17.090241+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"457'6390","last_scrub_stamp":"2023-10-09T06:37:02.769552+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:02.769552+0000","objects_scrubbed":3298,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:36.138793+0000","scrub_duration":1.1657576549999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207749120,"num_objects":3170,"num_object_clones":0,"num_object_copies":9510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":300,"num_write_kb":5504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":728,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385389+0000","last_change":"2023-10-09T06:37:08.252438+0000","last_active":"2023-10-09T06:37:16.385389+0000","last_peered":"2023-10-09T06:37:16.385389+0000","last_clean":"2023-10-09T06:37:16.385389+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:37:16.385389+0000","last_undegraded":"2023-10-09T06:37:16.385389+0000","last_fullsized":"2023-10-09T06:37:16.385389+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:30.160439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:30.160439+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":444,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289192+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:37:16.289192+0000","last_peered":"2023-10-09T06:37:16.289192+0000","last_clean":"2023-10-09T06:37:16.289192+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:37:16.289192+0000","last_undegraded":"2023-10-09T06:37:16.289192+0000","last_fullsized":"2023-10-09T06:37:16.289192+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301446+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:37:17.301446+0000","last_peered":"2023-10-09T06:37:17.301446+0000","last_clean":"2023-10-09T06:37:17.301446+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:37:17.301446+0000","last_undegraded":"2023-10-09T06:37:17.301446+0000","last_fullsized":"2023-10-09T06:37:17.301446+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.17","version":"464'6552","reported_seq":6736,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.198086+0000","last_change":"2023-10-09T06:37:09.430507+0000","last_active":"2023-10-09T06:37:17.198086+0000","last_peered":"2023-10-09T06:37:17.198086+0000","last_clean":"2023-10-09T06:37:17.198086+0000","last_became_active":"2023-10-09T06:36:57.205319+0000","last_became_peered":"2023-10-09T06:36:57.205319+0000","last_unstale":"2023-10-09T06:37:17.198086+0000","last_undegraded":"2023-10-09T06:37:17.198086+0000","last_fullsized":"2023-10-09T06:37:17.198086+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"460'6376","last_scrub_stamp":"2023-10-09T06:37:09.430427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:09.430427+0000","objects_scrubbed":3242,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:20.773784+0000","scrub_duration":1.0485898069999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199950336,"num_objects":3051,"num_object_clones":0,"num_object_copies":9153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":191,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":698,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289132+0000","last_change":"2023-10-09T06:37:08.296717+0000","last_active":"2023-10-09T06:37:16.289132+0000","last_peered":"2023-10-09T06:37:16.289132+0000","last_clean":"2023-10-09T06:37:16.289132+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:37:16.289132+0000","last_undegraded":"2023-10-09T06:37:16.289132+0000","last_fullsized":"2023-10-09T06:37:16.289132+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":460,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.292990+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:37:16.292990+0000","last_peered":"2023-10-09T06:37:16.292990+0000","last_clean":"2023-10-09T06:37:16.292990+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:37:16.292990+0000","last_undegraded":"2023-10-09T06:37:16.292990+0000","last_fullsized":"2023-10-09T06:37:16.292990+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":465,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298216+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:37:17.298216+0000","last_peered":"2023-10-09T06:37:17.298216+0000","last_clean":"2023-10-09T06:37:17.298216+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:37:17.298216+0000","last_undegraded":"2023-10-09T06:37:17.298216+0000","last_fullsized":"2023-10-09T06:37:17.298216+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.16","version":"464'6579","reported_seq":6628,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.231764+0000","last_change":"2023-10-09T06:36:57.266957+0000","last_active":"2023-10-09T06:37:17.231764+0000","last_peered":"2023-10-09T06:37:17.231764+0000","last_clean":"2023-10-09T06:37:17.231764+0000","last_became_active":"2023-10-09T06:36:57.241947+0000","last_became_peered":"2023-10-09T06:36:57.241947+0000","last_unstale":"2023-10-09T06:37:17.231764+0000","last_undegraded":"2023-10-09T06:37:17.231764+0000","last_fullsized":"2023-10-09T06:37:17.231764+0000","mapping_epoch":454,"log_start":"441'300","ondisk_log_start":"441'300","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199262208,"num_objects":3041,"num_object_clones":0,"num_object_copies":9123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3527,"num_write_kb":210144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":712,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289086+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:37:16.289086+0000","last_peered":"2023-10-09T06:37:16.289086+0000","last_clean":"2023-10-09T06:37:16.289086+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:37:16.289086+0000","last_undegraded":"2023-10-09T06:37:16.289086+0000","last_fullsized":"2023-10-09T06:37:16.289086+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300144+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:37:17.300144+0000","last_peered":"2023-10-09T06:37:17.300144+0000","last_clean":"2023-10-09T06:37:17.300144+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:37:17.300144+0000","last_undegraded":"2023-10-09T06:37:17.300144+0000","last_fullsized":"2023-10-09T06:37:17.300144+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":454,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.393002+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:37:16.393002+0000","last_peered":"2023-10-09T06:37:16.393002+0000","last_clean":"2023-10-09T06:37:16.393002+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:37:16.393002+0000","last_undegraded":"2023-10-09T06:37:16.393002+0000","last_fullsized":"2023-10-09T06:37:16.393002+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"464'6586","reported_seq":6632,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.234605+0000","last_change":"2023-10-09T06:36:57.231121+0000","last_active":"2023-10-09T06:37:17.234605+0000","last_peered":"2023-10-09T06:37:17.234605+0000","last_clean":"2023-10-09T06:37:17.234605+0000","last_became_active":"2023-10-09T06:36:57.228244+0000","last_became_peered":"2023-10-09T06:36:57.228244+0000","last_unstale":"2023-10-09T06:37:17.234605+0000","last_undegraded":"2023-10-09T06:37:17.234605+0000","last_fullsized":"2023-10-09T06:37:17.234605+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200474624,"num_objects":3059,"num_object_clones":0,"num_object_copies":9177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3515,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":573,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385758+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:37:16.385758+0000","last_peered":"2023-10-09T06:37:16.385758+0000","last_clean":"2023-10-09T06:37:16.385758+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:37:16.385758+0000","last_undegraded":"2023-10-09T06:37:16.385758+0000","last_fullsized":"2023-10-09T06:37:16.385758+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":140,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298580+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:37:17.298580+0000","last_peered":"2023-10-09T06:37:17.298580+0000","last_clean":"2023-10-09T06:37:17.298580+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:37:17.298580+0000","last_undegraded":"2023-10-09T06:37:17.298580+0000","last_fullsized":"2023-10-09T06:37:17.298580+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300253+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:37:17.300253+0000","last_peered":"2023-10-09T06:37:17.300253+0000","last_clean":"2023-10-09T06:37:17.300253+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:37:17.300253+0000","last_undegraded":"2023-10-09T06:37:17.300253+0000","last_fullsized":"2023-10-09T06:37:17.300253+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.18","version":"464'6637","reported_seq":6693,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.225843+0000","last_change":"2023-10-09T06:37:10.566231+0000","last_active":"2023-10-09T06:37:17.225843+0000","last_peered":"2023-10-09T06:37:17.225843+0000","last_clean":"2023-10-09T06:37:17.225843+0000","last_became_active":"2023-10-09T06:37:08.253146+0000","last_became_peered":"2023-10-09T06:37:08.253146+0000","last_unstale":"2023-10-09T06:37:17.225843+0000","last_undegraded":"2023-10-09T06:37:17.225843+0000","last_fullsized":"2023-10-09T06:37:17.225843+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6637,"log_dups_size":0,"ondisk_log_size":6637,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203161600,"num_objects":3100,"num_object_clones":0,"num_object_copies":9300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3542,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":610,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386042+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:37:16.386042+0000","last_peered":"2023-10-09T06:37:16.386042+0000","last_clean":"2023-10-09T06:37:16.386042+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:37:16.386042+0000","last_undegraded":"2023-10-09T06:37:16.386042+0000","last_fullsized":"2023-10-09T06:37:16.386042+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.297168+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:37:16.297168+0000","last_peered":"2023-10-09T06:37:16.297168+0000","last_clean":"2023-10-09T06:37:16.297168+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:37:16.297168+0000","last_undegraded":"2023-10-09T06:37:16.297168+0000","last_fullsized":"2023-10-09T06:37:16.297168+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300767+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:37:17.300767+0000","last_peered":"2023-10-09T06:37:17.300767+0000","last_clean":"2023-10-09T06:37:17.300767+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:37:17.300767+0000","last_undegraded":"2023-10-09T06:37:17.300767+0000","last_fullsized":"2023-10-09T06:37:17.300767+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":581,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297957+0000","last_change":"2023-10-09T06:37:11.071873+0000","last_active":"2023-10-09T06:37:17.297957+0000","last_peered":"2023-10-09T06:37:17.297957+0000","last_clean":"2023-10-09T06:37:17.297957+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:37:17.297957+0000","last_undegraded":"2023-10-09T06:37:17.297957+0000","last_fullsized":"2023-10-09T06:37:17.297957+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:31.210137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:31.210137+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":346,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386108+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:37:16.386108+0000","last_peered":"2023-10-09T06:37:16.386108+0000","last_clean":"2023-10-09T06:37:16.386108+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:37:16.386108+0000","last_undegraded":"2023-10-09T06:37:16.386108+0000","last_fullsized":"2023-10-09T06:37:16.386108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":401,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.293712+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:37:16.293712+0000","last_peered":"2023-10-09T06:37:16.293712+0000","last_clean":"2023-10-09T06:37:16.293712+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:37:16.293712+0000","last_undegraded":"2023-10-09T06:37:16.293712+0000","last_fullsized":"2023-10-09T06:37:16.293712+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":573,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294120+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:37:16.294120+0000","last_peered":"2023-10-09T06:37:16.294120+0000","last_clean":"2023-10-09T06:37:16.294120+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:37:16.294120+0000","last_undegraded":"2023-10-09T06:37:16.294120+0000","last_fullsized":"2023-10-09T06:37:16.294120+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":471,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289306+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:37:16.289306+0000","last_peered":"2023-10-09T06:37:16.289306+0000","last_clean":"2023-10-09T06:37:16.289306+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:37:16.289306+0000","last_undegraded":"2023-10-09T06:37:16.289306+0000","last_fullsized":"2023-10-09T06:37:16.289306+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":435,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289542+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:37:16.289542+0000","last_peered":"2023-10-09T06:37:16.289542+0000","last_clean":"2023-10-09T06:37:16.289542+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:37:16.289542+0000","last_undegraded":"2023-10-09T06:37:16.289542+0000","last_fullsized":"2023-10-09T06:37:16.289542+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":738,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297083+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:37:17.297083+0000","last_peered":"2023-10-09T06:37:17.297083+0000","last_clean":"2023-10-09T06:37:17.297083+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:37:17.297083+0000","last_undegraded":"2023-10-09T06:37:17.297083+0000","last_fullsized":"2023-10-09T06:37:17.297083+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":314,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385732+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:37:16.385732+0000","last_peered":"2023-10-09T06:37:16.385732+0000","last_clean":"2023-10-09T06:37:16.385732+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:37:16.385732+0000","last_undegraded":"2023-10-09T06:37:16.385732+0000","last_fullsized":"2023-10-09T06:37:16.385732+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301262+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:37:17.301262+0000","last_peered":"2023-10-09T06:37:17.301262+0000","last_clean":"2023-10-09T06:37:17.301262+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:37:17.301262+0000","last_undegraded":"2023-10-09T06:37:17.301262+0000","last_fullsized":"2023-10-09T06:37:17.301262+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":706,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385523+0000","last_change":"2023-10-09T06:37:10.361228+0000","last_active":"2023-10-09T06:37:16.385523+0000","last_peered":"2023-10-09T06:37:16.385523+0000","last_clean":"2023-10-09T06:37:16.385523+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:37:16.385523+0000","last_undegraded":"2023-10-09T06:37:16.385523+0000","last_fullsized":"2023-10-09T06:37:16.385523+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":463,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297504+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:37:17.297504+0000","last_peered":"2023-10-09T06:37:17.297504+0000","last_clean":"2023-10-09T06:37:17.297504+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:37:17.297504+0000","last_undegraded":"2023-10-09T06:37:17.297504+0000","last_fullsized":"2023-10-09T06:37:17.297504+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":285,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385289+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:37:16.385289+0000","last_peered":"2023-10-09T06:37:16.385289+0000","last_clean":"2023-10-09T06:37:16.385289+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:37:16.385289+0000","last_undegraded":"2023-10-09T06:37:16.385289+0000","last_fullsized":"2023-10-09T06:37:16.385289+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":791,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.292791+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:37:16.292791+0000","last_peered":"2023-10-09T06:37:16.292791+0000","last_clean":"2023-10-09T06:37:16.292791+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:37:16.292791+0000","last_undegraded":"2023-10-09T06:37:16.292791+0000","last_fullsized":"2023-10-09T06:37:16.292791+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":308,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385535+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:37:16.385535+0000","last_peered":"2023-10-09T06:37:16.385535+0000","last_clean":"2023-10-09T06:37:16.385535+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:37:16.385535+0000","last_undegraded":"2023-10-09T06:37:16.385535+0000","last_fullsized":"2023-10-09T06:37:16.385535+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":140,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298014+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:37:17.298014+0000","last_peered":"2023-10-09T06:37:17.298014+0000","last_clean":"2023-10-09T06:37:17.298014+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:37:17.298014+0000","last_undegraded":"2023-10-09T06:37:17.298014+0000","last_fullsized":"2023-10-09T06:37:17.298014+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":579,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.297346+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:37:16.297346+0000","last_peered":"2023-10-09T06:37:16.297346+0000","last_clean":"2023-10-09T06:37:16.297346+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:37:16.297346+0000","last_undegraded":"2023-10-09T06:37:16.297346+0000","last_fullsized":"2023-10-09T06:37:16.297346+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":543,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392111+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:37:16.392111+0000","last_peered":"2023-10-09T06:37:16.392111+0000","last_clean":"2023-10-09T06:37:16.392111+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:37:16.392111+0000","last_undegraded":"2023-10-09T06:37:16.392111+0000","last_fullsized":"2023-10-09T06:37:16.392111+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":167,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T06:37:18.306024+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:37:18.306024+0000","last_peered":"2023-10-09T06:37:18.306024+0000","last_clean":"2023-10-09T06:37:18.306024+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:37:18.306024+0000","last_undegraded":"2023-10-09T06:37:18.306024+0000","last_fullsized":"2023-10-09T06:37:18.306024+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"464'6616","reported_seq":6664,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.263997+0000","last_change":"2023-10-09T06:36:59.604586+0000","last_active":"2023-10-09T06:37:17.263997+0000","last_peered":"2023-10-09T06:37:17.263997+0000","last_clean":"2023-10-09T06:37:17.263997+0000","last_became_active":"2023-10-09T06:36:57.185490+0000","last_became_peered":"2023-10-09T06:36:57.185490+0000","last_unstale":"2023-10-09T06:37:17.263997+0000","last_undegraded":"2023-10-09T06:37:17.263997+0000","last_fullsized":"2023-10-09T06:37:17.263997+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201752576,"num_objects":3078,"num_object_clones":0,"num_object_copies":9234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3542,"num_write_kb":211872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":592,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385831+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:37:16.385831+0000","last_peered":"2023-10-09T06:37:16.385831+0000","last_clean":"2023-10-09T06:37:16.385831+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:37:16.385831+0000","last_undegraded":"2023-10-09T06:37:16.385831+0000","last_fullsized":"2023-10-09T06:37:16.385831+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":350,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386146+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:37:16.386146+0000","last_peered":"2023-10-09T06:37:16.386146+0000","last_clean":"2023-10-09T06:37:16.386146+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:37:16.386146+0000","last_undegraded":"2023-10-09T06:37:16.386146+0000","last_fullsized":"2023-10-09T06:37:16.386146+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385808+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:37:16.385808+0000","last_peered":"2023-10-09T06:37:16.385808+0000","last_clean":"2023-10-09T06:37:16.385808+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:37:16.385808+0000","last_undegraded":"2023-10-09T06:37:16.385808+0000","last_fullsized":"2023-10-09T06:37:16.385808+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.0","version":"464'6620","reported_seq":6802,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.195226+0000","last_change":"2023-10-09T06:36:59.980627+0000","last_active":"2023-10-09T06:37:17.195226+0000","last_peered":"2023-10-09T06:37:17.195226+0000","last_clean":"2023-10-09T06:37:17.195226+0000","last_became_active":"2023-10-09T06:36:57.178903+0000","last_became_peered":"2023-10-09T06:36:57.178903+0000","last_unstale":"2023-10-09T06:37:17.195226+0000","last_undegraded":"2023-10-09T06:37:17.195226+0000","last_fullsized":"2023-10-09T06:37:17.195226+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6277","last_scrub_stamp":"2023-10-09T06:36:59.980553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.980553+0000","objects_scrubbed":3134,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:50.243776+0000","scrub_duration":0.77626023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203096064,"num_objects":3099,"num_object_clones":0,"num_object_copies":9297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":353,"num_write_kb":10176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":851,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301097+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:37:17.301097+0000","last_peered":"2023-10-09T06:37:17.301097+0000","last_clean":"2023-10-09T06:37:17.301097+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:37:17.301097+0000","last_undegraded":"2023-10-09T06:37:17.301097+0000","last_fullsized":"2023-10-09T06:37:17.301097+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":312,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386203+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:37:16.386203+0000","last_peered":"2023-10-09T06:37:16.386203+0000","last_clean":"2023-10-09T06:37:16.386203+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:37:16.386203+0000","last_undegraded":"2023-10-09T06:37:16.386203+0000","last_fullsized":"2023-10-09T06:37:16.386203+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":462,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289020+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:37:16.289020+0000","last_peered":"2023-10-09T06:37:16.289020+0000","last_clean":"2023-10-09T06:37:16.289020+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:37:16.289020+0000","last_undegraded":"2023-10-09T06:37:16.289020+0000","last_fullsized":"2023-10-09T06:37:16.289020+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"466'6775","reported_seq":6896,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T06:37:21.066093+0000","last_change":"2023-10-09T06:36:58.845020+0000","last_active":"2023-10-09T06:37:21.066093+0000","last_peered":"2023-10-09T06:37:21.066093+0000","last_clean":"2023-10-09T06:37:21.066093+0000","last_became_active":"2023-10-09T06:36:57.185094+0000","last_became_peered":"2023-10-09T06:36:57.185094+0000","last_unstale":"2023-10-09T06:37:21.066093+0000","last_undegraded":"2023-10-09T06:37:21.066093+0000","last_fullsized":"2023-10-09T06:37:21.066093+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6294","last_scrub_stamp":"2023-10-09T06:36:58.844752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:58.844752+0000","objects_scrubbed":3160,"log_size":6775,"log_dups_size":0,"ondisk_log_size":6775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:07.044778+0000","scrub_duration":0.83882531500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198836224,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":494,"num_write_kb":11776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":766,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392933+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:37:16.392933+0000","last_peered":"2023-10-09T06:37:16.392933+0000","last_clean":"2023-10-09T06:37:16.392933+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:37:16.392933+0000","last_undegraded":"2023-10-09T06:37:16.392933+0000","last_fullsized":"2023-10-09T06:37:16.392933+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":368,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385967+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:37:16.385967+0000","last_peered":"2023-10-09T06:37:16.385967+0000","last_clean":"2023-10-09T06:37:16.385967+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:37:16.385967+0000","last_undegraded":"2023-10-09T06:37:16.385967+0000","last_fullsized":"2023-10-09T06:37:16.385967+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":441,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392330+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:37:16.392330+0000","last_peered":"2023-10-09T06:37:16.392330+0000","last_clean":"2023-10-09T06:37:16.392330+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:37:16.392330+0000","last_undegraded":"2023-10-09T06:37:16.392330+0000","last_fullsized":"2023-10-09T06:37:16.392330+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"464'6623","reported_seq":6678,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.302125+0000","last_change":"2023-10-09T06:37:11.285046+0000","last_active":"2023-10-09T06:37:17.302125+0000","last_peered":"2023-10-09T06:37:17.302125+0000","last_clean":"2023-10-09T06:37:17.302125+0000","last_became_active":"2023-10-09T06:37:08.349856+0000","last_became_peered":"2023-10-09T06:37:08.349856+0000","last_unstale":"2023-10-09T06:37:17.302125+0000","last_undegraded":"2023-10-09T06:37:17.302125+0000","last_fullsized":"2023-10-09T06:37:17.302125+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6623,"log_dups_size":0,"ondisk_log_size":6623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202801152,"num_objects":3094,"num_object_clones":0,"num_object_copies":9282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3530,"num_write_kb":212000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":589,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289761+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:37:16.289761+0000","last_peered":"2023-10-09T06:37:16.289761+0000","last_clean":"2023-10-09T06:37:16.289761+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:37:16.289761+0000","last_undegraded":"2023-10-09T06:37:16.289761+0000","last_fullsized":"2023-10-09T06:37:16.289761+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301347+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:37:17.301347+0000","last_peered":"2023-10-09T06:37:17.301347+0000","last_clean":"2023-10-09T06:37:17.301347+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:37:17.301347+0000","last_undegraded":"2023-10-09T06:37:17.301347+0000","last_fullsized":"2023-10-09T06:37:17.301347+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":427,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297654+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:37:17.297654+0000","last_peered":"2023-10-09T06:37:17.297654+0000","last_clean":"2023-10-09T06:37:17.297654+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:37:17.297654+0000","last_undegraded":"2023-10-09T06:37:17.297654+0000","last_fullsized":"2023-10-09T06:37:17.297654+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"464'6690","reported_seq":6877,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.254955+0000","last_change":"2023-10-09T06:37:01.498281+0000","last_active":"2023-10-09T06:37:17.254955+0000","last_peered":"2023-10-09T06:37:17.254955+0000","last_clean":"2023-10-09T06:37:17.254955+0000","last_became_active":"2023-10-09T06:36:57.211509+0000","last_became_peered":"2023-10-09T06:36:57.211509+0000","last_unstale":"2023-10-09T06:37:17.254955+0000","last_undegraded":"2023-10-09T06:37:17.254955+0000","last_fullsized":"2023-10-09T06:37:17.254955+0000","mapping_epoch":437,"log_start":"441'400","ondisk_log_start":"441'400","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6333","last_scrub_stamp":"2023-10-09T06:37:01.498222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:01.498222+0000","objects_scrubbed":3239,"log_size":6290,"log_dups_size":0,"ondisk_log_size":6290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:56.557159+0000","scrub_duration":1.395430696,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205586432,"num_objects":3137,"num_object_clones":0,"num_object_copies":9411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":386,"num_write_kb":9088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":817,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385459+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:37:16.385459+0000","last_peered":"2023-10-09T06:37:16.385459+0000","last_clean":"2023-10-09T06:37:16.385459+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:37:16.385459+0000","last_undegraded":"2023-10-09T06:37:16.385459+0000","last_fullsized":"2023-10-09T06:37:16.385459+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":193,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301017+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:37:17.301017+0000","last_peered":"2023-10-09T06:37:17.301017+0000","last_clean":"2023-10-09T06:37:17.301017+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:37:17.301017+0000","last_undegraded":"2023-10-09T06:37:17.301017+0000","last_fullsized":"2023-10-09T06:37:17.301017+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":445,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289103+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:37:16.289103+0000","last_peered":"2023-10-09T06:37:16.289103+0000","last_clean":"2023-10-09T06:37:16.289103+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:37:16.289103+0000","last_undegraded":"2023-10-09T06:37:16.289103+0000","last_fullsized":"2023-10-09T06:37:16.289103+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"464'6666","reported_seq":6848,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.242351+0000","last_change":"2023-10-09T06:36:59.004823+0000","last_active":"2023-10-09T06:37:17.242351+0000","last_peered":"2023-10-09T06:37:17.242351+0000","last_clean":"2023-10-09T06:37:17.242351+0000","last_became_active":"2023-10-09T06:36:57.221064+0000","last_became_peered":"2023-10-09T06:36:57.221064+0000","last_unstale":"2023-10-09T06:37:17.242351+0000","last_undegraded":"2023-10-09T06:37:17.242351+0000","last_fullsized":"2023-10-09T06:37:17.242351+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6279","last_scrub_stamp":"2023-10-09T06:36:59.004706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.004706+0000","objects_scrubbed":3113,"log_size":6666,"log_dups_size":0,"ondisk_log_size":6666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:30.643419+0000","scrub_duration":1.3564550360000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200605696,"num_objects":3061,"num_object_clones":0,"num_object_copies":9183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":414,"num_write_kb":11584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":487,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300320+0000","last_change":"2023-10-09T06:37:08.345883+0000","last_active":"2023-10-09T06:37:17.300320+0000","last_peered":"2023-10-09T06:37:17.300320+0000","last_clean":"2023-10-09T06:37:17.300320+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:37:17.300320+0000","last_undegraded":"2023-10-09T06:37:17.300320+0000","last_fullsized":"2023-10-09T06:37:17.300320+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":422,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392403+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:37:16.392403+0000","last_peered":"2023-10-09T06:37:16.392403+0000","last_clean":"2023-10-09T06:37:16.392403+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:37:16.392403+0000","last_undegraded":"2023-10-09T06:37:16.392403+0000","last_fullsized":"2023-10-09T06:37:16.392403+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":442,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289332+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:37:16.289332+0000","last_peered":"2023-10-09T06:37:16.289332+0000","last_clean":"2023-10-09T06:37:16.289332+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:37:16.289332+0000","last_undegraded":"2023-10-09T06:37:16.289332+0000","last_fullsized":"2023-10-09T06:37:16.289332+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"464'6538","reported_seq":6587,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.245009+0000","last_change":"2023-10-09T06:37:00.233234+0000","last_active":"2023-10-09T06:37:17.245009+0000","last_peered":"2023-10-09T06:37:17.245009+0000","last_clean":"2023-10-09T06:37:17.245009+0000","last_became_active":"2023-10-09T06:36:57.185079+0000","last_became_peered":"2023-10-09T06:36:57.185079+0000","last_unstale":"2023-10-09T06:37:17.245009+0000","last_undegraded":"2023-10-09T06:37:17.245009+0000","last_fullsized":"2023-10-09T06:37:17.245009+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200802304,"num_objects":3064,"num_object_clones":0,"num_object_copies":9192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3494,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":800,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.292875+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:37:16.292875+0000","last_peered":"2023-10-09T06:37:16.292875+0000","last_clean":"2023-10-09T06:37:16.292875+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:37:16.292875+0000","last_undegraded":"2023-10-09T06:37:16.292875+0000","last_fullsized":"2023-10-09T06:37:16.292875+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":446,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.288923+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:37:16.288923+0000","last_peered":"2023-10-09T06:37:16.288923+0000","last_clean":"2023-10-09T06:37:16.288923+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:37:16.288923+0000","last_undegraded":"2023-10-09T06:37:16.288923+0000","last_fullsized":"2023-10-09T06:37:16.288923+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":425,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289876+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:37:16.289876+0000","last_peered":"2023-10-09T06:37:16.289876+0000","last_clean":"2023-10-09T06:37:16.289876+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:37:16.289876+0000","last_undegraded":"2023-10-09T06:37:16.289876+0000","last_fullsized":"2023-10-09T06:37:16.289876+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"464'6567","reported_seq":6616,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.102096+0000","last_change":"2023-10-09T06:37:08.212088+0000","last_active":"2023-10-09T06:37:17.102096+0000","last_peered":"2023-10-09T06:37:17.102096+0000","last_clean":"2023-10-09T06:37:17.102096+0000","last_became_active":"2023-10-09T06:36:57.222809+0000","last_became_peered":"2023-10-09T06:36:57.222809+0000","last_unstale":"2023-10-09T06:37:17.102096+0000","last_undegraded":"2023-10-09T06:37:17.102096+0000","last_fullsized":"2023-10-09T06:37:17.102096+0000","mapping_epoch":437,"log_start":"441'300","ondisk_log_start":"441'300","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6267,"log_dups_size":0,"ondisk_log_size":6267,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200310784,"num_objects":3056,"num_object_clones":0,"num_object_copies":9168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3514,"num_write_kb":210272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":417,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392109+0000","last_change":"2023-10-09T06:37:08.251809+0000","last_active":"2023-10-09T06:37:16.392109+0000","last_peered":"2023-10-09T06:37:16.392109+0000","last_clean":"2023-10-09T06:37:16.392109+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:37:16.392109+0000","last_undegraded":"2023-10-09T06:37:16.392109+0000","last_fullsized":"2023-10-09T06:37:16.392109+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":167,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300243+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:37:17.300243+0000","last_peered":"2023-10-09T06:37:17.300243+0000","last_clean":"2023-10-09T06:37:17.300243+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:37:17.300243+0000","last_undegraded":"2023-10-09T06:37:17.300243+0000","last_fullsized":"2023-10-09T06:37:17.300243+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":419,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294061+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:37:16.294061+0000","last_peered":"2023-10-09T06:37:16.294061+0000","last_clean":"2023-10-09T06:37:16.294061+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:37:16.294061+0000","last_undegraded":"2023-10-09T06:37:16.294061+0000","last_fullsized":"2023-10-09T06:37:16.294061+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"464'7035","reported_seq":7077,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300998+0000","last_change":"2023-10-09T06:37:08.359543+0000","last_active":"2023-10-09T06:37:17.300998+0000","last_peered":"2023-10-09T06:37:17.300998+0000","last_clean":"2023-10-09T06:37:17.300998+0000","last_became_active":"2023-10-09T06:37:08.343368+0000","last_became_peered":"2023-10-09T06:37:08.343368+0000","last_unstale":"2023-10-09T06:37:17.300998+0000","last_undegraded":"2023-10-09T06:37:17.300998+0000","last_fullsized":"2023-10-09T06:37:17.300998+0000","mapping_epoch":459,"log_start":"441'1108","ondisk_log_start":"441'1108","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:20.011628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402718720,"num_objects":6145,"num_object_clones":0,"num_object_copies":18435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7035,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":757,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.393011+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:37:16.393011+0000","last_peered":"2023-10-09T06:37:16.393011+0000","last_clean":"2023-10-09T06:37:16.393011+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:37:16.393011+0000","last_undegraded":"2023-10-09T06:37:16.393011+0000","last_fullsized":"2023-10-09T06:37:16.393011+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":471,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294139+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:37:16.294139+0000","last_peered":"2023-10-09T06:37:16.294139+0000","last_clean":"2023-10-09T06:37:16.294139+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:37:16.294139+0000","last_undegraded":"2023-10-09T06:37:16.294139+0000","last_fullsized":"2023-10-09T06:37:16.294139+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":35,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301384+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:37:17.301384+0000","last_peered":"2023-10-09T06:37:17.301384+0000","last_clean":"2023-10-09T06:37:17.301384+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:37:17.301384+0000","last_undegraded":"2023-10-09T06:37:17.301384+0000","last_fullsized":"2023-10-09T06:37:17.301384+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":405,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.293361+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:37:16.293361+0000","last_peered":"2023-10-09T06:37:16.293361+0000","last_clean":"2023-10-09T06:37:16.293361+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:37:16.293361+0000","last_undegraded":"2023-10-09T06:37:16.293361+0000","last_fullsized":"2023-10-09T06:37:16.293361+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"464'7127","reported_seq":7164,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.120822+0000","last_change":"2023-10-09T06:35:28.539445+0000","last_active":"2023-10-09T06:37:17.120822+0000","last_peered":"2023-10-09T06:37:17.120822+0000","last_clean":"2023-10-09T06:37:17.120822+0000","last_became_active":"2023-10-09T06:35:28.539141+0000","last_became_peered":"2023-10-09T06:35:28.539141+0000","last_unstale":"2023-10-09T06:37:17.120822+0000","last_undegraded":"2023-10-09T06:37:17.120822+0000","last_fullsized":"2023-10-09T06:37:17.120822+0000","mapping_epoch":437,"log_start":"442'1867","ondisk_log_start":"442'1867","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5260,"log_dups_size":0,"ondisk_log_size":5260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:26.753486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407896088,"num_objects":6225,"num_object_clones":0,"num_object_copies":18675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7127,"num_write_kb":427201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":632,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297398+0000","last_change":"2023-10-09T06:37:08.296784+0000","last_active":"2023-10-09T06:37:17.297398+0000","last_peered":"2023-10-09T06:37:17.297398+0000","last_clean":"2023-10-09T06:37:17.297398+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:37:17.297398+0000","last_undegraded":"2023-10-09T06:37:17.297398+0000","last_fullsized":"2023-10-09T06:37:17.297398+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":195,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.293506+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:37:16.293506+0000","last_peered":"2023-10-09T06:37:16.293506+0000","last_clean":"2023-10-09T06:37:16.293506+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:37:16.293506+0000","last_undegraded":"2023-10-09T06:37:16.293506+0000","last_fullsized":"2023-10-09T06:37:16.293506+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":446,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289489+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:37:16.289489+0000","last_peered":"2023-10-09T06:37:16.289489+0000","last_clean":"2023-10-09T06:37:16.289489+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:37:16.289489+0000","last_undegraded":"2023-10-09T06:37:16.289489+0000","last_fullsized":"2023-10-09T06:37:16.289489+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294108+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:37:16.294108+0000","last_peered":"2023-10-09T06:37:16.294108+0000","last_clean":"2023-10-09T06:37:16.294108+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:37:16.294108+0000","last_undegraded":"2023-10-09T06:37:16.294108+0000","last_fullsized":"2023-10-09T06:37:16.294108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"464'7121","reported_seq":7157,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.162749+0000","last_change":"2023-10-09T06:35:28.544012+0000","last_active":"2023-10-09T06:37:17.162749+0000","last_peered":"2023-10-09T06:37:17.162749+0000","last_clean":"2023-10-09T06:37:17.162749+0000","last_became_active":"2023-10-09T06:35:28.543684+0000","last_became_peered":"2023-10-09T06:35:28.543684+0000","last_unstale":"2023-10-09T06:37:17.162749+0000","last_undegraded":"2023-10-09T06:37:17.162749+0000","last_fullsized":"2023-10-09T06:37:17.162749+0000","mapping_epoch":437,"log_start":"441'1112","ondisk_log_start":"441'1112","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:47.327388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402456576,"num_objects":6141,"num_object_clones":0,"num_object_copies":18423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7121,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":807,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385614+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:37:16.385614+0000","last_peered":"2023-10-09T06:37:16.385614+0000","last_clean":"2023-10-09T06:37:16.385614+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:37:16.385614+0000","last_undegraded":"2023-10-09T06:37:16.385614+0000","last_fullsized":"2023-10-09T06:37:16.385614+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":458,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300522+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:37:17.300522+0000","last_peered":"2023-10-09T06:37:17.300522+0000","last_clean":"2023-10-09T06:37:17.300522+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:37:17.300522+0000","last_undegraded":"2023-10-09T06:37:17.300522+0000","last_fullsized":"2023-10-09T06:37:17.300522+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":444,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.292962+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:37:16.292962+0000","last_peered":"2023-10-09T06:37:16.292962+0000","last_clean":"2023-10-09T06:37:16.292962+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:37:16.292962+0000","last_undegraded":"2023-10-09T06:37:16.292962+0000","last_fullsized":"2023-10-09T06:37:16.292962+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":466,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294554+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:37:16.294554+0000","last_peered":"2023-10-09T06:37:16.294554+0000","last_clean":"2023-10-09T06:37:16.294554+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:37:16.294554+0000","last_undegraded":"2023-10-09T06:37:16.294554+0000","last_fullsized":"2023-10-09T06:37:16.294554+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"464'7057","reported_seq":7094,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.270610+0000","last_change":"2023-10-09T06:35:28.750537+0000","last_active":"2023-10-09T06:37:17.270610+0000","last_peered":"2023-10-09T06:37:17.270610+0000","last_clean":"2023-10-09T06:37:17.270610+0000","last_became_active":"2023-10-09T06:35:28.750267+0000","last_became_peered":"2023-10-09T06:35:28.750267+0000","last_unstale":"2023-10-09T06:37:17.270610+0000","last_undegraded":"2023-10-09T06:37:17.270610+0000","last_fullsized":"2023-10-09T06:37:17.270610+0000","mapping_epoch":437,"log_start":"441'986","ondisk_log_start":"441'986","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:47.173545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403111936,"num_objects":6151,"num_object_clones":0,"num_object_copies":18453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7057,"num_write_kb":422656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":274,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385662+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:37:16.385662+0000","last_peered":"2023-10-09T06:37:16.385662+0000","last_clean":"2023-10-09T06:37:16.385662+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:37:16.385662+0000","last_undegraded":"2023-10-09T06:37:16.385662+0000","last_fullsized":"2023-10-09T06:37:16.385662+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":788,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385825+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:37:16.385825+0000","last_peered":"2023-10-09T06:37:16.385825+0000","last_clean":"2023-10-09T06:37:16.385825+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:37:16.385825+0000","last_undegraded":"2023-10-09T06:37:16.385825+0000","last_fullsized":"2023-10-09T06:37:16.385825+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.302200+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:37:17.302200+0000","last_peered":"2023-10-09T06:37:17.302200+0000","last_clean":"2023-10-09T06:37:17.302200+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:37:17.302200+0000","last_undegraded":"2023-10-09T06:37:17.302200+0000","last_fullsized":"2023-10-09T06:37:17.302200+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":427,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392657+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:37:16.392657+0000","last_peered":"2023-10-09T06:37:16.392657+0000","last_clean":"2023-10-09T06:37:16.392657+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:37:16.392657+0000","last_undegraded":"2023-10-09T06:37:16.392657+0000","last_fullsized":"2023-10-09T06:37:16.392657+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"464'7151","reported_seq":7187,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.252049+0000","last_change":"2023-10-09T06:35:28.750108+0000","last_active":"2023-10-09T06:37:17.252049+0000","last_peered":"2023-10-09T06:37:17.252049+0000","last_clean":"2023-10-09T06:37:17.252049+0000","last_became_active":"2023-10-09T06:35:28.749808+0000","last_became_peered":"2023-10-09T06:35:28.749808+0000","last_unstale":"2023-10-09T06:37:17.252049+0000","last_undegraded":"2023-10-09T06:37:17.252049+0000","last_fullsized":"2023-10-09T06:37:17.252049+0000","mapping_epoch":437,"log_start":"441'1000","ondisk_log_start":"441'1000","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:44.197980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407568384,"num_objects":6219,"num_object_clones":0,"num_object_copies":18657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7151,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":774,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.292954+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:37:16.292954+0000","last_peered":"2023-10-09T06:37:16.292954+0000","last_clean":"2023-10-09T06:37:16.292954+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:37:16.292954+0000","last_undegraded":"2023-10-09T06:37:16.292954+0000","last_fullsized":"2023-10-09T06:37:16.292954+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":328,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.385554+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:37:16.385554+0000","last_peered":"2023-10-09T06:37:16.385554+0000","last_clean":"2023-10-09T06:37:16.385554+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:37:16.385554+0000","last_undegraded":"2023-10-09T06:37:16.385554+0000","last_fullsized":"2023-10-09T06:37:16.385554+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":140,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297328+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:37:17.297328+0000","last_peered":"2023-10-09T06:37:17.297328+0000","last_clean":"2023-10-09T06:37:17.297328+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:37:17.297328+0000","last_undegraded":"2023-10-09T06:37:17.297328+0000","last_fullsized":"2023-10-09T06:37:17.297328+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":451,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298289+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:37:17.298289+0000","last_peered":"2023-10-09T06:37:17.298289+0000","last_clean":"2023-10-09T06:37:17.298289+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:37:17.298289+0000","last_undegraded":"2023-10-09T06:37:17.298289+0000","last_fullsized":"2023-10-09T06:37:17.298289+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"464'7063","reported_seq":7100,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.261319+0000","last_change":"2023-10-09T06:35:28.748964+0000","last_active":"2023-10-09T06:37:17.261319+0000","last_peered":"2023-10-09T06:37:17.261319+0000","last_clean":"2023-10-09T06:37:17.261319+0000","last_became_active":"2023-10-09T06:35:28.748674+0000","last_became_peered":"2023-10-09T06:35:28.748674+0000","last_unstale":"2023-10-09T06:37:17.261319+0000","last_undegraded":"2023-10-09T06:37:17.261319+0000","last_fullsized":"2023-10-09T06:37:17.261319+0000","mapping_epoch":437,"log_start":"441'988","ondisk_log_start":"441'988","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:02.079067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401801216,"num_objects":6131,"num_object_clones":0,"num_object_copies":18393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7063,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":787,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.392659+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:37:16.392659+0000","last_peered":"2023-10-09T06:37:16.392659+0000","last_clean":"2023-10-09T06:37:16.392659+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:37:16.392659+0000","last_undegraded":"2023-10-09T06:37:16.392659+0000","last_fullsized":"2023-10-09T06:37:16.392659+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":328,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386051+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:37:16.386051+0000","last_peered":"2023-10-09T06:37:16.386051+0000","last_clean":"2023-10-09T06:37:16.386051+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:37:16.386051+0000","last_undegraded":"2023-10-09T06:37:16.386051+0000","last_fullsized":"2023-10-09T06:37:16.386051+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":461,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294064+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:37:16.294064+0000","last_peered":"2023-10-09T06:37:16.294064+0000","last_clean":"2023-10-09T06:37:16.294064+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:37:16.294064+0000","last_undegraded":"2023-10-09T06:37:16.294064+0000","last_fullsized":"2023-10-09T06:37:16.294064+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":449,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297734+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:37:17.297734+0000","last_peered":"2023-10-09T06:37:17.297734+0000","last_clean":"2023-10-09T06:37:17.297734+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:37:17.297734+0000","last_undegraded":"2023-10-09T06:37:17.297734+0000","last_fullsized":"2023-10-09T06:37:17.297734+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"464'6607","reported_seq":6657,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.257649+0000","last_change":"2023-10-09T06:36:59.148541+0000","last_active":"2023-10-09T06:37:17.257649+0000","last_peered":"2023-10-09T06:37:17.257649+0000","last_clean":"2023-10-09T06:37:17.257649+0000","last_became_active":"2023-10-09T06:36:57.214586+0000","last_became_peered":"2023-10-09T06:36:57.214586+0000","last_unstale":"2023-10-09T06:37:17.257649+0000","last_undegraded":"2023-10-09T06:37:17.257649+0000","last_fullsized":"2023-10-09T06:37:17.257649+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199229440,"num_objects":3040,"num_object_clones":0,"num_object_copies":9120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3536,"num_write_kb":210432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":788,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289185+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:37:16.289185+0000","last_peered":"2023-10-09T06:37:16.289185+0000","last_clean":"2023-10-09T06:37:16.289185+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:37:16.289185+0000","last_undegraded":"2023-10-09T06:37:16.289185+0000","last_fullsized":"2023-10-09T06:37:16.289185+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":197,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.302014+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:37:17.302014+0000","last_peered":"2023-10-09T06:37:17.302014+0000","last_clean":"2023-10-09T06:37:17.302014+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:37:17.302014+0000","last_undegraded":"2023-10-09T06:37:17.302014+0000","last_fullsized":"2023-10-09T06:37:17.302014+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":429,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294328+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:37:16.294328+0000","last_peered":"2023-10-09T06:37:16.294328+0000","last_clean":"2023-10-09T06:37:16.294328+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:37:16.294328+0000","last_undegraded":"2023-10-09T06:37:16.294328+0000","last_fullsized":"2023-10-09T06:37:16.294328+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300153+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:37:17.300153+0000","last_peered":"2023-10-09T06:37:17.300153+0000","last_clean":"2023-10-09T06:37:17.300153+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:37:17.300153+0000","last_undegraded":"2023-10-09T06:37:17.300153+0000","last_fullsized":"2023-10-09T06:37:17.300153+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"464'6655","reported_seq":6851,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.200953+0000","last_change":"2023-10-09T06:37:08.334364+0000","last_active":"2023-10-09T06:37:17.200953+0000","last_peered":"2023-10-09T06:37:17.200953+0000","last_clean":"2023-10-09T06:37:17.200953+0000","last_became_active":"2023-10-09T06:37:08.252813+0000","last_became_peered":"2023-10-09T06:37:08.252813+0000","last_unstale":"2023-10-09T06:37:17.200953+0000","last_undegraded":"2023-10-09T06:37:17.200953+0000","last_fullsized":"2023-10-09T06:37:17.200953+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"457'6359","last_scrub_stamp":"2023-10-09T06:37:03.544833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:03.544833+0000","objects_scrubbed":3271,"log_size":6655,"log_dups_size":0,"ondisk_log_size":6655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:43.295290+0000","scrub_duration":0.43754141200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203161600,"num_objects":3100,"num_object_clones":0,"num_object_copies":9300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":299,"num_write_kb":4096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":819,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301427+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:37:17.301427+0000","last_peered":"2023-10-09T06:37:17.301427+0000","last_clean":"2023-10-09T06:37:17.301427+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:37:17.301427+0000","last_undegraded":"2023-10-09T06:37:17.301427+0000","last_fullsized":"2023-10-09T06:37:17.301427+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":449,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.393064+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:37:16.393064+0000","last_peered":"2023-10-09T06:37:16.393064+0000","last_clean":"2023-10-09T06:37:16.393064+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:37:16.393064+0000","last_undegraded":"2023-10-09T06:37:16.393064+0000","last_fullsized":"2023-10-09T06:37:16.393064+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":432,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298148+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:37:17.298148+0000","last_peered":"2023-10-09T06:37:17.298148+0000","last_clean":"2023-10-09T06:37:17.298148+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:37:17.298148+0000","last_undegraded":"2023-10-09T06:37:17.298148+0000","last_fullsized":"2023-10-09T06:37:17.298148+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":428,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.288998+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:37:16.288998+0000","last_peered":"2023-10-09T06:37:16.288998+0000","last_clean":"2023-10-09T06:37:16.288998+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:37:16.288998+0000","last_undegraded":"2023-10-09T06:37:16.288998+0000","last_fullsized":"2023-10-09T06:37:16.288998+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.15","version":"464'6655","reported_seq":6701,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.046350+0000","last_change":"2023-10-09T06:36:57.235313+0000","last_active":"2023-10-09T06:37:17.046350+0000","last_peered":"2023-10-09T06:37:17.046350+0000","last_clean":"2023-10-09T06:37:17.046350+0000","last_became_active":"2023-10-09T06:36:57.229253+0000","last_became_peered":"2023-10-09T06:36:57.229253+0000","last_unstale":"2023-10-09T06:37:17.046350+0000","last_undegraded":"2023-10-09T06:37:17.046350+0000","last_fullsized":"2023-10-09T06:37:17.046350+0000","mapping_epoch":454,"log_start":"441'400","ondisk_log_start":"441'400","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203816960,"num_objects":3110,"num_object_clones":0,"num_object_copies":9330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3558,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":746,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300273+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:37:17.300273+0000","last_peered":"2023-10-09T06:37:17.300273+0000","last_clean":"2023-10-09T06:37:17.300273+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:37:17.300273+0000","last_undegraded":"2023-10-09T06:37:17.300273+0000","last_fullsized":"2023-10-09T06:37:17.300273+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":433,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.297740+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:37:17.297740+0000","last_peered":"2023-10-09T06:37:17.297740+0000","last_clean":"2023-10-09T06:37:17.297740+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:37:17.297740+0000","last_undegraded":"2023-10-09T06:37:17.297740+0000","last_fullsized":"2023-10-09T06:37:17.297740+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":454,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.294026+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:37:16.294026+0000","last_peered":"2023-10-09T06:37:16.294026+0000","last_clean":"2023-10-09T06:37:16.294026+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:37:16.294026+0000","last_undegraded":"2023-10-09T06:37:16.294026+0000","last_fullsized":"2023-10-09T06:37:16.294026+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.12","version":"462'6631","reported_seq":6686,"reported_epoch":466,"state":"activating+undersized+remapped","last_fresh":"2023-10-09T06:37:18.304959+0000","last_change":"2023-10-09T06:37:18.463618+0000","last_active":"2023-10-09T06:37:17.300948+0000","last_peered":"2023-10-09T06:37:15.283407+0000","last_clean":"2023-10-09T06:37:15.283407+0000","last_became_active":"2023-10-09T06:37:08.298235+0000","last_became_peered":"2023-10-09T06:37:08.298235+0000","last_unstale":"2023-10-09T06:37:18.304959+0000","last_undegraded":"2023-10-09T06:37:18.304959+0000","last_fullsized":"2023-10-09T06:37:18.304959+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204898304,"num_objects":3127,"num_object_clones":0,"num_object_copies":9381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3342,"num_objects_misplaced":3342,"num_objects_unfound":0,"num_objects_dirty":3127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3539,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[3,4],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4","objects":-9}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":676,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301271+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:37:17.301271+0000","last_peered":"2023-10-09T06:37:17.301271+0000","last_clean":"2023-10-09T06:37:17.301271+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:37:17.301271+0000","last_undegraded":"2023-10-09T06:37:17.301271+0000","last_fullsized":"2023-10-09T06:37:17.301271+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":143,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289209+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:37:16.289209+0000","last_peered":"2023-10-09T06:37:16.289209+0000","last_clean":"2023-10-09T06:37:16.289209+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:37:16.289209+0000","last_undegraded":"2023-10-09T06:37:16.289209+0000","last_fullsized":"2023-10-09T06:37:16.289209+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":472,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289372+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:37:16.289372+0000","last_peered":"2023-10-09T06:37:16.289372+0000","last_clean":"2023-10-09T06:37:16.289372+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:37:16.289372+0000","last_undegraded":"2023-10-09T06:37:16.289372+0000","last_fullsized":"2023-10-09T06:37:16.289372+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.13","version":"466'6769","reported_seq":6890,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T06:37:20.884896+0000","last_change":"2023-10-09T06:36:59.640496+0000","last_active":"2023-10-09T06:37:20.884896+0000","last_peered":"2023-10-09T06:37:20.884896+0000","last_clean":"2023-10-09T06:37:20.884896+0000","last_became_active":"2023-10-09T06:36:57.226174+0000","last_became_peered":"2023-10-09T06:36:57.226174+0000","last_unstale":"2023-10-09T06:37:20.884896+0000","last_undegraded":"2023-10-09T06:37:20.884896+0000","last_fullsized":"2023-10-09T06:37:20.884896+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"455'6312","last_scrub_stamp":"2023-10-09T06:36:59.640187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.640187+0000","objects_scrubbed":3175,"log_size":6769,"log_dups_size":0,"ondisk_log_size":6769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:01.040409+0000","scrub_duration":0.65782189000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":473,"num_write_kb":10496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":143,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289401+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:37:16.289401+0000","last_peered":"2023-10-09T06:37:16.289401+0000","last_clean":"2023-10-09T06:37:16.289401+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:37:16.289401+0000","last_undegraded":"2023-10-09T06:37:16.289401+0000","last_fullsized":"2023-10-09T06:37:16.289401+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":482,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.289276+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:37:16.289276+0000","last_peered":"2023-10-09T06:37:16.289276+0000","last_clean":"2023-10-09T06:37:16.289276+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:37:16.289276+0000","last_undegraded":"2023-10-09T06:37:16.289276+0000","last_fullsized":"2023-10-09T06:37:16.289276+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.10","version":"464'6656","reported_seq":8924,"reported_epoch":465,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:37:17.969171+0000","last_change":"2023-10-09T06:37:04.406619+0000","last_active":"2023-10-09T06:37:17.969171+0000","last_peered":"2023-10-09T06:37:17.969171+0000","last_clean":"2023-10-09T06:37:02.157053+0000","last_became_active":"2023-10-09T06:37:04.315163+0000","last_became_peered":"2023-10-09T06:37:04.315163+0000","last_unstale":"2023-10-09T06:37:17.969171+0000","last_undegraded":"2023-10-09T06:37:17.969171+0000","last_fullsized":"2023-10-09T06:37:03.549404+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6656,"log_dups_size":0,"ondisk_log_size":6656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203751424,"num_objects":3109,"num_object_clones":0,"num_object_copies":9327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2184,"num_objects_unfound":0,"num_objects_dirty":3109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3559,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1106,"num_bytes_recovered":72482816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":925},{"shards":"1,2,7","objects":2184}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":821,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.386182+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:37:16.386182+0000","last_peered":"2023-10-09T06:37:16.386182+0000","last_clean":"2023-10-09T06:37:16.386182+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:37:16.386182+0000","last_undegraded":"2023-10-09T06:37:16.386182+0000","last_fullsized":"2023-10-09T06:37:16.386182+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":167,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.301404+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:37:17.301404+0000","last_peered":"2023-10-09T06:37:17.301404+0000","last_clean":"2023-10-09T06:37:17.301404+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:37:17.301404+0000","last_undegraded":"2023-10-09T06:37:17.301404+0000","last_fullsized":"2023-10-09T06:37:17.301404+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":184,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.300698+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:37:17.300698+0000","last_peered":"2023-10-09T06:37:17.300698+0000","last_clean":"2023-10-09T06:37:17.300698+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:37:17.300698+0000","last_undegraded":"2023-10-09T06:37:17.300698+0000","last_fullsized":"2023-10-09T06:37:17.300698+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.11","version":"466'6680","reported_seq":6734,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:37:21.153091+0000","last_change":"2023-10-09T06:37:16.440148+0000","last_active":"2023-10-09T06:37:21.153091+0000","last_peered":"2023-10-09T06:37:21.153091+0000","last_clean":"2023-10-09T06:37:14.228029+0000","last_became_active":"2023-10-09T06:37:16.396220+0000","last_became_peered":"2023-10-09T06:37:16.396220+0000","last_unstale":"2023-10-09T06:37:21.153091+0000","last_undegraded":"2023-10-09T06:37:16.386662+0000","last_fullsized":"2023-10-09T06:37:16.288380+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196640768,"num_objects":3001,"num_object_clones":0,"num_object_copies":9003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3269,"num_objects_misplaced":3269,"num_objects_unfound":0,"num_objects_dirty":3001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3607,"num_write_kb":211424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2966},{"shards":"2,6","objects":-67}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":587,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.304196+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:37:17.304196+0000","last_peered":"2023-10-09T06:37:17.304196+0000","last_clean":"2023-10-09T06:37:17.304196+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:37:17.304196+0000","last_undegraded":"2023-10-09T06:37:17.304196+0000","last_fullsized":"2023-10-09T06:37:17.304196+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":462,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T06:37:17.298515+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:37:17.298515+0000","last_peered":"2023-10-09T06:37:17.298515+0000","last_clean":"2023-10-09T06:37:17.298515+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:37:17.298515+0000","last_undegraded":"2023-10-09T06:37:17.298515+0000","last_fullsized":"2023-10-09T06:37:17.298515+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":425,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-09T06:37:16.288753+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:37:16.288753+0000","last_peered":"2023-10-09T06:37:16.288753+0000","last_clean":"2023-10-09T06:37:16.288753+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:37:16.288753+0000","last_undegraded":"2023-10-09T06:37:16.288753+0000","last_fullsized":"2023-10-09T06:37:16.288753+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":6460014616,"num_objects":98573,"num_object_clones":0,"num_object_copies":295719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6611,"num_objects_misplaced":8795,"num_objects_unfound":0,"num_objects_dirty":98573,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":87927,"num_write_kb":5147713,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72548352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19368390656,"data_stored":19368378440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":166871,"ondisk_log_size":166871,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":512693,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":346022,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234480,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3183936,"kb_used_data":1995320,"kb_used_omap":0,"kb_used_meta":1188608,"kb_avail":90561216,"statfs":{"total":95995035648,"available":92734685184,"internally_reserved":0,"allocated":2043207680,"data_stored":2042163734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1217134592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.7269999999999999,"15min":6.085},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":8.8819999999999997,"5min":59.609000000000002,"15min":219.41900000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.8220000000000001,"5min":2.9319999999999999,"15min":6.2069999999999999},"min":{"1min":0.28799999999999998,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":5.2149999999999999,"5min":59.639000000000003,"15min":219.399},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 06:36:48 2023","interfaces":[{"interface":"back","average":{"1min":12.742000000000001,"5min":5.9649999999999999,"15min":5.4450000000000003},"min":{"1min":0.47399999999999998,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":181.15299999999999,"5min":208.626,"15min":215.61699999999999},"last":0.627},{"interface":"front","average":{"1min":13.336,"5min":3.8439999999999999,"15min":4.6029999999999998},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":181.24199999999999,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.57299999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:36:48 2023","interfaces":[{"interface":"back","average":{"1min":13.401999999999999,"5min":4.4939999999999998,"15min":7.6459999999999999},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":181.90899999999999,"5min":181.90899999999999,"15min":216.34299999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":12.965,"5min":4.359,"15min":7.7640000000000002},"min":{"1min":0.33500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":182.136,"5min":182.136,"15min":216.25200000000001},"last":0.81699999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.105,"15min":4.3200000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":5.2149999999999999,"5min":97.441999999999993,"15min":215.922},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.0049999999999999,"15min":4.3710000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":5.2839999999999998,"5min":97.162000000000006,"15min":215.94900000000001},"last":0.53600000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":2.649,"5min":1.0269999999999999,"15min":2.391},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":11.367000000000001,"5min":11.367000000000001,"15min":218.446},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.599,"5min":0.878,"15min":2.2919999999999998},"min":{"1min":0.29599999999999999,"5min":0.28399999999999997,"15min":0.23699999999999999},"max":{"1min":5.7889999999999997,"5min":5.7889999999999997,"15min":216.172},"last":0.59199999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":0.99299999999999999,"15min":2.129},"min":{"1min":0.32300000000000001,"5min":0.27600000000000002,"15min":0.22900000000000001},"max":{"1min":13.378,"5min":13.378,"15min":216.37200000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.5880000000000001,"5min":1.202,"15min":2.169},"min":{"1min":0.41099999999999998,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":17.087,"5min":17.087,"15min":218.15899999999999},"last":0.76000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":1.5529999999999999,"15min":2.4329999999999998},"min":{"1min":0.42799999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":11.321999999999999,"5min":48.283000000000001,"15min":218.19999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.4550000000000001,"15min":2.21},"min":{"1min":0.38200000000000001,"5min":0.36099999999999999,"15min":0.33200000000000002},"max":{"1min":7.3639999999999999,"5min":41.448999999999998,"15min":218.05500000000001},"last":0.439}]}]},{"osd":6,"up_from":46,"seq":197568496116,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2891364,"kb_used_data":1992348,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":90853788,"statfs":{"total":95995035648,"available":93034278912,"internally_reserved":0,"allocated":2040164352,"data_stored":2039147776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":4.1349999999999998,"15min":3.1680000000000001},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":4.1449999999999996,"5min":75.061999999999998,"15min":110.926},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.657,"5min":4.3639999999999999,"15min":3.4119999999999999},"min":{"1min":0.312,"5min":0.28299999999999997,"15min":0.23599999999999999},"max":{"1min":7.6520000000000001,"5min":75.021000000000001,"15min":110.911},"last":0.437}]},{"osd":1,"last update":"Mon Oct 9 06:36:56 2023","interfaces":[{"interface":"back","average":{"1min":2.774,"5min":3.2959999999999998,"15min":4.3209999999999997},"min":{"1min":0.26500000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.6029999999999998,"5min":65.153000000000006,"15min":292.92200000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.5529999999999999,"5min":3.2709999999999999,"15min":4.2460000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.255000000000001,"5min":65.132000000000005,"15min":292.904},"last":0.314}]},{"osd":2,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":3.4790000000000001,"15min":3.0419999999999998},"min":{"1min":0.35699999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.0270000000000001,"5min":74.748999999999995,"15min":75.191999999999993},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.048,"5min":3.2719999999999998,"15min":2.8679999999999999},"min":{"1min":0.34300000000000003,"5min":0.29399999999999998,"15min":0.27100000000000002},"max":{"1min":15.679,"5min":74.733999999999995,"15min":75.120000000000005},"last":0.42599999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":3.2210000000000001,"15min":3.5289999999999999},"min":{"1min":0.34100000000000003,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.8580000000000005,"5min":62.223999999999997,"15min":110.358},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.282,"5min":2.9460000000000002,"15min":3.4740000000000002},"min":{"1min":0.42999999999999999,"5min":0.34599999999999997,"15min":0.311},"max":{"1min":5.5780000000000003,"5min":61.637,"15min":110.43899999999999},"last":0.38600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7989999999999999,"5min":1.9570000000000001,"15min":1.48},"min":{"1min":0.316,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.3700000000000001,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.51100000000000001},{"interface":"front","average":{"1min":3.8380000000000001,"5min":2.2080000000000002,"15min":1.5369999999999999},"min":{"1min":0.33600000000000002,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":14.372,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":1.387,"15min":1.359},"min":{"1min":0.34399999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":22.379999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":4.1859999999999999,"5min":1.6419999999999999,"15min":1.5009999999999999},"min":{"1min":0.42399999999999999,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":12.244999999999999,"5min":12.244999999999999,"15min":24.401},"last":0.48699999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:36:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.028,"15min":2.5179999999999998},"min":{"1min":0.37,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":6.0069999999999997,"5min":7.4029999999999996,"15min":122.937},"last":0.498},{"interface":"front","average":{"1min":2.5640000000000001,"5min":1.319,"15min":2.702},"min":{"1min":0.45100000000000001,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":5.3810000000000002,"5min":7.4729999999999999,"15min":122.92100000000001},"last":0.52300000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402853,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2848652,"kb_used_data":1988356,"kb_used_omap":4,"kb_used_meta":860283,"kb_avail":90896500,"statfs":{"total":95995035648,"available":93078016000,"internally_reserved":0,"allocated":2036076544,"data_stored":2034037360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":880930702},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.7040000000000002,"15min":5.8140000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.189},"max":{"1min":5.0460000000000003,"5min":40.969999999999999,"15min":269.79199999999997},"last":2.4060000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.7629999999999999,"15min":6.2519999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.1890000000000001,"5min":40.880000000000003,"15min":271.20999999999998},"last":1.006}]},{"osd":2,"last update":"Mon Oct 9 06:36:47 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":2.3220000000000001,"15min":3.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":10.284000000000001,"5min":22.614999999999998,"15min":139.75399999999999},"last":2.3260000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.2669999999999999,"15min":4.2039999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.577,"5min":24.372,"15min":140.29900000000001},"last":1.0389999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":3.0449999999999999,"15min":3.2069999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.215},"max":{"1min":4.8959999999999999,"5min":27.064,"15min":77.331999999999994},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.081,"5min":2.1749999999999998,"15min":2.2610000000000001},"min":{"1min":0.251,"5min":0.23000000000000001,"15min":0.22700000000000001},"max":{"1min":10.228999999999999,"5min":26.626999999999999,"15min":77.197000000000003},"last":2.3730000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":9.3109999999999999,"5min":4.0359999999999996,"15min":3.4849999999999999},"min":{"1min":0.32300000000000001,"5min":0.317,"15min":0.27500000000000002},"max":{"1min":120.309,"5min":120.309,"15min":120.309},"last":2.2530000000000001},{"interface":"front","average":{"1min":9.6270000000000007,"5min":3.4990000000000001,"15min":3.3109999999999999},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":120.297,"5min":120.297,"15min":120.297},"last":2.3420000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":12.443,"5min":4.1639999999999997,"15min":2.7989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":119.408,"5min":119.408,"15min":119.408},"last":2.4359999999999999},{"interface":"front","average":{"1min":12.667999999999999,"5min":4.1890000000000001,"15min":3.1930000000000001},"min":{"1min":0.38300000000000001,"5min":0.371,"15min":0.32700000000000001},"max":{"1min":119.483,"5min":119.483,"15min":119.483},"last":1.0660000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":6.6100000000000003,"5min":3.8719999999999999,"15min":3.3700000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":121.60899999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":1.115},{"interface":"front","average":{"1min":6.3579999999999997,"5min":4.4619999999999997,"15min":3.5350000000000001},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":121.559,"5min":121.559,"15min":121.559},"last":1.149}]},{"osd":7,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8859999999999992,"5min":4.8499999999999996,"15min":3.8860000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":123.20699999999999,"5min":123.20699999999999,"15min":123.20699999999999},"last":1.857},{"interface":"front","average":{"1min":8.7650000000000006,"5min":4.2770000000000001,"15min":3.359},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":123.48999999999999,"5min":123.48999999999999,"15min":123.48999999999999},"last":2.4550000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024287,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2154440,"kb_used_data":1162560,"kb_used_omap":5,"kb_used_meta":991674,"kb_avail":91590712,"statfs":{"total":95995035648,"available":93788889088,"internally_reserved":0,"allocated":1190461440,"data_stored":1188086471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1015474810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:36:40 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.7759999999999998,"15min":2.6899999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":73.135000000000005,"5min":73.135000000000005,"15min":191.446},"last":0.46700000000000003},{"interface":"front","average":{"1min":5.1619999999999999,"5min":3.4329999999999998,"15min":3.0710000000000002},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.187},"max":{"1min":73.254999999999995,"5min":73.254999999999995,"15min":191.393},"last":0.751}]},{"osd":2,"last update":"Mon Oct 9 06:36:55 2023","interfaces":[{"interface":"back","average":{"1min":9.7970000000000006,"5min":4.931,"15min":3.2909999999999999},"min":{"1min":0.22700000000000001,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":100.425,"5min":100.425,"15min":100.425},"last":0.79400000000000004},{"interface":"front","average":{"1min":9.1319999999999997,"5min":5.1589999999999998,"15min":3.7080000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":99.769999999999996,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.70099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":3.0459999999999998,"15min":4.5170000000000003},"min":{"1min":0.253,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":9.1869999999999994,"5min":24.129000000000001,"15min":208.11199999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.369,"5min":3.1539999999999999,"15min":4.3659999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.7160000000000002,"5min":24.797000000000001,"15min":208.25899999999999},"last":0.83399999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":22.741,"5min":5.8259999999999996,"15min":2.9510000000000001},"min":{"1min":0.309,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":200.654,"5min":200.654,"15min":200.654},"last":0.34899999999999998},{"interface":"front","average":{"1min":23.821000000000002,"5min":6.4109999999999996,"15min":3.2829999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":200.86600000000001,"5min":200.86600000000001,"15min":200.86600000000001},"last":0.86599999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":5.4249999999999998,"5min":2.5310000000000001,"15min":1.887},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":80.885999999999996,"5min":80.885999999999996,"15min":80.885999999999996},"last":1.1910000000000001},{"interface":"front","average":{"1min":5.4409999999999998,"5min":2.5459999999999998,"15min":2.2469999999999999},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":80.438999999999993,"5min":80.438999999999993,"15min":80.438999999999993},"last":0.94699999999999995}]},{"osd":6,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":7.4649999999999999,"5min":3.1680000000000001,"15min":3.2450000000000001},"min":{"1min":0.36799999999999999,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":118.056,"5min":118.056,"15min":120.572},"last":0.81899999999999995},{"interface":"front","average":{"1min":7.3579999999999997,"5min":3.0619999999999998,"15min":2.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":118.274,"5min":118.274,"15min":120.64400000000001},"last":0.438}]},{"osd":7,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":2.2429999999999999,"15min":2.4089999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.32800000000000001},"max":{"1min":3.52,"5min":22.879000000000001,"15min":122.018},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.3180000000000001,"15min":2.7810000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.6840000000000002,"5min":22.683,"15min":122.32299999999999},"last":3.5710000000000002}]}]},{"osd":2,"up_from":128,"seq":549755814156,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4291596,"kb_used_data":3472068,"kb_used_omap":7,"kb_used_meta":819512,"kb_avail":89453556,"statfs":{"total":95995035648,"available":91600441344,"internally_reserved":0,"allocated":3555397632,"data_stored":3553422570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":839180369},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":8.5120000000000005,"15min":5.8860000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.183},"max":{"1min":3.4119999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.379},{"interface":"front","average":{"1min":0.79500000000000004,"5min":7.8609999999999998,"15min":5.3540000000000001},"min":{"1min":0.25600000000000001,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":4.1689999999999996,"5min":156.922,"15min":168.49700000000001},"last":0.26000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":6.9000000000000004,"15min":4.7729999999999997},"min":{"1min":0.216,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":4.5709999999999997,"5min":140.452,"15min":205.148},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.91400000000000003,"5min":6.5090000000000003,"15min":4.9669999999999996},"min":{"1min":0.28299999999999997,"5min":0.183,"15min":0.183},"max":{"1min":4.5209999999999999,"5min":139.75299999999999,"15min":205.267},"last":0.224}]},{"osd":3,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":4.6680000000000001,"5min":4.2919999999999998,"15min":3.5019999999999998},"min":{"1min":0.23799999999999999,"5min":0.216,"15min":0.20599999999999999},"max":{"1min":73.953000000000003,"5min":77.692999999999998,"15min":91.644000000000005},"last":0.28000000000000003},{"interface":"front","average":{"1min":4.5830000000000002,"5min":4.609,"15min":3.843},"min":{"1min":0.312,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":73.911000000000001,"5min":78.575999999999993,"15min":91.685000000000002},"last":0.29799999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":3.0249999999999999,"15min":2.5800000000000001},"min":{"1min":0.44800000000000001,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":27.542999999999999,"5min":73.932000000000002,"15min":90.938999999999993},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.379,"5min":3.347,"15min":2.8029999999999999},"min":{"1min":0.41799999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":27.533000000000001,"5min":74.349999999999994,"15min":90.950000000000003},"last":0.36399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":5.1159999999999997,"5min":2.9889999999999999,"15min":2.1680000000000001},"min":{"1min":0.34100000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":33.814,"5min":74.204999999999998,"15min":74.204999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.1840000000000002,"15min":2.2749999999999999},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":33.917000000000002,"5min":74.683000000000007,"15min":74.683000000000007},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":3.556,"15min":3.617},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.30199999999999999},"max":{"1min":6.6319999999999997,"5min":74.653999999999996,"15min":138.291},"last":0.313},{"interface":"front","average":{"1min":1.823,"5min":3.8540000000000001,"15min":3.4820000000000002},"min":{"1min":0.45400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.6050000000000004,"5min":73.837000000000003,"15min":138.01599999999999},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1379999999999999,"5min":4.0800000000000001,"15min":2.7959999999999998},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":36.155999999999999,"5min":73.863,"15min":73.863},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.6970000000000001,"5min":3.1920000000000002,"15min":2.512},"min":{"1min":0.51700000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":36.228000000000002,"5min":74.171999999999997,"15min":74.171999999999997},"last":0.35499999999999998}]}]},{"osd":3,"up_from":149,"seq":639950127353,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2317544,"kb_used_data":1584992,"kb_used_omap":2,"kb_used_meta":732541,"kb_avail":91427608,"statfs":{"total":95995035648,"available":93621870592,"internally_reserved":0,"allocated":1623031808,"data_stored":1621142498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":750122037},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":6.069,"15min":5.7510000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.19},"max":{"1min":2.0150000000000001,"5min":161.821,"15min":187.852},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":5.7850000000000001,"15min":5.766},"min":{"1min":0.20599999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":12.298,"5min":163.52000000000001,"15min":187.923},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":12.403,"5min":6.0599999999999996,"15min":4.9459999999999997},"min":{"1min":0.22800000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":184.012,"5min":184.012,"15min":187.89099999999999},"last":3.8479999999999999},{"interface":"front","average":{"1min":12.103999999999999,"5min":7.0019999999999998,"15min":4.9169999999999998},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":184.703,"5min":184.703,"15min":184.703},"last":0.44400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.893000000000001,"5min":6.0300000000000002,"15min":4.7149999999999999},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":182.44900000000001,"5min":182.44900000000001,"15min":187.76400000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":11.491,"5min":6.4260000000000002,"15min":4.8929999999999998},"min":{"1min":0.29599999999999999,"5min":0.27700000000000002,"15min":0.22500000000000001},"max":{"1min":182.56399999999999,"5min":182.56399999999999,"15min":187.79499999999999},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.643000000000001,"5min":3.9580000000000002,"15min":2.7189999999999999},"min":{"1min":0.41299999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":178.22200000000001,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":12.106999999999999,"5min":4.0609999999999999,"15min":3.331},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":180.43600000000001,"5min":180.43600000000001,"15min":187.67400000000001},"last":1.177}]},{"osd":5,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":11.898,"5min":4.9109999999999996,"15min":3.5379999999999998},"min":{"1min":0.441,"5min":0.35399999999999998,"15min":0.34000000000000002},"max":{"1min":177.56299999999999,"5min":177.56299999999999,"15min":187.833},"last":1.0840000000000001},{"interface":"front","average":{"1min":11.882999999999999,"5min":3.6309999999999998,"15min":2.9119999999999999},"min":{"1min":0.40899999999999997,"5min":0.32800000000000001,"15min":0.32100000000000001},"max":{"1min":177.59399999999999,"5min":177.59399999999999,"15min":187.952},"last":0.624}]},{"osd":6,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":2.1379999999999999,"15min":3.2829999999999999},"min":{"1min":0.41799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.1970000000000001,"5min":16.462,"15min":187.78200000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.363,"5min":1.7310000000000001,"15min":2.843},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":8.3350000000000009,"5min":22.260000000000002,"15min":187.81800000000001},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":2.0070000000000001,"15min":3.3319999999999999},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.195,"5min":13.695,"15min":192.20400000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.607,"5min":2.8849999999999998,"15min":3.6840000000000002},"min":{"1min":0.39100000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2619999999999996,"5min":87.001999999999995,"15min":187.86500000000001},"last":1.1519999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019389,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4447216,"kb_used_data":3560424,"kb_used_omap":0,"kb_used_meta":886784,"kb_avail":89297936,"statfs":{"total":95995035648,"available":91441086464,"internally_reserved":0,"allocated":3645874176,"data_stored":3644845713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":908066816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":10.279,"15min":6.7510000000000003},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.7160000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":0.26500000000000001},{"interface":"front","average":{"1min":2.988,"5min":10.6,"15min":6.9429999999999996},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":9.2110000000000003,"5min":286.36000000000001,"15min":287.57100000000003},"last":0.35199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":4.2270000000000003,"15min":3.1699999999999999},"min":{"1min":0.246,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":34.548999999999999,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":3.262,"5min":4.327,"15min":3.335},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":31.024999999999999,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.423,"5min":5.8159999999999998,"15min":3.8050000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":33.085000000000001,"5min":151.66399999999999,"15min":151.66399999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.569,"5min":5.6989999999999998,"15min":3.7519999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":32.299999999999997,"5min":151.58699999999999,"15min":151.58699999999999},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":6.1849999999999996,"15min":5.3040000000000003},"min":{"1min":0.39200000000000002,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":9.5359999999999996,"5min":161.19300000000001,"15min":161.19300000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.3360000000000001,"5min":6.1909999999999998,"15min":5.0860000000000003},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":5.4329999999999998,"5min":161.934,"15min":161.934},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":0.94299999999999995,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.252},"max":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":9.1720000000000006},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.3359999999999999,"5min":0.98699999999999999,"15min":0.99199999999999999},"min":{"1min":0.309,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":9.5009999999999994,"5min":9.5009999999999994,"15min":9.6379999999999999},"last":0.31}]},{"osd":6,"last update":"Mon Oct 9 06:37:14 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":2.4900000000000002,"15min":1.8620000000000001},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":9.4239999999999995,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.0819999999999999,"5min":2.7799999999999998,"15min":2.165},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":10.705,"5min":115.68600000000001,"15min":163.381},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.8029999999999999,"15min":1.863},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":3.3940000000000001,"5min":106.801,"15min":106.801},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.724,"5min":2.9319999999999999,"15min":1.877},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":16.713000000000001,"5min":106.827,"15min":106.827},"last":0.42099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757754,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4110436,"kb_used_data":3212828,"kb_used_omap":0,"kb_used_meta":897600,"kb_avail":89634716,"statfs":{"total":95995035648,"available":91785949184,"internally_reserved":0,"allocated":3289935872,"data_stored":3288915925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919142400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:08 2023","interfaces":[{"interface":"back","average":{"1min":10.744,"5min":6.327,"15min":6.3399999999999999},"min":{"1min":0.21099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":147.405,"5min":149.149,"15min":294.89699999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":15.861000000000001,"5min":6.9119999999999999,"15min":6.4699999999999998},"min":{"1min":0.25600000000000001,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":146.31700000000001,"5min":149.21100000000001,"15min":294.30399999999997},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":21.149000000000001,"5min":6.5359999999999996,"15min":4.3869999999999996},"min":{"1min":0.29699999999999999,"5min":0.22,"15min":0.22},"max":{"1min":188.34100000000001,"5min":188.34100000000001,"15min":188.34100000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":21.187999999999999,"5min":6.4420000000000002,"15min":4.4930000000000003},"min":{"1min":0.34200000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":188.839,"5min":188.839,"15min":188.839},"last":0.43099999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":9.5540000000000003,"5min":4.8109999999999999,"15min":3.2250000000000001},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":80.715999999999994,"5min":120.613,"15min":120.613},"last":0.47699999999999998},{"interface":"front","average":{"1min":7.4580000000000002,"5min":4.4749999999999996,"15min":3.0529999999999999},"min":{"1min":0.315,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":75.640000000000001,"5min":120.962,"15min":120.962},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:37:15 2023","interfaces":[{"interface":"back","average":{"1min":7.6399999999999997,"5min":4.8300000000000001,"15min":5.3170000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":78.859999999999999,"5min":85.671000000000006,"15min":144.56399999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":7.077,"5min":4.3570000000000002,"15min":5.0819999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.28699999999999998},"max":{"1min":78.944999999999993,"5min":85.593999999999994,"15min":144.62799999999999},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4359999999999999,"5min":2.0270000000000001,"15min":1.444},"min":{"1min":0.35499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":79.043000000000006,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.33300000000000002},{"interface":"front","average":{"1min":7.7809999999999997,"5min":2.0419999999999998,"15min":1.385},"min":{"1min":0.35599999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":79.165999999999997,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:37:08 2023","interfaces":[{"interface":"back","average":{"1min":5.2359999999999998,"5min":1.611,"15min":1.673},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":79.013999999999996,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.55400000000000005},{"interface":"front","average":{"1min":5.5810000000000004,"5min":1.764,"15min":1.764},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":80.781999999999996,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:36:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0369999999999999,"5min":1.901,"15min":1.583},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":78.890000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":5.3070000000000004,"5min":1.6839999999999999,"15min":1.587},"min":{"1min":0.40300000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":79.138999999999996,"5min":79.138999999999996,"15min":79.138999999999996},"last":0.51800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1183711232,"data_stored":1183711232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2030374912,"data_stored":2030370840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3544711168,"data_stored":3544711168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1617166336,"data_stored":1617166336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3640725504,"data_stored":3640721432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3287089152,"data_stored":3287089152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2026242048,"data_stored":2026242048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2038370304,"data_stored":2038366232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:37:23.029 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 26 2023-10-09T06:37:23.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:23.081 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:23.162 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:23.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:23.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:23.269 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:23.310 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:23.381 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:23.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:23.411 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:23 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 6611/295791 objects degraded (2.235%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:37:23.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:23 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 6611/295791 objects degraded (2.235%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:37:23.487 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:23.563 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:23.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:23 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 6611/295791 objects degraded (2.235%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:37:23.662 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:23.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:23.780 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:23.806 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:23.848 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:23.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:23.954 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:24.019 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:24.112 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:24.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:24.223 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:24.283 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:24.378 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:24.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:24.493 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:24.516 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:24 smithi107 ceph-mon[212750]: pgmap v1415: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 6611/286179 objects degraded (2.310%); 8417/286179 objects misplaced (2.941%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T06:37:24.516 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:24 smithi107 ceph-mon[220921]: pgmap v1415: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 6611/286179 objects degraded (2.310%); 8417/286179 objects misplaced (2.941%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T06:37:24.538 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:24 smithi150 ceph-mon[194178]: pgmap v1415: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 6611/286179 objects degraded (2.310%); 8417/286179 objects misplaced (2.941%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T06:37:24.542 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:24.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:24.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:24.749 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:24.790 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:24.887 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:24.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:24.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:25.084 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:25.370 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:25.404 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:37:25.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:25.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:25.694 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:25.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:25.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:25.803 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:25.868 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:25.920 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:26.251 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:26.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:26.364 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:26.436 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:26.520 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:26.520 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:26.528 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:26.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[220921]: pgmap v1417: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 6611/284427 objects degraded (2.324%); 8417/284427 objects misplaced (2.959%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.548 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:26 smithi107 ceph-mon[212750]: pgmap v1417: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 6611/284427 objects degraded (2.324%); 8417/284427 objects misplaced (2.959%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:37:26.644 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/180080526' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: failed to encode map e467 with expected crc 2023-10-09T06:37:26.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:26 smithi150 ceph-mon[194178]: pgmap v1417: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 6611/284427 objects degraded (2.324%); 8417/284427 objects misplaced (2.959%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T06:37:26.733 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:26.920 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:26.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:27.029 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:27.055 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:27.542 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:27.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:27.647 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:27.703 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:27.746 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:37:27.824 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6611/283077 objects degraded (2.335%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:27.825 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6611/283077 objects degraded (2.335%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6611/283077 objects degraded (2.335%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:27.909 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:27.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:27.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:28.014 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:28.048 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:28.111 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:37:28.129 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:28.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:28.149 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:37:28.191 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:37:28.232 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:37:28.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:28.277 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:37:28.301 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:28.316 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:37:28.356 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:37:28.380 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:28.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:28.402 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.410 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:37:28.488 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:28.554 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:28.648 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:28.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:28.753 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[212750]: pgmap v1418: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 6611/283077 objects degraded (2.335%); 8417/283077 objects misplaced (2.973%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4194416002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[220921]: pgmap v1418: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 6611/283077 objects degraded (2.335%); 8417/283077 objects misplaced (2.973%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:28.779 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4194416002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:28.781 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:28 smithi150 ceph-mon[194178]: pgmap v1418: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 6611/283077 objects degraded (2.335%); 8417/283077 objects misplaced (2.973%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:37:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:28 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4194416002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:29.658 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:37:29.999 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:30.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:30.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:30.113 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:30.146 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:30.172 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:30.265 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:30.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:30.377 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:30.459 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:30.597 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:30 smithi150 ceph-mon[194178]: pgmap v1419: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 6611/269589 objects degraded (2.452%); 8065/269589 objects misplaced (2.992%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:30.716 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:30.729 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:30 smithi107 ceph-mon[220921]: pgmap v1419: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 6611/269589 objects degraded (2.452%); 8065/269589 objects misplaced (2.992%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:30.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:30.730 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:30 smithi107 ceph-mon[212750]: pgmap v1419: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 6611/269589 objects degraded (2.452%); 8065/269589 objects misplaced (2.992%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:30.830 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:30.896 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:30.994 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:31.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:31.002 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:31.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:31.158 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:31.268 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:31.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:31.383 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:31.458 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:31.558 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:31.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:31.610 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 13 pgp_num to 26 2023-10-09T06:37:31.633 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:31.633 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:31.667 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:31.697 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:31.743 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:31.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:31.846 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:31.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:31.908 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:32.133 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:37:32.189 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:32] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T06:37:32.418 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:32.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:32.524 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:32.578 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:32.665 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:32.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:32.777 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: pgmap v1420: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 6611/268170 objects degraded (2.465%); 8065/268170 objects misplaced (3.007%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:32.825 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6611/268170 objects degraded (2.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:32.829 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:32.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:32.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: pgmap v1420: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 6611/268170 objects degraded (2.465%); 8065/268170 objects misplaced (3.007%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6611/268170 objects degraded (2.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: pgmap v1420: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 6611/268170 objects degraded (2.465%); 8065/268170 objects misplaced (3.007%); 4.6 MiB/s, 72 objects/s recovering 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: failed to encode map e468 with expected crc 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4077480359' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6611/268170 objects degraded (2.465%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:32.990 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:33.038 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:33.945 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:33.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:34.054 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:34.111 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:34.233 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:34.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:34.341 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:34.415 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:34.510 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:34.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:34.622 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:34.699 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:34.711 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:34.873 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:34.878 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:37:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:37:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:37:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104909 12.13 12:cd32261f:::benchmark_data_smithi107_479504_object5298:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.45700267299998, 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.143602357, 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104909 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413794+0000", 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413797+0000", 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.965e-06 2023-10-09T06:37:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413803+0000", 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1959999999999996e-06 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413809+0000", 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3020000000000002e-06 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413951+0000", 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014192100000000001 2023-10-09T06:37:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413960+0000", 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5750000000000007e-06 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.536884+0000", 2023-10-09T06:37:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.122924037 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.537024+0000", 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000139682 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.537140+0000", 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116098 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542524+0000", 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053841330000000001 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546061+0000", 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003536951 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555262+0000", 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0092012450000000003 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555328+0000", 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5580999999999994e-05 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557397+0000", 2023-10-09T06:37:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020690410000000002 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104929 12.13 12:c993a819:::benchmark_data_smithi107_479504_object5318:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.45697856999999, 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1455242480000001, 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104929 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413818+0000", 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413821+0000", 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2630000000000002e-06 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413827+0000", 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6440000000000003e-06 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413832+0000", 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0320000000000003e-06 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413980+0000", 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000148188 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413987+0000", 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3110000000000002e-06 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540392+0000", 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1264056950000001 2023-10-09T06:37:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540559+0000", 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000166832 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540720+0000", 2023-10-09T06:37:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016054 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542567+0000", 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018473140000000001 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546204+0000", 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036369829999999999 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557461+0000", 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01125724 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557519+0000", 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7157000000000002e-05 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559343+0000", 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018243999999999999 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104981 12.13 12:ca7186c7:::benchmark_data_smithi107_479504_object5370:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.45696134399998, 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1475939610000001, 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104981 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413838+0000", 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9589999999999998e-06 2023-10-09T06:37:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413843+0000", 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6010000000000002e-06 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413848+0000", 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5379999999999996e-06 2023-10-09T06:37:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.414006+0000", 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015859700000000001 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.414012+0000", 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0499999999999997e-06 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542596+0000", 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.128583541 2023-10-09T06:37:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542754+0000", 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015784600000000001 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542904+0000", 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015026899999999999 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546127+0000", 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032231130000000001 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546213+0000", 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6285000000000005e-05 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559478+0000", 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013264664000000001 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559548+0000", 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9995999999999999e-05 2023-10-09T06:37:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561430+0000", 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018817879999999999 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105015 12.13 12:cc2c9607:::benchmark_data_smithi107_479504_object5404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:37:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.44429549099999, 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1370335789999999, 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105015 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426501+0000", 2023-10-09T06:37:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426504+0000", 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4200000000000001e-06 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426510+0000", 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7710000000000003e-06 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426515+0000", 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1329999999999998e-06 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.428629+0000", 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021145769999999999 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.912 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.428638+0000", 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3359999999999995e-06 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544481+0000", 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.115843734 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544606+0000", 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000124144 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544711+0000", 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105208 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546191+0000", 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001480669 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548067+0000", 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018758469999999999 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561516+0000", 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013448422999999999 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561553+0000", 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7020999999999997e-05 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563535+0000", 2023-10-09T06:37:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001982667 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105018 12.13 12:cbda2284:::benchmark_data_smithi107_479504_object5407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.44110898999998, 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1380620459999999, 2023-10-09T06:37:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105018 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429687+0000", 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429693+0000", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3940000000000004e-06 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429702+0000", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8470000000000008e-06 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429710+0000", 2023-10-09T06:37:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9019999999999999e-06 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429756+0000", 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6026999999999998e-05 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429767+0000", 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0538e-05 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546258+0000", 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.116491232 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546428+0000", 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016985500000000001 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546592+0000", 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016419699999999999 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548119+0000", 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015267690000000001 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551585+0000", 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034658010000000001 2023-10-09T06:37:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.565667+0000", 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014082479 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.565724+0000", 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6938000000000003e-05 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567750+0000", 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020260959999999998 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105034 12.13 12:c9f14c5b:::benchmark_data_smithi107_479504_object5423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.421446398, 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.126998041, 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105034 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449350+0000", 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449354+0000", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4649999999999999e-06 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449367+0000", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2629000000000001e-05 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449373+0000", 2023-10-09T06:37:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449896+0000", 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052338799999999996 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449907+0000", 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1019000000000001e-05 2023-10-09T06:37:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548166+0000", 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0982583720000001 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548285+0000", 2023-10-09T06:37:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011966 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548386+0000", 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100751 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551503+0000", 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031171810000000001 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551596+0000", 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2949999999999996e-05 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574084+0000", 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.022488121999999999 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574136+0000", 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1932999999999999e-05 2023-10-09T06:37:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576349+0000", 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002212533 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105111 12.13 12:c8794572:::benchmark_data_smithi107_479504_object5500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.33662958899998, 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.044615676, 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105111 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534167+0000", 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534170+0000", 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7980000000000001e-06 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534176+0000", 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8270000000000003e-06 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534181+0000", 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0250000000000002e-06 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.939 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534253+0000", 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2373999999999995e-05 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534263+0000", 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3230000000000004e-06 2023-10-09T06:37:34.940 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.549708+0000", 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0154453960000001 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.941 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.549877+0000", 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016942699999999999 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.550047+0000", 2023-10-09T06:37:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000169836 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551570+0000", 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015229989999999999 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555183+0000", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036130289999999998 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576445+0000", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021261571999999999 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576506+0000", 2023-10-09T06:37:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1013e-05 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578783+0000", 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002277477 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105179 12.13 12:cdc11d2f:::benchmark_data_smithi107_479504_object5568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.25036929100003, 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96059906500000003, 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105179 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620426+0000", 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620434+0000", 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1610000000000003e-06 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620451+0000", 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6575999999999999e-05 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620460+0000", 2023-10-09T06:37:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4739999999999994e-06 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620504+0000", 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3748000000000002e-05 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620512+0000", 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6279999999999996e-06 2023-10-09T06:37:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551645+0000", 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.93113328500000003 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551821+0000", 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017628899999999999 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551998+0000", 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017652 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559364+0000", 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073662279999999998 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559423+0000", 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9576e-05 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578913+0000", 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019489689000000001 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578975+0000", 2023-10-09T06:37:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1656999999999996e-05 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581027+0000", 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020522909999999999 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105184 12.13 12:cef60faf:::benchmark_data_smithi107_479504_object5573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.24508353099998, 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.95766810999999996, 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105184 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625712+0000", 2023-10-09T06:37:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625720+0000", 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625742+0000", 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2001000000000002e-05 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625752+0000", 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0223e-05 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625803+0000", 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0695000000000002e-05 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625814+0000", 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.095e-05 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553587+0000", 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.92777310000000002 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553744+0000", 2023-10-09T06:37:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015638300000000001 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553844+0000", 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9994000000000005e-05 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559433+0000", 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055894350000000002 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561489+0000", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002055996 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581100+0000", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019610909999999999 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581155+0000", 2023-10-09T06:37:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4753000000000002e-05 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583382+0000", 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022271740000000002 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105256 12.13 12:cfe2b70d:::benchmark_data_smithi107_479504_object5645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.141770431, 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.86023841499999998, 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105256 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729025+0000", 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729034+0000", 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5469999999999997e-06 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729051+0000", 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7402999999999999e-05 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729061+0000", 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8989999999999996e-06 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.731930+0000", 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002869099 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.731948+0000", 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7538e-05 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563570+0000", 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83162247 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563710+0000", 2023-10-09T06:37:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013946400000000001 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563844+0000", 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013422999999999999 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.569803+0000", 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059588599999999999 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.573966+0000", 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0041634949999999997 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.586857+0000", 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012890301999999999 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.586907+0000", 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0142999999999997e-05 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589265+0000", 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023584819999999999 2023-10-09T06:37:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105289 12.13 12:cbcbb33f:::benchmark_data_smithi107_479504_object5678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.096273338, 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.81706288400000004, 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105289 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774522+0000", 2023-10-09T06:37:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774532+0000", 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0095000000000001e-05 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774560+0000", 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8200000000000001e-05 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774572+0000", 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1697e-05 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774625+0000", 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3347000000000003e-05 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774635+0000", 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.815e-06 2023-10-09T06:37:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567795+0000", 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.79315939099999999 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567947+0000", 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015192999999999999 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.568055+0000", 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108917 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574011+0000", 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059552269999999996 2023-10-09T06:37:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574020+0000", 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6970000000000005e-06 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589336+0000", 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015315702 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589393+0000", 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6461e-05 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.591587+0000", 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002194227 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105323 12.13 12:cc0ff923:::benchmark_data_smithi107_479504_object5712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.041069284, 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.76640025000000001, 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105323 2023-10-09T06:37:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829726+0000", 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829734+0000", 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9030000000000004e-06 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829753+0000", 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9004999999999998e-05 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829763+0000", 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3130000000000004e-06 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829812+0000", 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9391000000000002e-05 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829824+0000", 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1806e-05 2023-10-09T06:37:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.569832+0000", 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.74000779400000005 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.570011+0000", 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000179548 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.570169+0000", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000157448 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574027+0000", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038583659999999998 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576408+0000", 2023-10-09T06:37:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002380868 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593714+0000", 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017305803000000002 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593760+0000", 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6144999999999999e-05 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596128+0000", 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023683630000000001 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105350 12.13 12:ca4b3448:::benchmark_data_smithi107_479504_object5739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:37:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 282.00368647599998, 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.73142369299999999, 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105350 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867109+0000", 2023-10-09T06:37:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867116+0000", 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8619999999999997e-06 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867132+0000", 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6353000000000001e-05 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867141+0000", 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3659999999999992e-06 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867199+0000", 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.817e-05 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867210+0000", 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1029999999999999e-05 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.571921+0000", 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70471069399999997 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.572088+0000", 2023-10-09T06:37:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016699999999999999 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.572281+0000", 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00019307300000000001 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578846+0000", 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065657179999999999 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583402+0000", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045557030000000004 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596203+0000", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012800638 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596270+0000", 2023-10-09T06:37:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6743999999999997e-05 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598534+0000", 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022649459999999999 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105356 12.13 12:c80cc14f:::benchmark_data_smithi107_479504_object5745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 281.993666711, 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72982949600000002, 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105356 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877129+0000", 2023-10-09T06:37:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877137+0000", 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.622e-06 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877149+0000", 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1925e-05 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877157+0000", 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2349999999999999e-06 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877198+0000", 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0849000000000001e-05 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877205+0000", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0530000000000001e-06 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583440+0000", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70623574200000006 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583607+0000", 2023-10-09T06:37:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000166047 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583764+0000", 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015757600000000001 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593599+0000", 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.009835313 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593646+0000", 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6295999999999999e-05 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598596+0000", 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0049505399999999998 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598649+0000", 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2311000000000001e-05 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.606960+0000", 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083114710000000008 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:180931 12.13 12:ca7369d7:::benchmark_data_smithi107_479504_object81320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 176.11364093200001, 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34981270800000003, 2023-10-09T06:37:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 180931 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757155+0000", 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757160+0000", 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4e-06 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757172+0000", 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1929999999999999e-05 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757178+0000", 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0190000000000003e-06 2023-10-09T06:37:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758011+0000", 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00083210300000000005 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758143+0000", 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000132414 2023-10-09T06:37:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758646+0000", 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00050300799999999999 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.764111+0000", 2023-10-09T06:37:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054646440000000003 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.768008+0000", 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038976800000000002 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106146+0000", 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33813771100000001 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106198+0000", 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1839999999999998e-05 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106969+0000", 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00077123899999999995 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194026 12.e 12:7a8b5efc:::benchmark_data_smithi107_479504_object94415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:37:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 156.93954294900001, 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35583712499999998, 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194026 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931257+0000", 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7099999999999999e-06 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931263+0000", 2023-10-09T06:37:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2750000000000001e-06 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931268+0000", 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931301+0000", 2023-10-09T06:37:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3293999999999997e-05 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931411+0000", 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109904 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931511+0000", 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9732e-05 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940660+0000", 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091492589999999999 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940805+0000", 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014499900000000001 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946883+0000", 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006077753 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946930+0000", 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6730000000000002e-05 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287091+0000", 2023-10-09T06:37:35.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016162900000002 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194028 12.e 12:78e231c3:::benchmark_data_smithi107_479504_object94417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 156.93951759999999, 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35590891000000002, 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:35.025 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194028 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931279+0000", 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931282+0000", 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6410000000000002e-06 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931288+0000", 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9329999999999998e-06 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931293+0000", 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.147e-06 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936701+0000", 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054079769999999996 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936818+0000", 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011694 2023-10-09T06:37:35.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936931+0000", 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011294999999999999 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946850+0000", 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099191780000000007 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946867+0000", 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6532e-05 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287104+0000", 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34023740699999999 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287151+0000", 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7079000000000001e-05 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287189+0000", 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7576000000000001e-05 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.032 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194030 12.a 12:5bb1da5c:::benchmark_data_smithi107_479504_object94419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 156.93950024, 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35582703900000001, 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194030 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931299+0000", 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0710000000000002e-06 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931304+0000", 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9439999999999996e-06 2023-10-09T06:37:35.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931309+0000", 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4859999999999997e-06 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938266+0000", 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069569489999999996 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938372+0000", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105999 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938476+0000", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103669 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.942149+0000", 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003673277 2023-10-09T06:37:35.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.945613+0000", 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034637270000000002 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946916+0000", 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013029070000000001 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287077+0000", 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016102300000001 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287124+0000", 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7277999999999997e-05 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98792 13.13 13:cfe6566e:::benchmark_data_smithi107_535588_object98791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.617169938000004, 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97467514200000005, 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98792 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.040 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253625+0000", 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253660+0000", 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5043000000000003e-05 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253679+0000", 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9123999999999999e-05 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253690+0000", 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0679000000000001e-05 2023-10-09T06:37:35.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259065+0000", 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053750710000000004 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259078+0000", 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3473e-05 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228190+0000", 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96911134899999996 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228302+0000", 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000112927 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98797 13.3 13:c2aca6be:::benchmark_data_smithi107_535588_object98796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 38.517675136000001, 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.85765422099999999, 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:37:35.046 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98797 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:37:35.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353120+0000", 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353150+0000", 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0685000000000002e-05 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353166+0000", 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5914e-05 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353175+0000", 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.935e-06 2023-10-09T06:37:35.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353222+0000", 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6635999999999999e-05 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353232+0000", 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0478e-05 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188558+0000", 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83532556599999996 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188732+0000", 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173661 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188879+0000", 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014706499999999999 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:37:35.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.192739+0000", 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038604580000000002 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.201831+0000", 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090918390000000009 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210658+0000", 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088270789999999998 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210725+0000", 2023-10-09T06:37:35.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7260000000000003e-05 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210776+0000", 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1085000000000003e-05 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.055 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:37:35.056 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:37:35.056 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:37:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:35.057 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:34 smithi150 ceph-mon[194178]: pgmap v1422: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 6611/256875 objects degraded (2.574%); 7674/256875 objects misplaced (2.987%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:37:35.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:35.058 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:34 smithi107 ceph-mon[212750]: pgmap v1422: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 6611/256875 objects degraded (2.574%); 7674/256875 objects misplaced (2.987%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:37:35.059 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:34 smithi107 ceph-mon[220921]: pgmap v1422: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 6611/256875 objects degraded (2.574%); 7674/256875 objects misplaced (2.987%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T06:37:35.160 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:35.231 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:35.372 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:35.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:35.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:35.481 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:35.567 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:36.153 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:36.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:36.219 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:37:36.258 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.362 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:36.595 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:36.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:36.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:36.696 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:37:36.706 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:36.738 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[212750]: pgmap v1423: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 6611/255255 objects degraded (2.590%); 7674/255255 objects misplaced (3.006%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3781933674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[220921]: pgmap v1423: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 6611/255255 objects degraded (2.590%); 7674/255255 objects misplaced (3.006%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3781933674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:36.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:36 smithi150 ceph-mon[194178]: pgmap v1423: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 6611/255255 objects degraded (2.590%); 7674/255255 objects misplaced (3.006%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T06:37:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3781933674' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:36.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:37:37.475 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:37.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:37.582 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[220921]: failed to encode map e469 with expected crc 2023-10-09T06:37:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[220921]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:37:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6611/255255 objects degraded (2.590%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:37.652 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:37.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:37.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[212750]: failed to encode map e469 with expected crc 2023-10-09T06:37:37.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:37.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[212750]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:37:37.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6611/255255 objects degraded (2.590%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:37.751 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:37.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:37.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:37.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:37 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:37:37.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:37 smithi150 ceph-mon[194178]: failed to encode map e469 with expected crc 2023-10-09T06:37:37.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:37:37.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:37 smithi150 ceph-mon[194178]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T06:37:37.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6611/255255 objects degraded (2.590%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:37.933 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:38.248 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:38.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:38.357 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:38.385 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:38.406 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:38.473 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:38.481 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:37:38.481 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:37:38.481 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-10-09T06:37:38.482 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:37:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:38.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:38.584 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:37:38.590 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:37:38.634 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:37:38.684 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:37:38.700 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:38.744 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:37:38.780 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:37:38.809 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[212750]: failed to encode map e469 with expected crc 2023-10-09T06:37:38.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:37:38.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[212750]: pgmap v1425: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 6611/253953 objects degraded (2.603%); 7674/253953 objects misplaced (3.022%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:37:38.810 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/697299782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:38.813 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[220921]: failed to encode map e469 with expected crc 2023-10-09T06:37:38.813 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:37:38.813 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[220921]: pgmap v1425: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 6611/253953 objects degraded (2.603%); 7674/253953 objects misplaced (3.022%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:37:38.813 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/697299782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:38.840 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:37:38.841 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:37:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:38.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:37:38.877 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:37:38.912 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:37:38.920 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.920 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.920 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.921 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.921 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.921 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.921 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.921 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:37:38.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:39.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:38 smithi150 ceph-mon[194178]: failed to encode map e469 with expected crc 2023-10-09T06:37:39.001 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:38 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:37:39.001 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:38 smithi150 ceph-mon[194178]: pgmap v1425: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 6611/253953 objects degraded (2.603%); 7674/253953 objects misplaced (3.022%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:37:39.001 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/697299782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:39.003 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:39.089 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:39.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:39.197 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:39.282 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:39.380 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:39.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:39.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:39.557 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:39 smithi107 ceph-mon[212750]: 13.18 scrub starts 2023-10-09T06:37:39.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:39 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:37:39.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:39 smithi107 ceph-mon[220921]: 13.18 scrub starts 2023-10-09T06:37:39.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:39 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:37:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:39 smithi150 ceph-mon[194178]: 13.18 scrub starts 2023-10-09T06:37:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:39 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:37:40.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:37:40.600 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:40.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:40.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:40.705 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:40.781 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:40.871 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:40 smithi107 ceph-mon[212750]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 6611/240534 objects degraded (2.748%); 7272/240534 objects misplaced (3.023%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:37:40.871 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:40 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:37:40.872 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:40 smithi107 ceph-mon[220921]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 6611/240534 objects degraded (2.748%); 7272/240534 objects misplaced (3.023%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:37:40.872 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:40 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:37:40.928 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:40.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:41.039 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:41.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:40 smithi150 ceph-mon[194178]: pgmap v1426: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 6611/240534 objects degraded (2.748%); 7272/240534 objects misplaced (3.023%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T06:37:41.083 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:40 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:37:41.089 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:41.154 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:41.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:41.246 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:41.263 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:41.304 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:41.387 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:41.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:41.410 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:37:41.410 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:37:41.442 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:41.499 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:41.560 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:41.665 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:41.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:41.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:41.713 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:41 smithi107 ceph-mon[212750]: 13.15 scrub starts 2023-10-09T06:37:41.713 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:41 smithi107 ceph-mon[220921]: 13.15 scrub starts 2023-10-09T06:37:41.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:41.854 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:41 smithi150 ceph-mon[194178]: 13.15 scrub starts 2023-10-09T06:37:42.184 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:42.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:42.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:42.288 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:42.338 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:42] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T06:37:42.349 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:42.460 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:42.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:42.573 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:42.628 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:42.750 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:42.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:42.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:42.825 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:37:42.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:42.870 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:37:42.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:42 smithi150 ceph-mon[194178]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 6611/239130 objects degraded (2.765%); 7272/239130 objects misplaced (3.041%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:37:42.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:42 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:37:42.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:42 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6611/239130 objects degraded (2.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:42.919 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:42.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[220921]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 6611/239130 objects degraded (2.765%); 7272/239130 objects misplaced (3.041%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:37:42.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:37:42.986 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6611/239130 objects degraded (2.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:42.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[212750]: pgmap v1427: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 4.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 980 op/s; 6611/239130 objects degraded (2.765%); 7272/239130 objects misplaced (3.041%); 5.0 MiB/s, 80 objects/s recovering 2023-10-09T06:37:42.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:37:42.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:42 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6611/239130 objects degraded (2.765%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:43.058 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:43.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:43.164 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:43.228 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:43.325 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:43.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:43.432 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:43.480 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:43.600 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:43.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:43.715 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:43.772 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:43 smithi150 ceph-mon[194178]: 13.7 scrub starts 2023-10-09T06:37:43.788 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:43.877 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:43.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:43.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:43 smithi107 ceph-mon[212750]: 13.7 scrub starts 2023-10-09T06:37:43.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:43 smithi107 ceph-mon[220921]: 13.7 scrub starts 2023-10-09T06:37:43.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:44.048 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:44.712 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:44.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:44.820 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:44.866 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:44.963 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:44.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:44.973 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:45.071 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:45.117 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:44 smithi107 ceph-mon[212750]: 13.7 scrub ok 2023-10-09T06:37:45.118 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:44 smithi107 ceph-mon[212750]: pgmap v1428: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 6611/227838 objects degraded (2.902%); 6820/227838 objects misplaced (2.993%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:45.123 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:44 smithi107 ceph-mon[220921]: 13.7 scrub ok 2023-10-09T06:37:45.123 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:44 smithi107 ceph-mon[220921]: pgmap v1428: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 6611/227838 objects degraded (2.902%); 6820/227838 objects misplaced (2.993%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:45.123 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:45.132 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:44 smithi150 ceph-mon[194178]: 13.7 scrub ok 2023-10-09T06:37:45.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:44 smithi150 ceph-mon[194178]: pgmap v1428: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 6611/227838 objects degraded (2.902%); 6820/227838 objects misplaced (2.993%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:45.426 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:45.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:45.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:45.487 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:37:45.502 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:37:45.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:45.617 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:45.825 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:45.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:45.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:45.948 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:45.988 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:46.052 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:46.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:46.063 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:45 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:37:46.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:45 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:37:46.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:45 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:37:46.163 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:46.204 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:46.295 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:46.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:46.403 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:46.429 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:46.491 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:46.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:46.601 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:46.633 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:46.695 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:46.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:46.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:46.751 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:37:46.802 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:46.848 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 6611/226089 objects degraded (2.924%); 6820/226089 objects misplaced (3.017%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:37:47.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 6611/226089 objects degraded (2.924%); 6820/226089 objects misplaced (3.017%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:47.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: from='client.36924 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:37:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: pgmap v1429: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 6611/226089 objects degraded (2.924%); 6820/226089 objects misplaced (3.017%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T06:37:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T06:37:47.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: failed to encode map e470 with expected crc 2023-10-09T06:37:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:37:47.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 6611/226089 objects degraded (2.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:47 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 6611/226089 objects degraded (2.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3701890039' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: failed to encode map e471 with expected crc 2023-10-09T06:37:48.160 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:47 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 6611/226089 objects degraded (2.924%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:48.756 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:48 smithi107 ceph-mon[212750]: pgmap v1432: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4140/224838 objects degraded (1.841%); 8815/224838 objects misplaced (3.921%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:37:48.756 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:48 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:37:49.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:48 smithi107 ceph-mon[220921]: pgmap v1432: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4140/224838 objects degraded (1.841%); 8815/224838 objects misplaced (3.921%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:37:49.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:48 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:37:49.149 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:48 smithi150 ceph-mon[194178]: pgmap v1432: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 125 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4140/224838 objects degraded (1.841%); 8815/224838 objects misplaced (3.921%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T06:37:49.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:48 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:37:49.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:49.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:49.257 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:49.285 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:49.586 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1428,"stamp":"2023-10-09T06:37:43.477319+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4976076877,"num_objects":75946,"num_object_clones":0,"num_object_copies":227838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6611,"num_objects_misplaced":6820,"num_objects_unfound":0,"num_objects_dirty":75946,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":110660,"num_write_kb":5147960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3137,"num_bytes_recovered":202338162,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185586,"ondisk_log_size":185586,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22912760,"kb_used_data":14978232,"kb_used_omap":20,"kb_used_meta":7933931,"kb_avail":727048456,"statfs":{"total":767960285184,"available":744497618944,"internally_reserved":0,"allocated":15337709568,"data_stored":15325154199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8124345998},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-634322944,"num_objects":-9679,"num_object_clones":0,"num_object_copies":-29037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-854,"num_objects_unfound":0,"num_objects_dirty":-9679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007405"},"pg_stats":[{"pgid":"13.14","version":"469'7419","reported_seq":7610,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.554410+0000","last_change":"2023-10-09T06:37:02.769628+0000","last_active":"2023-10-09T06:37:42.554410+0000","last_peered":"2023-10-09T06:37:42.554410+0000","last_clean":"2023-10-09T06:37:42.554410+0000","last_became_active":"2023-10-09T06:36:57.240495+0000","last_became_peered":"2023-10-09T06:36:57.240495+0000","last_unstale":"2023-10-09T06:37:42.554410+0000","last_undegraded":"2023-10-09T06:37:42.554410+0000","last_fullsized":"2023-10-09T06:37:42.554410+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"457'6390","last_scrub_stamp":"2023-10-09T06:37:02.769552+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:02.769552+0000","objects_scrubbed":3298,"log_size":7419,"log_dups_size":0,"ondisk_log_size":7419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:36.138793+0000","scrub_duration":1.1657576549999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055872,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1043,"num_write_kb":5504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":732,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609981+0000","last_change":"2023-10-09T06:37:08.252438+0000","last_active":"2023-10-09T06:37:31.609981+0000","last_peered":"2023-10-09T06:37:31.609981+0000","last_clean":"2023-10-09T06:37:31.609981+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:37:31.609981+0000","last_undegraded":"2023-10-09T06:37:31.609981+0000","last_fullsized":"2023-10-09T06:37:31.609981+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:30.160439+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:30.160439+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012481058,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":131344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":448,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609270+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:37:31.609270+0000","last_peered":"2023-10-09T06:37:31.609270+0000","last_clean":"2023-10-09T06:37:31.609270+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:37:31.609270+0000","last_undegraded":"2023-10-09T06:37:31.609270+0000","last_fullsized":"2023-10-09T06:37:31.609270+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":169,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763799+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:37:31.763799+0000","last_peered":"2023-10-09T06:37:31.763799+0000","last_clean":"2023-10-09T06:37:31.763799+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:37:31.763799+0000","last_undegraded":"2023-10-09T06:37:31.763799+0000","last_fullsized":"2023-10-09T06:37:31.763799+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.17","version":"469'7266","reported_seq":7454,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.564354+0000","last_change":"2023-10-09T06:37:09.430507+0000","last_active":"2023-10-09T06:37:42.564354+0000","last_peered":"2023-10-09T06:37:42.564354+0000","last_clean":"2023-10-09T06:37:42.564354+0000","last_became_active":"2023-10-09T06:36:57.205319+0000","last_became_peered":"2023-10-09T06:36:57.205319+0000","last_unstale":"2023-10-09T06:37:42.564354+0000","last_undegraded":"2023-10-09T06:37:42.564354+0000","last_fullsized":"2023-10-09T06:37:42.564354+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"460'6376","last_scrub_stamp":"2023-10-09T06:37:09.430427+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:09.430427+0000","objects_scrubbed":3242,"log_size":7266,"log_dups_size":0,"ondisk_log_size":7266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:20.773784+0000","scrub_duration":1.0485898069999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153157632,"num_objects":2337,"num_object_clones":0,"num_object_copies":7011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":905,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":705,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.286007+0000","last_change":"2023-10-09T06:37:42.286007+0000","last_active":"2023-10-09T06:37:42.286007+0000","last_peered":"2023-10-09T06:37:42.286007+0000","last_clean":"2023-10-09T06:37:42.286007+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:37:42.286007+0000","last_undegraded":"2023-10-09T06:37:42.286007+0000","last_fullsized":"2023-10-09T06:37:42.286007+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:03.005341+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:03.005341+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.012596781,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":464,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765234+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:37:31.765234+0000","last_peered":"2023-10-09T06:37:31.765234+0000","last_clean":"2023-10-09T06:37:31.765234+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:37:31.765234+0000","last_undegraded":"2023-10-09T06:37:31.765234+0000","last_fullsized":"2023-10-09T06:37:31.765234+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":468,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.778980+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:37:31.778980+0000","last_peered":"2023-10-09T06:37:31.778980+0000","last_clean":"2023-10-09T06:37:31.778980+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:37:31.778980+0000","last_undegraded":"2023-10-09T06:37:31.778980+0000","last_fullsized":"2023-10-09T06:37:31.778980+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.16","version":"469'7315","reported_seq":7371,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.527281+0000","last_change":"2023-10-09T06:37:40.197269+0000","last_active":"2023-10-09T06:37:42.527281+0000","last_peered":"2023-10-09T06:37:42.527281+0000","last_clean":"2023-10-09T06:37:42.527281+0000","last_became_active":"2023-10-09T06:36:57.241947+0000","last_became_peered":"2023-10-09T06:36:57.241947+0000","last_unstale":"2023-10-09T06:37:42.527281+0000","last_undegraded":"2023-10-09T06:37:42.527281+0000","last_fullsized":"2023-10-09T06:37:42.527281+0000","mapping_epoch":454,"log_start":"441'300","ondisk_log_start":"441'300","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7015,"log_dups_size":0,"ondisk_log_size":7015,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151027712,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4263,"num_write_kb":210144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":716,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609216+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:37:31.609216+0000","last_peered":"2023-10-09T06:37:31.609216+0000","last_clean":"2023-10-09T06:37:31.609216+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:37:31.609216+0000","last_undegraded":"2023-10-09T06:37:31.609216+0000","last_fullsized":"2023-10-09T06:37:31.609216+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762634+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:37:31.762634+0000","last_peered":"2023-10-09T06:37:31.762634+0000","last_clean":"2023-10-09T06:37:31.762634+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:37:31.762634+0000","last_undegraded":"2023-10-09T06:37:31.762634+0000","last_fullsized":"2023-10-09T06:37:31.762634+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":458,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767902+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:37:31.767902+0000","last_peered":"2023-10-09T06:37:31.767902+0000","last_clean":"2023-10-09T06:37:31.767902+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:37:31.767902+0000","last_undegraded":"2023-10-09T06:37:31.767902+0000","last_fullsized":"2023-10-09T06:37:31.767902+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.19","version":"469'7300","reported_seq":7350,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.470604+0000","last_change":"2023-10-09T06:36:57.231121+0000","last_active":"2023-10-09T06:37:42.470604+0000","last_peered":"2023-10-09T06:37:42.470604+0000","last_clean":"2023-10-09T06:37:42.470604+0000","last_became_active":"2023-10-09T06:36:57.228244+0000","last_became_peered":"2023-10-09T06:36:57.228244+0000","last_unstale":"2023-10-09T06:37:42.470604+0000","last_undegraded":"2023-10-09T06:37:42.470604+0000","last_fullsized":"2023-10-09T06:37:42.470604+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7300,"log_dups_size":0,"ondisk_log_size":7300,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153681920,"num_objects":2345,"num_object_clones":0,"num_object_copies":7035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4229,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":577,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610538+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:37:31.610538+0000","last_peered":"2023-10-09T06:37:31.610538+0000","last_clean":"2023-10-09T06:37:31.610538+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:37:31.610538+0000","last_undegraded":"2023-10-09T06:37:31.610538+0000","last_fullsized":"2023-10-09T06:37:31.610538+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":143,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.781330+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:37:31.781330+0000","last_peered":"2023-10-09T06:37:31.781330+0000","last_clean":"2023-10-09T06:37:31.781330+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:37:31.781330+0000","last_undegraded":"2023-10-09T06:37:31.781330+0000","last_fullsized":"2023-10-09T06:37:31.781330+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762830+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:37:31.762830+0000","last_peered":"2023-10-09T06:37:31.762830+0000","last_clean":"2023-10-09T06:37:31.762830+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:37:31.762830+0000","last_undegraded":"2023-10-09T06:37:31.762830+0000","last_fullsized":"2023-10-09T06:37:31.762830+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.18","version":"469'7379","reported_seq":7440,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.570398+0000","last_change":"2023-10-09T06:37:38.861072+0000","last_active":"2023-10-09T06:37:42.570398+0000","last_peered":"2023-10-09T06:37:42.570398+0000","last_clean":"2023-10-09T06:37:42.570398+0000","last_became_active":"2023-10-09T06:37:08.253146+0000","last_became_peered":"2023-10-09T06:37:08.253146+0000","last_unstale":"2023-10-09T06:37:42.570398+0000","last_undegraded":"2023-10-09T06:37:42.570398+0000","last_fullsized":"2023-10-09T06:37:42.570398+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7379,"log_dups_size":0,"ondisk_log_size":7379,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4284,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":614,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610537+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:37:31.610537+0000","last_peered":"2023-10-09T06:37:31.610537+0000","last_clean":"2023-10-09T06:37:31.610537+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:37:31.610537+0000","last_undegraded":"2023-10-09T06:37:31.610537+0000","last_fullsized":"2023-10-09T06:37:31.610537+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":156,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.769079+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:37:31.769079+0000","last_peered":"2023-10-09T06:37:31.769079+0000","last_clean":"2023-10-09T06:37:31.769079+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:37:31.769079+0000","last_undegraded":"2023-10-09T06:37:31.769079+0000","last_fullsized":"2023-10-09T06:37:31.769079+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763177+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:37:31.763177+0000","last_peered":"2023-10-09T06:37:31.763177+0000","last_clean":"2023-10-09T06:37:31.763177+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:37:31.763177+0000","last_undegraded":"2023-10-09T06:37:31.763177+0000","last_fullsized":"2023-10-09T06:37:31.763177+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":584,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.779202+0000","last_change":"2023-10-09T06:37:11.071873+0000","last_active":"2023-10-09T06:37:31.779202+0000","last_peered":"2023-10-09T06:37:31.779202+0000","last_clean":"2023-10-09T06:37:31.779202+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:37:31.779202+0000","last_undegraded":"2023-10-09T06:37:31.779202+0000","last_fullsized":"2023-10-09T06:37:31.779202+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:31.210137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:31.210137+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":350,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610787+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:37:31.610787+0000","last_peered":"2023-10-09T06:37:31.610787+0000","last_clean":"2023-10-09T06:37:31.610787+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:37:31.610787+0000","last_undegraded":"2023-10-09T06:37:31.610787+0000","last_fullsized":"2023-10-09T06:37:31.610787+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":405,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.770095+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:37:31.770095+0000","last_peered":"2023-10-09T06:37:31.770095+0000","last_clean":"2023-10-09T06:37:31.770095+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:37:31.770095+0000","last_undegraded":"2023-10-09T06:37:31.770095+0000","last_fullsized":"2023-10-09T06:37:31.770095+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":577,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765638+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:37:31.765638+0000","last_peered":"2023-10-09T06:37:31.765638+0000","last_clean":"2023-10-09T06:37:31.765638+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:37:31.765638+0000","last_undegraded":"2023-10-09T06:37:31.765638+0000","last_fullsized":"2023-10-09T06:37:31.765638+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":475,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609716+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:37:31.609716+0000","last_peered":"2023-10-09T06:37:31.609716+0000","last_clean":"2023-10-09T06:37:31.609716+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:37:31.609716+0000","last_undegraded":"2023-10-09T06:37:31.609716+0000","last_fullsized":"2023-10-09T06:37:31.609716+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":439,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609593+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:37:31.609593+0000","last_peered":"2023-10-09T06:37:31.609593+0000","last_clean":"2023-10-09T06:37:31.609593+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:37:31.609593+0000","last_undegraded":"2023-10-09T06:37:31.609593+0000","last_fullsized":"2023-10-09T06:37:31.609593+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":741,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.779042+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:37:31.779042+0000","last_peered":"2023-10-09T06:37:31.779042+0000","last_clean":"2023-10-09T06:37:31.779042+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:37:31.779042+0000","last_undegraded":"2023-10-09T06:37:31.779042+0000","last_fullsized":"2023-10-09T06:37:31.779042+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":318,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610290+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:37:31.610290+0000","last_peered":"2023-10-09T06:37:31.610290+0000","last_clean":"2023-10-09T06:37:31.610290+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:37:31.610290+0000","last_undegraded":"2023-10-09T06:37:31.610290+0000","last_fullsized":"2023-10-09T06:37:31.610290+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763388+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:37:31.763388+0000","last_peered":"2023-10-09T06:37:31.763388+0000","last_clean":"2023-10-09T06:37:31.763388+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:37:31.763388+0000","last_undegraded":"2023-10-09T06:37:31.763388+0000","last_fullsized":"2023-10-09T06:37:31.763388+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":710,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610070+0000","last_change":"2023-10-09T06:37:10.361228+0000","last_active":"2023-10-09T06:37:31.610070+0000","last_peered":"2023-10-09T06:37:31.610070+0000","last_clean":"2023-10-09T06:37:31.610070+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:37:31.610070+0000","last_undegraded":"2023-10-09T06:37:31.610070+0000","last_fullsized":"2023-10-09T06:37:31.610070+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.207018+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.207018+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0045645399999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":466,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.791510+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:37:31.791510+0000","last_peered":"2023-10-09T06:37:31.791510+0000","last_clean":"2023-10-09T06:37:31.791510+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:37:31.791510+0000","last_undegraded":"2023-10-09T06:37:31.791510+0000","last_fullsized":"2023-10-09T06:37:31.791510+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":289,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610722+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:37:31.610722+0000","last_peered":"2023-10-09T06:37:31.610722+0000","last_clean":"2023-10-09T06:37:31.610722+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:37:31.610722+0000","last_undegraded":"2023-10-09T06:37:31.610722+0000","last_fullsized":"2023-10-09T06:37:31.610722+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":795,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765259+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:37:31.765259+0000","last_peered":"2023-10-09T06:37:31.765259+0000","last_clean":"2023-10-09T06:37:31.765259+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:37:31.765259+0000","last_undegraded":"2023-10-09T06:37:31.765259+0000","last_fullsized":"2023-10-09T06:37:31.765259+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":312,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610122+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:37:31.610122+0000","last_peered":"2023-10-09T06:37:31.610122+0000","last_clean":"2023-10-09T06:37:31.610122+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:37:31.610122+0000","last_undegraded":"2023-10-09T06:37:31.610122+0000","last_fullsized":"2023-10-09T06:37:31.610122+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":143,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.778729+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:37:31.778729+0000","last_peered":"2023-10-09T06:37:31.778729+0000","last_clean":"2023-10-09T06:37:31.778729+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:37:31.778729+0000","last_undegraded":"2023-10-09T06:37:31.778729+0000","last_fullsized":"2023-10-09T06:37:31.778729+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":583,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.769716+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:37:31.769716+0000","last_peered":"2023-10-09T06:37:31.769716+0000","last_clean":"2023-10-09T06:37:31.769716+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:37:31.769716+0000","last_undegraded":"2023-10-09T06:37:31.769716+0000","last_fullsized":"2023-10-09T06:37:31.769716+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":547,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766270+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:37:31.766270+0000","last_peered":"2023-10-09T06:37:31.766270+0000","last_clean":"2023-10-09T06:37:31.766270+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:37:31.766270+0000","last_undegraded":"2023-10-09T06:37:31.766270+0000","last_fullsized":"2023-10-09T06:37:31.766270+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.764074+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:37:31.764074+0000","last_peered":"2023-10-09T06:37:31.764074+0000","last_clean":"2023-10-09T06:37:31.764074+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:37:31.764074+0000","last_undegraded":"2023-10-09T06:37:31.764074+0000","last_fullsized":"2023-10-09T06:37:31.764074+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"469'7335","reported_seq":7390,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.590654+0000","last_change":"2023-10-09T06:37:36.878072+0000","last_active":"2023-10-09T06:37:42.590654+0000","last_peered":"2023-10-09T06:37:42.590654+0000","last_clean":"2023-10-09T06:37:42.590654+0000","last_became_active":"2023-10-09T06:36:57.185490+0000","last_became_peered":"2023-10-09T06:36:57.185490+0000","last_unstale":"2023-10-09T06:37:42.590654+0000","last_undegraded":"2023-10-09T06:37:42.590654+0000","last_fullsized":"2023-10-09T06:37:42.590654+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154632192,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":211872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":596,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610540+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:37:31.610540+0000","last_peered":"2023-10-09T06:37:31.610540+0000","last_clean":"2023-10-09T06:37:31.610540+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:37:31.610540+0000","last_undegraded":"2023-10-09T06:37:31.610540+0000","last_fullsized":"2023-10-09T06:37:31.610540+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":354,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610862+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:37:31.610862+0000","last_peered":"2023-10-09T06:37:31.610862+0000","last_clean":"2023-10-09T06:37:31.610862+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:37:31.610862+0000","last_undegraded":"2023-10-09T06:37:31.610862+0000","last_fullsized":"2023-10-09T06:37:31.610862+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610418+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:37:31.610418+0000","last_peered":"2023-10-09T06:37:31.610418+0000","last_clean":"2023-10-09T06:37:31.610418+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:37:31.610418+0000","last_undegraded":"2023-10-09T06:37:31.610418+0000","last_fullsized":"2023-10-09T06:37:31.610418+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.0","version":"469'7366","reported_seq":7552,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.556314+0000","last_change":"2023-10-09T06:36:59.980627+0000","last_active":"2023-10-09T06:37:42.556314+0000","last_peered":"2023-10-09T06:37:42.556314+0000","last_clean":"2023-10-09T06:37:42.556314+0000","last_became_active":"2023-10-09T06:36:57.178903+0000","last_became_peered":"2023-10-09T06:36:57.178903+0000","last_unstale":"2023-10-09T06:37:42.556314+0000","last_undegraded":"2023-10-09T06:37:42.556314+0000","last_fullsized":"2023-10-09T06:37:42.556314+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6277","last_scrub_stamp":"2023-10-09T06:36:59.980553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.980553+0000","objects_scrubbed":3134,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:19:50.243776+0000","scrub_duration":0.77626023,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154206208,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1099,"num_write_kb":10176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":854,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763176+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:37:31.763176+0000","last_peered":"2023-10-09T06:37:31.763176+0000","last_clean":"2023-10-09T06:37:31.763176+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:37:31.763176+0000","last_undegraded":"2023-10-09T06:37:31.763176+0000","last_fullsized":"2023-10-09T06:37:31.763176+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":316,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610755+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:37:31.610755+0000","last_peered":"2023-10-09T06:37:31.610755+0000","last_clean":"2023-10-09T06:37:31.610755+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:37:31.610755+0000","last_undegraded":"2023-10-09T06:37:31.610755+0000","last_fullsized":"2023-10-09T06:37:31.610755+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":466,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609287+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:37:31.609287+0000","last_peered":"2023-10-09T06:37:31.609287+0000","last_clean":"2023-10-09T06:37:31.609287+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:37:31.609287+0000","last_undegraded":"2023-10-09T06:37:31.609287+0000","last_fullsized":"2023-10-09T06:37:31.609287+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"469'7395","reported_seq":7520,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:40.982140+0000","last_change":"2023-10-09T06:36:58.845020+0000","last_active":"2023-10-09T06:37:40.982140+0000","last_peered":"2023-10-09T06:37:40.982140+0000","last_clean":"2023-10-09T06:37:40.982140+0000","last_became_active":"2023-10-09T06:36:57.185094+0000","last_became_peered":"2023-10-09T06:36:57.185094+0000","last_unstale":"2023-10-09T06:37:40.982140+0000","last_undegraded":"2023-10-09T06:37:40.982140+0000","last_fullsized":"2023-10-09T06:37:40.982140+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6294","last_scrub_stamp":"2023-10-09T06:36:58.844752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:58.844752+0000","objects_scrubbed":3160,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:07.044778+0000","scrub_duration":0.83882531500000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158203904,"num_objects":2414,"num_object_clones":0,"num_object_copies":7242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1114,"num_write_kb":11776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":770,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767832+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:37:31.767832+0000","last_peered":"2023-10-09T06:37:31.767832+0000","last_clean":"2023-10-09T06:37:31.767832+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:37:31.767832+0000","last_undegraded":"2023-10-09T06:37:31.767832+0000","last_fullsized":"2023-10-09T06:37:31.767832+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":372,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610646+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:37:31.610646+0000","last_peered":"2023-10-09T06:37:31.610646+0000","last_clean":"2023-10-09T06:37:31.610646+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:37:31.610646+0000","last_undegraded":"2023-10-09T06:37:31.610646+0000","last_fullsized":"2023-10-09T06:37:31.610646+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":445,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766634+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:37:31.766634+0000","last_peered":"2023-10-09T06:37:31.766634+0000","last_clean":"2023-10-09T06:37:31.766634+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:37:31.766634+0000","last_undegraded":"2023-10-09T06:37:31.766634+0000","last_fullsized":"2023-10-09T06:37:31.766634+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"469'7209","reported_seq":7267,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:38.393088+0000","last_change":"2023-10-09T06:37:11.285046+0000","last_active":"2023-10-09T06:37:38.393088+0000","last_peered":"2023-10-09T06:37:38.393088+0000","last_clean":"2023-10-09T06:37:38.393088+0000","last_became_active":"2023-10-09T06:37:08.349856+0000","last_became_peered":"2023-10-09T06:37:08.349856+0000","last_unstale":"2023-10-09T06:37:38.393088+0000","last_undegraded":"2023-10-09T06:37:38.393088+0000","last_fullsized":"2023-10-09T06:37:38.393088+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164397056,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4116,"num_write_kb":212000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":593,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609711+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:37:31.609711+0000","last_peered":"2023-10-09T06:37:31.609711+0000","last_clean":"2023-10-09T06:37:31.609711+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:37:31.609711+0000","last_undegraded":"2023-10-09T06:37:31.609711+0000","last_fullsized":"2023-10-09T06:37:31.609711+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":38,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763460+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:37:31.763460+0000","last_peered":"2023-10-09T06:37:31.763460+0000","last_clean":"2023-10-09T06:37:31.763460+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:37:31.763460+0000","last_undegraded":"2023-10-09T06:37:31.763460+0000","last_fullsized":"2023-10-09T06:37:31.763460+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":431,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.778864+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:37:31.778864+0000","last_peered":"2023-10-09T06:37:31.778864+0000","last_clean":"2023-10-09T06:37:31.778864+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:37:31.778864+0000","last_undegraded":"2023-10-09T06:37:31.778864+0000","last_fullsized":"2023-10-09T06:37:31.778864+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"469'7468","reported_seq":7659,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.527180+0000","last_change":"2023-10-09T06:37:01.498281+0000","last_active":"2023-10-09T06:37:42.527180+0000","last_peered":"2023-10-09T06:37:42.527180+0000","last_clean":"2023-10-09T06:37:42.527180+0000","last_became_active":"2023-10-09T06:36:57.211509+0000","last_became_peered":"2023-10-09T06:36:57.211509+0000","last_unstale":"2023-10-09T06:37:42.527180+0000","last_undegraded":"2023-10-09T06:37:42.527180+0000","last_fullsized":"2023-10-09T06:37:42.527180+0000","mapping_epoch":437,"log_start":"441'400","ondisk_log_start":"441'400","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6333","last_scrub_stamp":"2023-10-09T06:37:01.498222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:01.498222+0000","objects_scrubbed":3239,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:56.557159+0000","scrub_duration":1.395430696,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1164,"num_write_kb":9088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":821,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609980+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:37:31.609980+0000","last_peered":"2023-10-09T06:37:31.609980+0000","last_clean":"2023-10-09T06:37:31.609980+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:37:31.609980+0000","last_undegraded":"2023-10-09T06:37:31.609980+0000","last_fullsized":"2023-10-09T06:37:31.609980+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":196,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.764537+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:37:31.764537+0000","last_peered":"2023-10-09T06:37:31.764537+0000","last_clean":"2023-10-09T06:37:31.764537+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:37:31.764537+0000","last_undegraded":"2023-10-09T06:37:31.764537+0000","last_fullsized":"2023-10-09T06:37:31.764537+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":449,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609481+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:37:31.609481+0000","last_peered":"2023-10-09T06:37:31.609481+0000","last_clean":"2023-10-09T06:37:31.609481+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:37:31.609481+0000","last_undegraded":"2023-10-09T06:37:31.609481+0000","last_fullsized":"2023-10-09T06:37:31.609481+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"469'7342","reported_seq":7528,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.496947+0000","last_change":"2023-10-09T06:36:59.004823+0000","last_active":"2023-10-09T06:37:42.496947+0000","last_peered":"2023-10-09T06:37:42.496947+0000","last_clean":"2023-10-09T06:37:42.496947+0000","last_became_active":"2023-10-09T06:36:57.221064+0000","last_became_peered":"2023-10-09T06:36:57.221064+0000","last_unstale":"2023-10-09T06:37:42.496947+0000","last_undegraded":"2023-10-09T06:37:42.496947+0000","last_fullsized":"2023-10-09T06:37:42.496947+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"455'6279","last_scrub_stamp":"2023-10-09T06:36:59.004706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.004706+0000","objects_scrubbed":3113,"log_size":7342,"log_dups_size":0,"ondisk_log_size":7342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:30.643419+0000","scrub_duration":1.3564550360000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156303360,"num_objects":2385,"num_object_clones":0,"num_object_copies":7155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1090,"num_write_kb":11584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":490,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762857+0000","last_change":"2023-10-09T06:37:08.345883+0000","last_active":"2023-10-09T06:37:31.762857+0000","last_peered":"2023-10-09T06:37:31.762857+0000","last_clean":"2023-10-09T06:37:31.762857+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:37:31.762857+0000","last_undegraded":"2023-10-09T06:37:31.762857+0000","last_fullsized":"2023-10-09T06:37:31.762857+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.271362+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.271362+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":426,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766510+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:37:31.766510+0000","last_peered":"2023-10-09T06:37:31.766510+0000","last_clean":"2023-10-09T06:37:31.766510+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:37:31.766510+0000","last_undegraded":"2023-10-09T06:37:31.766510+0000","last_fullsized":"2023-10-09T06:37:31.766510+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":446,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609892+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:37:31.609892+0000","last_peered":"2023-10-09T06:37:31.609892+0000","last_clean":"2023-10-09T06:37:31.609892+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:37:31.609892+0000","last_undegraded":"2023-10-09T06:37:31.609892+0000","last_fullsized":"2023-10-09T06:37:31.609892+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"469'7250","reported_seq":7325,"reported_epoch":469,"state":"active+clean+scrubbing","last_fresh":"2023-10-09T06:37:42.597097+0000","last_change":"2023-10-09T06:37:42.406399+0000","last_active":"2023-10-09T06:37:42.597097+0000","last_peered":"2023-10-09T06:37:42.597097+0000","last_clean":"2023-10-09T06:37:42.597097+0000","last_became_active":"2023-10-09T06:36:57.185079+0000","last_became_peered":"2023-10-09T06:36:57.185079+0000","last_unstale":"2023-10-09T06:37:42.597097+0000","last_undegraded":"2023-10-09T06:37:42.597097+0000","last_fullsized":"2023-10-09T06:37:42.597097+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":449,"log_size":7250,"log_dups_size":0,"ondisk_log_size":7250,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"scrubbing for 0s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140672,"num_objects":2352,"num_object_clones":0,"num_object_copies":7056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4206,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":804,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765492+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:37:31.765492+0000","last_peered":"2023-10-09T06:37:31.765492+0000","last_clean":"2023-10-09T06:37:31.765492+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:37:31.765492+0000","last_undegraded":"2023-10-09T06:37:31.765492+0000","last_fullsized":"2023-10-09T06:37:31.765492+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":450,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609125+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:37:31.609125+0000","last_peered":"2023-10-09T06:37:31.609125+0000","last_clean":"2023-10-09T06:37:31.609125+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:37:31.609125+0000","last_undegraded":"2023-10-09T06:37:31.609125+0000","last_fullsized":"2023-10-09T06:37:31.609125+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":429,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609988+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:37:31.609988+0000","last_peered":"2023-10-09T06:37:31.609988+0000","last_clean":"2023-10-09T06:37:31.609988+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:37:31.609988+0000","last_undegraded":"2023-10-09T06:37:31.609988+0000","last_fullsized":"2023-10-09T06:37:31.609988+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"469'7281","reported_seq":7337,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.543247+0000","last_change":"2023-10-09T06:37:39.236815+0000","last_active":"2023-10-09T06:37:42.543247+0000","last_peered":"2023-10-09T06:37:42.543247+0000","last_clean":"2023-10-09T06:37:42.543247+0000","last_became_active":"2023-10-09T06:36:57.222809+0000","last_became_peered":"2023-10-09T06:36:57.222809+0000","last_unstale":"2023-10-09T06:37:42.543247+0000","last_undegraded":"2023-10-09T06:37:42.543247+0000","last_fullsized":"2023-10-09T06:37:42.543247+0000","mapping_epoch":437,"log_start":"441'300","ondisk_log_start":"441'300","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153518080,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4228,"num_write_kb":210272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":421,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766428+0000","last_change":"2023-10-09T06:37:08.251809+0000","last_active":"2023-10-09T06:37:31.766428+0000","last_peered":"2023-10-09T06:37:31.766428+0000","last_clean":"2023-10-09T06:37:31.766428+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:37:31.766428+0000","last_undegraded":"2023-10-09T06:37:31.766428+0000","last_fullsized":"2023-10-09T06:37:31.766428+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.246218+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.246218+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762785+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:37:31.762785+0000","last_peered":"2023-10-09T06:37:31.762785+0000","last_clean":"2023-10-09T06:37:31.762785+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:37:31.762785+0000","last_undegraded":"2023-10-09T06:37:31.762785+0000","last_fullsized":"2023-10-09T06:37:31.762785+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":423,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765980+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:37:31.765980+0000","last_peered":"2023-10-09T06:37:31.765980+0000","last_clean":"2023-10-09T06:37:31.765980+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:37:31.765980+0000","last_undegraded":"2023-10-09T06:37:31.765980+0000","last_fullsized":"2023-10-09T06:37:31.765980+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"469'8254","reported_seq":8298,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:38.478103+0000","last_change":"2023-10-09T06:37:08.359543+0000","last_active":"2023-10-09T06:37:38.478103+0000","last_peered":"2023-10-09T06:37:38.478103+0000","last_clean":"2023-10-09T06:37:38.478103+0000","last_became_active":"2023-10-09T06:37:08.343368+0000","last_became_peered":"2023-10-09T06:37:08.343368+0000","last_unstale":"2023-10-09T06:37:38.478103+0000","last_undegraded":"2023-10-09T06:37:38.478103+0000","last_fullsized":"2023-10-09T06:37:38.478103+0000","mapping_epoch":459,"log_start":"443'2308","ondisk_log_start":"443'2308","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:20.011628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322830336,"num_objects":4926,"num_object_clones":0,"num_object_copies":14778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8254,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":761,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767952+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:37:31.767952+0000","last_peered":"2023-10-09T06:37:31.767952+0000","last_clean":"2023-10-09T06:37:31.767952+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:37:31.767952+0000","last_undegraded":"2023-10-09T06:37:31.767952+0000","last_fullsized":"2023-10-09T06:37:31.767952+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":475,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765896+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:37:31.765896+0000","last_peered":"2023-10-09T06:37:31.765896+0000","last_clean":"2023-10-09T06:37:31.765896+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:37:31.765896+0000","last_undegraded":"2023-10-09T06:37:31.765896+0000","last_fullsized":"2023-10-09T06:37:31.765896+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":38,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763894+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:37:31.763894+0000","last_peered":"2023-10-09T06:37:31.763894+0000","last_clean":"2023-10-09T06:37:31.763894+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:37:31.763894+0000","last_undegraded":"2023-10-09T06:37:31.763894+0000","last_fullsized":"2023-10-09T06:37:31.763894+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":409,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767417+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:37:31.767417+0000","last_peered":"2023-10-09T06:37:31.767417+0000","last_clean":"2023-10-09T06:37:31.767417+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:37:31.767417+0000","last_undegraded":"2023-10-09T06:37:31.767417+0000","last_fullsized":"2023-10-09T06:37:31.767417+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"469'8550","reported_seq":8591,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.534643+0000","last_change":"2023-10-09T06:35:28.539445+0000","last_active":"2023-10-09T06:37:42.534643+0000","last_peered":"2023-10-09T06:37:42.534643+0000","last_clean":"2023-10-09T06:37:42.534643+0000","last_became_active":"2023-10-09T06:35:28.539141+0000","last_became_peered":"2023-10-09T06:35:28.539141+0000","last_unstale":"2023-10-09T06:37:42.534643+0000","last_undegraded":"2023-10-09T06:37:42.534643+0000","last_fullsized":"2023-10-09T06:37:42.534643+0000","mapping_epoch":437,"log_start":"445'3367","ondisk_log_start":"445'3367","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5183,"log_dups_size":0,"ondisk_log_size":5183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:20:26.753486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314638360,"num_objects":4802,"num_object_clones":0,"num_object_copies":14406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4802,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8550,"num_write_kb":427201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":635,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.779393+0000","last_change":"2023-10-09T06:37:08.296784+0000","last_active":"2023-10-09T06:37:31.779393+0000","last_peered":"2023-10-09T06:37:31.779393+0000","last_clean":"2023-10-09T06:37:31.779393+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:37:31.779393+0000","last_undegraded":"2023-10-09T06:37:31.779393+0000","last_fullsized":"2023-10-09T06:37:31.779393+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":199,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.769982+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:37:31.769982+0000","last_peered":"2023-10-09T06:37:31.769982+0000","last_clean":"2023-10-09T06:37:31.769982+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:37:31.769982+0000","last_undegraded":"2023-10-09T06:37:31.769982+0000","last_fullsized":"2023-10-09T06:37:31.769982+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":450,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609544+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:37:31.609544+0000","last_peered":"2023-10-09T06:37:31.609544+0000","last_clean":"2023-10-09T06:37:31.609544+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:37:31.609544+0000","last_undegraded":"2023-10-09T06:37:31.609544+0000","last_fullsized":"2023-10-09T06:37:31.609544+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":439,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765853+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:37:31.765853+0000","last_peered":"2023-10-09T06:37:31.765853+0000","last_clean":"2023-10-09T06:37:31.765853+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:37:31.765853+0000","last_undegraded":"2023-10-09T06:37:31.765853+0000","last_fullsized":"2023-10-09T06:37:31.765853+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"469'8618","reported_seq":8658,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.596657+0000","last_change":"2023-10-09T06:35:28.544012+0000","last_active":"2023-10-09T06:37:42.596657+0000","last_peered":"2023-10-09T06:37:42.596657+0000","last_clean":"2023-10-09T06:37:42.596657+0000","last_became_active":"2023-10-09T06:35:28.543684+0000","last_became_peered":"2023-10-09T06:35:28.543684+0000","last_unstale":"2023-10-09T06:37:42.596657+0000","last_undegraded":"2023-10-09T06:37:42.596657+0000","last_fullsized":"2023-10-09T06:37:42.596657+0000","mapping_epoch":437,"log_start":"443'2612","ondisk_log_start":"443'2612","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:47.327388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8618,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":811,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610107+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:37:31.610107+0000","last_peered":"2023-10-09T06:37:31.610107+0000","last_clean":"2023-10-09T06:37:31.610107+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:37:31.610107+0000","last_undegraded":"2023-10-09T06:37:31.610107+0000","last_fullsized":"2023-10-09T06:37:31.610107+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":461,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762748+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:37:31.762748+0000","last_peered":"2023-10-09T06:37:31.762748+0000","last_clean":"2023-10-09T06:37:31.762748+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:37:31.762748+0000","last_undegraded":"2023-10-09T06:37:31.762748+0000","last_fullsized":"2023-10-09T06:37:31.762748+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":448,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767835+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:37:31.767835+0000","last_peered":"2023-10-09T06:37:31.767835+0000","last_clean":"2023-10-09T06:37:31.767835+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:37:31.767835+0000","last_undegraded":"2023-10-09T06:37:31.767835+0000","last_fullsized":"2023-10-09T06:37:31.767835+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":470,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766060+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:37:31.766060+0000","last_peered":"2023-10-09T06:37:31.766060+0000","last_clean":"2023-10-09T06:37:31.766060+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:37:31.766060+0000","last_undegraded":"2023-10-09T06:37:31.766060+0000","last_fullsized":"2023-10-09T06:37:31.766060+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"469'8542","reported_seq":8584,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:43.002675+0000","last_change":"2023-10-09T06:35:28.750537+0000","last_active":"2023-10-09T06:37:43.002675+0000","last_peered":"2023-10-09T06:37:43.002675+0000","last_clean":"2023-10-09T06:37:43.002675+0000","last_became_active":"2023-10-09T06:35:28.750267+0000","last_became_peered":"2023-10-09T06:35:28.750267+0000","last_unstale":"2023-10-09T06:37:43.002675+0000","last_undegraded":"2023-10-09T06:37:43.002675+0000","last_fullsized":"2023-10-09T06:37:43.002675+0000","mapping_epoch":437,"log_start":"443'2575","ondisk_log_start":"443'2575","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:47.173545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305790976,"num_objects":4666,"num_object_clones":0,"num_object_copies":13998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8542,"num_write_kb":422656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":278,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610242+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:37:31.610242+0000","last_peered":"2023-10-09T06:37:31.610242+0000","last_clean":"2023-10-09T06:37:31.610242+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:37:31.610242+0000","last_undegraded":"2023-10-09T06:37:31.610242+0000","last_fullsized":"2023-10-09T06:37:31.610242+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":792,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610469+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:37:31.610469+0000","last_peered":"2023-10-09T06:37:31.610469+0000","last_clean":"2023-10-09T06:37:31.610469+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:37:31.610469+0000","last_undegraded":"2023-10-09T06:37:31.610469+0000","last_fullsized":"2023-10-09T06:37:31.610469+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":179,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767681+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:37:31.767681+0000","last_peered":"2023-10-09T06:37:31.767681+0000","last_clean":"2023-10-09T06:37:31.767681+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:37:31.767681+0000","last_undegraded":"2023-10-09T06:37:31.767681+0000","last_fullsized":"2023-10-09T06:37:31.767681+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":431,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767055+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:37:31.767055+0000","last_peered":"2023-10-09T06:37:31.767055+0000","last_clean":"2023-10-09T06:37:31.767055+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:37:31.767055+0000","last_undegraded":"2023-10-09T06:37:31.767055+0000","last_fullsized":"2023-10-09T06:37:31.767055+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"469'8706","reported_seq":8746,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.592948+0000","last_change":"2023-10-09T06:35:28.750108+0000","last_active":"2023-10-09T06:37:42.592948+0000","last_peered":"2023-10-09T06:37:42.592948+0000","last_clean":"2023-10-09T06:37:42.592948+0000","last_became_active":"2023-10-09T06:35:28.749808+0000","last_became_peered":"2023-10-09T06:35:28.749808+0000","last_unstale":"2023-10-09T06:37:42.592948+0000","last_undegraded":"2023-10-09T06:37:42.592948+0000","last_fullsized":"2023-10-09T06:37:42.592948+0000","mapping_epoch":437,"log_start":"443'2500","ondisk_log_start":"443'2500","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6206,"log_dups_size":0,"ondisk_log_size":6206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:44.197980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":778,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765488+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:37:31.765488+0000","last_peered":"2023-10-09T06:37:31.765488+0000","last_clean":"2023-10-09T06:37:31.765488+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:37:31.765488+0000","last_undegraded":"2023-10-09T06:37:31.765488+0000","last_fullsized":"2023-10-09T06:37:31.765488+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":332,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610209+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:37:31.610209+0000","last_peered":"2023-10-09T06:37:31.610209+0000","last_clean":"2023-10-09T06:37:31.610209+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:37:31.610209+0000","last_undegraded":"2023-10-09T06:37:31.610209+0000","last_fullsized":"2023-10-09T06:37:31.610209+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":144,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:36.081529+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:37:36.081529+0000","last_peered":"2023-10-09T06:37:36.081529+0000","last_clean":"2023-10-09T06:37:36.081529+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:37:36.081529+0000","last_undegraded":"2023-10-09T06:37:36.081529+0000","last_fullsized":"2023-10-09T06:37:36.081529+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":454,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.779054+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:37:31.779054+0000","last_peered":"2023-10-09T06:37:31.779054+0000","last_clean":"2023-10-09T06:37:31.779054+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:37:31.779054+0000","last_undegraded":"2023-10-09T06:37:31.779054+0000","last_fullsized":"2023-10-09T06:37:31.779054+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"469'8583","reported_seq":8624,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:43.020172+0000","last_change":"2023-10-09T06:35:28.748964+0000","last_active":"2023-10-09T06:37:43.020172+0000","last_peered":"2023-10-09T06:37:43.020172+0000","last_clean":"2023-10-09T06:37:43.020172+0000","last_became_active":"2023-10-09T06:35:28.748674+0000","last_became_peered":"2023-10-09T06:35:28.748674+0000","last_unstale":"2023-10-09T06:37:43.020172+0000","last_undegraded":"2023-10-09T06:37:43.020172+0000","last_fullsized":"2023-10-09T06:37:43.020172+0000","mapping_epoch":437,"log_start":"443'2681","ondisk_log_start":"443'2681","created":437,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:02.079067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302186496,"num_objects":4611,"num_object_clones":0,"num_object_copies":13833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8583,"num_write_kb":422208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":791,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.767499+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:37:31.767499+0000","last_peered":"2023-10-09T06:37:31.767499+0000","last_clean":"2023-10-09T06:37:31.767499+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:37:31.767499+0000","last_undegraded":"2023-10-09T06:37:31.767499+0000","last_fullsized":"2023-10-09T06:37:31.767499+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610799+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:37:31.610799+0000","last_peered":"2023-10-09T06:37:31.610799+0000","last_clean":"2023-10-09T06:37:31.610799+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:37:31.610799+0000","last_undegraded":"2023-10-09T06:37:31.610799+0000","last_fullsized":"2023-10-09T06:37:31.610799+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":465,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765828+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:37:31.765828+0000","last_peered":"2023-10-09T06:37:31.765828+0000","last_clean":"2023-10-09T06:37:31.765828+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:37:31.765828+0000","last_undegraded":"2023-10-09T06:37:31.765828+0000","last_fullsized":"2023-10-09T06:37:31.765828+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":453,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:36.081704+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:37:36.081704+0000","last_peered":"2023-10-09T06:37:36.081704+0000","last_clean":"2023-10-09T06:37:36.081704+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:37:36.081704+0000","last_undegraded":"2023-10-09T06:37:36.081704+0000","last_fullsized":"2023-10-09T06:37:36.081704+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"469'7330","reported_seq":7384,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.477304+0000","last_change":"2023-10-09T06:36:59.148541+0000","last_active":"2023-10-09T06:37:42.477304+0000","last_peered":"2023-10-09T06:37:42.477304+0000","last_clean":"2023-10-09T06:37:42.477304+0000","last_became_active":"2023-10-09T06:36:57.214586+0000","last_became_peered":"2023-10-09T06:36:57.214586+0000","last_unstale":"2023-10-09T06:37:42.477304+0000","last_undegraded":"2023-10-09T06:37:42.477304+0000","last_fullsized":"2023-10-09T06:37:42.477304+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4259,"num_write_kb":210432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":792,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609747+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:37:31.609747+0000","last_peered":"2023-10-09T06:37:31.609747+0000","last_clean":"2023-10-09T06:37:31.609747+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:37:31.609747+0000","last_undegraded":"2023-10-09T06:37:31.609747+0000","last_fullsized":"2023-10-09T06:37:31.609747+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":200,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.764214+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:37:31.764214+0000","last_peered":"2023-10-09T06:37:31.764214+0000","last_clean":"2023-10-09T06:37:31.764214+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:37:31.764214+0000","last_undegraded":"2023-10-09T06:37:31.764214+0000","last_fullsized":"2023-10-09T06:37:31.764214+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":433,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765824+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:37:31.765824+0000","last_peered":"2023-10-09T06:37:31.765824+0000","last_clean":"2023-10-09T06:37:31.765824+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:37:31.765824+0000","last_undegraded":"2023-10-09T06:37:31.765824+0000","last_fullsized":"2023-10-09T06:37:31.765824+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":188,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762709+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:37:31.762709+0000","last_peered":"2023-10-09T06:37:31.762709+0000","last_clean":"2023-10-09T06:37:31.762709+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:37:31.762709+0000","last_undegraded":"2023-10-09T06:37:31.762709+0000","last_fullsized":"2023-10-09T06:37:31.762709+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"469'7397","reported_seq":7597,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.480897+0000","last_change":"2023-10-09T06:37:08.334364+0000","last_active":"2023-10-09T06:37:42.480897+0000","last_peered":"2023-10-09T06:37:42.480897+0000","last_clean":"2023-10-09T06:37:42.480897+0000","last_became_active":"2023-10-09T06:37:08.252813+0000","last_became_peered":"2023-10-09T06:37:08.252813+0000","last_unstale":"2023-10-09T06:37:42.480897+0000","last_undegraded":"2023-10-09T06:37:42.480897+0000","last_fullsized":"2023-10-09T06:37:42.480897+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":437,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"457'6359","last_scrub_stamp":"2023-10-09T06:37:03.544833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:37:03.544833+0000","objects_scrubbed":3271,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:43.295290+0000","scrub_duration":0.43754141200000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1041,"num_write_kb":4096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":822,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763535+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:37:31.763535+0000","last_peered":"2023-10-09T06:37:31.763535+0000","last_clean":"2023-10-09T06:37:31.763535+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:37:31.763535+0000","last_undegraded":"2023-10-09T06:37:31.763535+0000","last_fullsized":"2023-10-09T06:37:31.763535+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":453,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.768019+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:37:31.768019+0000","last_peered":"2023-10-09T06:37:31.768019+0000","last_clean":"2023-10-09T06:37:31.768019+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:37:31.768019+0000","last_undegraded":"2023-10-09T06:37:31.768019+0000","last_fullsized":"2023-10-09T06:37:31.768019+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":435,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.778897+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:37:31.778897+0000","last_peered":"2023-10-09T06:37:31.778897+0000","last_clean":"2023-10-09T06:37:31.778897+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:37:31.778897+0000","last_undegraded":"2023-10-09T06:37:31.778897+0000","last_fullsized":"2023-10-09T06:37:31.778897+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":432,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609224+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:37:31.609224+0000","last_peered":"2023-10-09T06:37:31.609224+0000","last_clean":"2023-10-09T06:37:31.609224+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:37:31.609224+0000","last_undegraded":"2023-10-09T06:37:31.609224+0000","last_fullsized":"2023-10-09T06:37:31.609224+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.15","version":"469'7363","reported_seq":7416,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:42.480413+0000","last_change":"2023-10-09T06:37:41.247031+0000","last_active":"2023-10-09T06:37:42.480413+0000","last_peered":"2023-10-09T06:37:42.480413+0000","last_clean":"2023-10-09T06:37:42.480413+0000","last_became_active":"2023-10-09T06:36:57.229253+0000","last_became_peered":"2023-10-09T06:36:57.229253+0000","last_unstale":"2023-10-09T06:37:42.480413+0000","last_undegraded":"2023-10-09T06:37:42.480413+0000","last_fullsized":"2023-10-09T06:37:42.480413+0000","mapping_epoch":454,"log_start":"441'400","ondisk_log_start":"441'400","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":6963,"log_dups_size":0,"ondisk_log_size":6963,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157417472,"num_objects":2402,"num_object_clones":0,"num_object_copies":7206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4266,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":749,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.762916+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:37:31.762916+0000","last_peered":"2023-10-09T06:37:31.762916+0000","last_clean":"2023-10-09T06:37:31.762916+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:37:31.762916+0000","last_undegraded":"2023-10-09T06:37:31.762916+0000","last_fullsized":"2023-10-09T06:37:31.762916+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":436,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.779018+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:37:31.779018+0000","last_peered":"2023-10-09T06:37:31.779018+0000","last_clean":"2023-10-09T06:37:31.779018+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:37:31.779018+0000","last_undegraded":"2023-10-09T06:37:31.779018+0000","last_fullsized":"2023-10-09T06:37:31.779018+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":458,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.765782+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:37:31.765782+0000","last_peered":"2023-10-09T06:37:31.765782+0000","last_clean":"2023-10-09T06:37:31.765782+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:37:31.765782+0000","last_undegraded":"2023-10-09T06:37:31.765782+0000","last_fullsized":"2023-10-09T06:37:31.765782+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.12","version":"469'7205","reported_seq":7269,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:37:38.449881+0000","last_change":"2023-10-09T06:37:18.507362+0000","last_active":"2023-10-09T06:37:38.449881+0000","last_peered":"2023-10-09T06:37:38.449881+0000","last_clean":"2023-10-09T06:37:15.283407+0000","last_became_active":"2023-10-09T06:37:18.468289+0000","last_became_peered":"2023-10-09T06:37:18.468289+0000","last_unstale":"2023-10-09T06:37:38.449881+0000","last_undegraded":"2023-10-09T06:37:18.463618+0000","last_fullsized":"2023-10-09T06:37:18.304959+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7205,"log_dups_size":0,"ondisk_log_size":7205,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167280640,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3342,"num_objects_misplaced":3342,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4113,"num_write_kb":213280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[3,4],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4","objects":-583},{"shards":"3,4,5","objects":2562}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":679,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763519+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:37:31.763519+0000","last_peered":"2023-10-09T06:37:31.763519+0000","last_clean":"2023-10-09T06:37:31.763519+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:37:31.763519+0000","last_undegraded":"2023-10-09T06:37:31.763519+0000","last_fullsized":"2023-10-09T06:37:31.763519+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":147,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609509+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:37:31.609509+0000","last_peered":"2023-10-09T06:37:31.609509+0000","last_clean":"2023-10-09T06:37:31.609509+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:37:31.609509+0000","last_undegraded":"2023-10-09T06:37:31.609509+0000","last_fullsized":"2023-10-09T06:37:31.609509+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":476,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609424+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:37:31.609424+0000","last_peered":"2023-10-09T06:37:31.609424+0000","last_clean":"2023-10-09T06:37:31.609424+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:37:31.609424+0000","last_undegraded":"2023-10-09T06:37:31.609424+0000","last_fullsized":"2023-10-09T06:37:31.609424+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.13","version":"469'7405","reported_seq":7529,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-10-09T06:37:41.079283+0000","last_change":"2023-10-09T06:36:59.640496+0000","last_active":"2023-10-09T06:37:41.079283+0000","last_peered":"2023-10-09T06:37:41.079283+0000","last_clean":"2023-10-09T06:37:41.079283+0000","last_became_active":"2023-10-09T06:36:57.226174+0000","last_became_peered":"2023-10-09T06:36:57.226174+0000","last_unstale":"2023-10-09T06:37:41.079283+0000","last_undegraded":"2023-10-09T06:37:41.079283+0000","last_fullsized":"2023-10-09T06:37:41.079283+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"455'6312","last_scrub_stamp":"2023-10-09T06:36:59.640187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:36:59.640187+0000","objects_scrubbed":3175,"log_size":7405,"log_dups_size":0,"ondisk_log_size":7405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:27:01.040409+0000","scrub_duration":0.65782189000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156893184,"num_objects":2394,"num_object_clones":0,"num_object_copies":7182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1109,"num_write_kb":10496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":147,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609978+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:37:31.609978+0000","last_peered":"2023-10-09T06:37:31.609978+0000","last_clean":"2023-10-09T06:37:31.609978+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:37:31.609978+0000","last_undegraded":"2023-10-09T06:37:31.609978+0000","last_fullsized":"2023-10-09T06:37:31.609978+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":486,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.609293+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:37:31.609293+0000","last_peered":"2023-10-09T06:37:31.609293+0000","last_clean":"2023-10-09T06:37:31.609293+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:37:31.609293+0000","last_undegraded":"2023-10-09T06:37:31.609293+0000","last_fullsized":"2023-10-09T06:37:31.609293+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.10","version":"469'7456","reported_seq":13677,"reported_epoch":469,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:37:43.020625+0000","last_change":"2023-10-09T06:37:04.406619+0000","last_active":"2023-10-09T06:37:43.020625+0000","last_peered":"2023-10-09T06:37:43.020625+0000","last_clean":"2023-10-09T06:37:02.157053+0000","last_became_active":"2023-10-09T06:37:04.315163+0000","last_became_peered":"2023-10-09T06:37:04.315163+0000","last_unstale":"2023-10-09T06:37:43.020625+0000","last_undegraded":"2023-10-09T06:37:43.020625+0000","last_fullsized":"2023-10-09T06:37:03.549404+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7456,"log_dups_size":0,"ondisk_log_size":7456,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151322624,"num_objects":2309,"num_object_clones":0,"num_object_copies":6927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":209,"num_objects_unfound":0,"num_objects_dirty":2309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4359,"num_write_kb":213376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3081,"num_bytes_recovered":201916416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":2100},{"shards":"1,2,7","objects":209}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":825,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.610546+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:37:31.610546+0000","last_peered":"2023-10-09T06:37:31.610546+0000","last_clean":"2023-10-09T06:37:31.610546+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:37:31.610546+0000","last_undegraded":"2023-10-09T06:37:31.610546+0000","last_fullsized":"2023-10-09T06:37:31.610546+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763718+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:37:31.763718+0000","last_peered":"2023-10-09T06:37:31.763718+0000","last_clean":"2023-10-09T06:37:31.763718+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:37:31.763718+0000","last_undegraded":"2023-10-09T06:37:31.763718+0000","last_fullsized":"2023-10-09T06:37:31.763718+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.763105+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:37:31.763105+0000","last_peered":"2023-10-09T06:37:31.763105+0000","last_clean":"2023-10-09T06:37:31.763105+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:37:31.763105+0000","last_undegraded":"2023-10-09T06:37:31.763105+0000","last_fullsized":"2023-10-09T06:37:31.763105+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.11","version":"469'7249","reported_seq":7305,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:37:41.198556+0000","last_change":"2023-10-09T06:37:16.440148+0000","last_active":"2023-10-09T06:37:41.198556+0000","last_peered":"2023-10-09T06:37:41.198556+0000","last_clean":"2023-10-09T06:37:14.228029+0000","last_became_active":"2023-10-09T06:37:16.396220+0000","last_became_peered":"2023-10-09T06:37:16.396220+0000","last_unstale":"2023-10-09T06:37:41.198556+0000","last_undegraded":"2023-10-09T06:37:16.386662+0000","last_fullsized":"2023-10-09T06:37:16.288380+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":454,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:35:27.517411+0000","last_clean_scrub_stamp":"2023-10-09T06:35:27.517411+0000","objects_scrubbed":0,"log_size":7249,"log_dups_size":0,"ondisk_log_size":7249,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159350784,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3269,"num_objects_misplaced":3269,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4176,"num_write_kb":211424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2397},{"shards":"2,6","objects":-636}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":591,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.766342+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:37:31.766342+0000","last_peered":"2023-10-09T06:37:31.766342+0000","last_clean":"2023-10-09T06:37:31.766342+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:37:31.766342+0000","last_undegraded":"2023-10-09T06:37:31.766342+0000","last_fullsized":"2023-10-09T06:37:31.766342+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":466,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:36.081748+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:37:36.081748+0000","last_peered":"2023-10-09T06:37:36.081748+0000","last_clean":"2023-10-09T06:37:36.081748+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:37:36.081748+0000","last_undegraded":"2023-10-09T06:37:36.081748+0000","last_fullsized":"2023-10-09T06:37:36.081748+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":429,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-10-09T06:37:31.608960+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:37:31.608960+0000","last_peered":"2023-10-09T06:37:31.608960+0000","last_clean":"2023-10-09T06:37:31.608960+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:37:31.608960+0000","last_undegraded":"2023-10-09T06:37:31.608960+0000","last_fullsized":"2023-10-09T06:37:31.608960+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":4975558680,"num_objects":75922,"num_object_clones":0,"num_object_copies":227766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6611,"num_objects_misplaced":6820,"num_objects_unfound":0,"num_objects_dirty":75922,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110578,"num_write_kb":5147713,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3082,"num_bytes_recovered":201981952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15236083712,"data_stored":15236071496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180540,"ondisk_log_size":180540,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":512693,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":218,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":346022,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234485,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078076,"kb_used_data":1572980,"kb_used_omap":0,"kb_used_meta":1505088,"kb_avail":90667076,"statfs":{"total":95995035648,"available":92843085824,"internally_reserved":0,"allocated":1610731520,"data_stored":1609667224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541210112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.7269999999999999,"15min":6.085},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":8.8819999999999997,"5min":59.609000000000002,"15min":219.41900000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.8220000000000001,"5min":2.9319999999999999,"15min":6.2069999999999999},"min":{"1min":0.28799999999999998,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":5.2149999999999999,"5min":59.639000000000003,"15min":219.399},"last":0.375}]},{"osd":1,"last update":"Mon Oct 9 06:36:48 2023","interfaces":[{"interface":"back","average":{"1min":12.742000000000001,"5min":5.9649999999999999,"15min":5.4450000000000003},"min":{"1min":0.47399999999999998,"5min":0.23599999999999999,"15min":0.20599999999999999},"max":{"1min":181.15299999999999,"5min":208.626,"15min":215.61699999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":13.336,"5min":3.8439999999999999,"15min":4.6029999999999998},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":181.24199999999999,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:36:48 2023","interfaces":[{"interface":"back","average":{"1min":13.401999999999999,"5min":4.4939999999999998,"15min":7.6459999999999999},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.27700000000000002},"max":{"1min":181.90899999999999,"5min":181.90899999999999,"15min":216.34299999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":12.965,"5min":4.359,"15min":7.7640000000000002},"min":{"1min":0.33500000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":182.136,"5min":182.136,"15min":216.25200000000001},"last":0.36099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:37:31 2023","interfaces":[{"interface":"back","average":{"1min":12.720000000000001,"5min":5.3150000000000004,"15min":4.8099999999999996},"min":{"1min":0.38,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":175.40600000000001,"5min":175.40600000000001,"15min":215.922},"last":1.2849999999999999},{"interface":"front","average":{"1min":12.616,"5min":5.2080000000000002,"15min":4.8220000000000001},"min":{"1min":0.34200000000000003,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":175.30099999999999,"5min":175.30099999999999,"15min":215.94900000000001},"last":1.728}]},{"osd":4,"last update":"Mon Oct 9 06:37:31 2023","interfaces":[{"interface":"back","average":{"1min":7.1630000000000003,"5min":2.27,"15min":2.7589999999999999},"min":{"1min":0.34799999999999998,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":73.688999999999993,"5min":73.688999999999993,"15min":218.446},"last":0.42799999999999999},{"interface":"front","average":{"1min":6.6230000000000002,"5min":2.0499999999999998,"15min":2.5920000000000001},"min":{"1min":0.36699999999999999,"5min":0.28399999999999997,"15min":0.23699999999999999},"max":{"1min":71.831000000000003,"5min":71.831000000000003,"15min":216.172},"last":0.439}]},{"osd":5,"last update":"Mon Oct 9 06:37:31 2023","interfaces":[{"interface":"back","average":{"1min":10.577,"5min":2.972,"15min":2.6699999999999999},"min":{"1min":0.52900000000000003,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":71.116,"5min":71.116,"15min":216.37200000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":9.1099999999999994,"5min":2.8100000000000001,"15min":2.6739999999999999},"min":{"1min":0.36499999999999999,"5min":0.307,"15min":0.28999999999999998},"max":{"1min":70.984999999999999,"5min":70.984999999999999,"15min":218.15899999999999},"last":0.45800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:37:31 2023","interfaces":[{"interface":"back","average":{"1min":8.5030000000000001,"5min":3.0289999999999999,"15min":2.8540000000000001},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":71.087999999999994,"5min":71.087999999999994,"15min":218.19999999999999},"last":0.501},{"interface":"front","average":{"1min":8.6980000000000004,"5min":3.0329999999999999,"15min":2.6379999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33200000000000002},"max":{"1min":71.149000000000001,"5min":71.149000000000001,"15min":218.05500000000001},"last":0.51800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496121,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2438112,"kb_used_data":1530840,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91307040,"statfs":{"total":95995035648,"available":93498408960,"internally_reserved":0,"allocated":1567580160,"data_stored":1566543234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":3.194,"15min":2.746},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":6.0430000000000001,"5min":66.867000000000004,"15min":75.061999999999998},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.5389999999999999,"5min":3.4830000000000001,"15min":2.9350000000000001},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.23599999999999999},"max":{"1min":5.3239999999999998,"5min":66.856999999999999,"15min":75.021000000000001},"last":1.0289999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:36:56 2023","interfaces":[{"interface":"back","average":{"1min":2.774,"5min":3.2959999999999998,"15min":4.3209999999999997},"min":{"1min":0.26500000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.6029999999999998,"5min":65.153000000000006,"15min":292.92200000000003},"last":2.972},{"interface":"front","average":{"1min":2.5529999999999999,"5min":3.2709999999999999,"15min":4.2460000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":10.255000000000001,"5min":65.132000000000005,"15min":292.904},"last":2.2450000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":3.4790000000000001,"15min":3.0419999999999998},"min":{"1min":0.35699999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.0270000000000001,"5min":74.748999999999995,"15min":75.191999999999993},"last":3.4239999999999999},{"interface":"front","average":{"1min":2.048,"5min":3.2719999999999998,"15min":2.8679999999999999},"min":{"1min":0.34300000000000003,"5min":0.29399999999999998,"15min":0.27100000000000002},"max":{"1min":15.679,"5min":74.733999999999995,"15min":75.120000000000005},"last":3.1549999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:37:24 2023","interfaces":[{"interface":"back","average":{"1min":3.8109999999999999,"5min":3.681,"15min":3.234},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":49.051000000000002,"5min":62.223999999999997,"15min":91.174000000000007},"last":2.258},{"interface":"front","average":{"1min":4.5800000000000001,"5min":3.5680000000000001,"15min":3.2010000000000001},"min":{"1min":0.375,"5min":0.34599999999999997,"15min":0.311},"max":{"1min":49.011000000000003,"5min":61.637,"15min":91.438999999999993},"last":2.286}]},{"osd":4,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7989999999999999,"5min":1.9570000000000001,"15min":1.48},"min":{"1min":0.316,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.3700000000000001,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.98199999999999998},{"interface":"front","average":{"1min":3.8380000000000001,"5min":2.2080000000000002,"15min":1.5369999999999999},"min":{"1min":0.33600000000000002,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":14.372,"5min":86.727000000000004,"15min":86.727000000000004},"last":2.165}]},{"osd":5,"last update":"Mon Oct 9 06:36:58 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":1.387,"15min":1.359},"min":{"1min":0.34399999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":22.379999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":4.1859999999999999,"5min":1.6419999999999999,"15min":1.5009999999999999},"min":{"1min":0.42399999999999999,"5min":0.32200000000000001,"15min":0.29999999999999999},"max":{"1min":12.244999999999999,"5min":12.244999999999999,"15min":24.401},"last":2.6840000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:37:38 2023","interfaces":[{"interface":"back","average":{"1min":9.7569999999999997,"5min":2.7879999999999998,"15min":2.7029999999999998},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.26600000000000001},"max":{"1min":165.37,"5min":165.37,"15min":165.37},"last":2.1920000000000002},{"interface":"front","average":{"1min":11.199999999999999,"5min":3.2810000000000001,"15min":2.9729999999999999},"min":{"1min":0.41099999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":165.40199999999999,"5min":165.40199999999999,"15min":165.40199999999999},"last":2.2050000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402858,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2397636,"kb_used_data":1522172,"kb_used_omap":4,"kb_used_meta":875451,"kb_avail":91347516,"statfs":{"total":95995035648,"available":93539856384,"internally_reserved":0,"allocated":1558704128,"data_stored":1556641200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":896462734},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":2.7040000000000002,"15min":5.8140000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.189},"max":{"1min":5.0460000000000003,"5min":40.969999999999999,"15min":269.79199999999997},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.7629999999999999,"15min":6.2519999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.1890000000000001,"5min":40.880000000000003,"15min":271.20999999999998},"last":4.7699999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:36:47 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":2.3220000000000001,"15min":3.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.184,"15min":0.184},"max":{"1min":10.284000000000001,"5min":22.614999999999998,"15min":139.75399999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.2669999999999999,"15min":4.2039999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.577,"5min":24.372,"15min":140.29900000000001},"last":4.7999999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:37:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":2.7010000000000001,"15min":3.0369999999999999},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.215},"max":{"1min":11.292,"5min":27.064,"15min":77.331999999999994},"last":4.5620000000000003},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.99,"15min":2.1440000000000001},"min":{"1min":0.26500000000000001,"5min":0.23899999999999999,"15min":0.22700000000000001},"max":{"1min":3.2050000000000001,"5min":26.626999999999999,"15min":77.197000000000003},"last":1.964}]},{"osd":4,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":9.3109999999999999,"5min":4.0359999999999996,"15min":3.4849999999999999},"min":{"1min":0.32300000000000001,"5min":0.317,"15min":0.27500000000000002},"max":{"1min":120.309,"5min":120.309,"15min":120.309},"last":4.625},{"interface":"front","average":{"1min":9.6270000000000007,"5min":3.4990000000000001,"15min":3.3109999999999999},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":120.297,"5min":120.297,"15min":120.297},"last":4.9489999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":12.443,"5min":4.1639999999999997,"15min":2.7989999999999999},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":119.408,"5min":119.408,"15min":119.408},"last":0.58999999999999997},{"interface":"front","average":{"1min":12.667999999999999,"5min":4.1890000000000001,"15min":3.1930000000000001},"min":{"1min":0.38300000000000001,"5min":0.371,"15min":0.32700000000000001},"max":{"1min":119.483,"5min":119.483,"15min":119.483},"last":4.8470000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":6.6100000000000003,"5min":3.8719999999999999,"15min":3.3700000000000001},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":121.60899999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":4.875},{"interface":"front","average":{"1min":6.3579999999999997,"5min":4.4619999999999997,"15min":3.5350000000000001},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":121.559,"5min":121.559,"15min":121.559},"last":4.6600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:36:42 2023","interfaces":[{"interface":"back","average":{"1min":8.8859999999999992,"5min":4.8499999999999996,"15min":3.8860000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":123.20699999999999,"5min":123.20699999999999,"15min":123.20699999999999},"last":4.6890000000000001},{"interface":"front","average":{"1min":8.7650000000000006,"5min":4.2770000000000001,"15min":3.359},"min":{"1min":0.40400000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":123.48999999999999,"5min":123.48999999999999,"15min":123.48999999999999},"last":2.2949999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024291,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926136,"kb_used_data":934448,"kb_used_omap":5,"kb_used_meta":991674,"kb_avail":91819016,"statfs":{"total":95995035648,"available":94022672384,"internally_reserved":0,"allocated":956874752,"data_stored":954476705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1015474810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:36:40 2023","interfaces":[{"interface":"back","average":{"1min":4.7990000000000004,"5min":2.7759999999999998,"15min":2.6899999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.16600000000000001},"max":{"1min":73.135000000000005,"5min":73.135000000000005,"15min":191.446},"last":3.605},{"interface":"front","average":{"1min":5.1619999999999999,"5min":3.4329999999999998,"15min":3.0710000000000002},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.187},"max":{"1min":73.254999999999995,"5min":73.254999999999995,"15min":191.393},"last":6.0510000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:36:55 2023","interfaces":[{"interface":"back","average":{"1min":9.7970000000000006,"5min":4.931,"15min":3.2909999999999999},"min":{"1min":0.22700000000000001,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":100.425,"5min":100.425,"15min":100.425},"last":1.72},{"interface":"front","average":{"1min":9.1319999999999997,"5min":5.1589999999999998,"15min":3.7080000000000002},"min":{"1min":0.217,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":99.769999999999996,"5min":99.769999999999996,"15min":99.769999999999996},"last":4.4669999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":3.0459999999999998,"15min":4.5170000000000003},"min":{"1min":0.253,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":9.1869999999999994,"5min":24.129000000000001,"15min":208.11199999999999},"last":4.5019999999999998},{"interface":"front","average":{"1min":1.369,"5min":3.1539999999999999,"15min":4.3659999999999997},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.7160000000000002,"5min":24.797000000000001,"15min":208.25899999999999},"last":1.6899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":22.741,"5min":5.8259999999999996,"15min":2.9510000000000001},"min":{"1min":0.309,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":200.654,"5min":200.654,"15min":200.654},"last":3.6299999999999999},{"interface":"front","average":{"1min":23.821000000000002,"5min":6.4109999999999996,"15min":3.2829999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":200.86600000000001,"5min":200.86600000000001,"15min":200.86600000000001},"last":3.589}]},{"osd":5,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":5.4249999999999998,"5min":2.5310000000000001,"15min":1.887},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":80.885999999999996,"5min":80.885999999999996,"15min":80.885999999999996},"last":3.5409999999999999},{"interface":"front","average":{"1min":5.4409999999999998,"5min":2.5459999999999998,"15min":2.2469999999999999},"min":{"1min":0.33300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":80.438999999999993,"5min":80.438999999999993,"15min":80.438999999999993},"last":4.5300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":7.4649999999999999,"5min":3.1680000000000001,"15min":3.2450000000000001},"min":{"1min":0.36799999999999999,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":118.056,"5min":118.056,"15min":120.572},"last":4.6609999999999996},{"interface":"front","average":{"1min":7.3579999999999997,"5min":3.0619999999999998,"15min":2.831},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":118.274,"5min":118.274,"15min":120.64400000000001},"last":3.5779999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":2.2429999999999999,"15min":2.4089999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.32800000000000001},"max":{"1min":3.52,"5min":22.879000000000001,"15min":122.018},"last":3.3300000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.3180000000000001,"15min":2.7810000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.6840000000000002,"5min":22.683,"15min":122.32299999999999},"last":4.5709999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814160,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3591412,"kb_used_data":2766124,"kb_used_omap":7,"kb_used_meta":825272,"kb_avail":90153740,"statfs":{"total":95995035648,"available":92317429760,"internally_reserved":0,"allocated":2832510976,"data_stored":2830519968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":845078609},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":8.5120000000000005,"15min":5.8860000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.183},"max":{"1min":3.4119999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":2.6070000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":7.8609999999999998,"15min":5.3540000000000001},"min":{"1min":0.25600000000000001,"5min":0.20200000000000001,"15min":0.182},"max":{"1min":4.1689999999999996,"5min":156.922,"15min":168.49700000000001},"last":0.38}]},{"osd":1,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":6.9000000000000004,"15min":4.7729999999999997},"min":{"1min":0.216,"5min":0.21099999999999999,"15min":0.20699999999999999},"max":{"1min":4.5709999999999997,"5min":140.452,"15min":205.148},"last":1.3540000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":6.5090000000000003,"15min":4.9669999999999996},"min":{"1min":0.28299999999999997,"5min":0.183,"15min":0.183},"max":{"1min":4.5209999999999999,"5min":139.75299999999999,"15min":205.267},"last":2.5699999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:37:28 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":3.5059999999999998,"15min":3.2040000000000002},"min":{"1min":0.28000000000000003,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":5.5880000000000001,"5min":75.498999999999995,"15min":91.644000000000005},"last":1.5780000000000001},{"interface":"front","average":{"1min":1.4399999999999999,"5min":3.7189999999999999,"15min":3.532},"min":{"1min":0.29099999999999998,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":5.6260000000000003,"5min":73.911000000000001,"15min":91.685000000000002},"last":3.7749999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":3.0539999999999998,"5min":3.0249999999999999,"15min":2.5800000000000001},"min":{"1min":0.44800000000000001,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":27.542999999999999,"5min":73.932000000000002,"15min":90.938999999999993},"last":0.42199999999999999},{"interface":"front","average":{"1min":3.379,"5min":3.347,"15min":2.8029999999999999},"min":{"1min":0.41799999999999998,"5min":0.29999999999999999,"15min":0.26900000000000002},"max":{"1min":27.533000000000001,"5min":74.349999999999994,"15min":90.950000000000003},"last":4}]},{"osd":5,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":5.1159999999999997,"5min":2.9889999999999999,"15min":2.1680000000000001},"min":{"1min":0.34100000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":33.814,"5min":74.204999999999998,"15min":74.204999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":4.6790000000000003,"5min":3.1840000000000002,"15min":2.2749999999999999},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":33.917000000000002,"5min":74.683000000000007,"15min":74.683000000000007},"last":1.23}]},{"osd":6,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":3.556,"15min":3.617},"min":{"1min":0.35999999999999999,"5min":0.35199999999999998,"15min":0.30199999999999999},"max":{"1min":6.6319999999999997,"5min":74.653999999999996,"15min":138.291},"last":1.196},{"interface":"front","average":{"1min":1.823,"5min":3.8540000000000001,"15min":3.4820000000000002},"min":{"1min":0.45400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.6050000000000004,"5min":73.837000000000003,"15min":138.01599999999999},"last":0.60699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1379999999999999,"5min":4.0800000000000001,"15min":2.7959999999999998},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":36.155999999999999,"5min":73.863,"15min":73.863},"last":0.63400000000000001},{"interface":"front","average":{"1min":3.6970000000000001,"5min":3.1920000000000002,"15min":2.512},"min":{"1min":0.51700000000000002,"5min":0.36399999999999999,"15min":0.34999999999999998},"max":{"1min":36.228000000000002,"5min":74.171999999999997,"15min":74.171999999999997},"last":0.48599999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127357,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029408,"kb_used_data":1277016,"kb_used_omap":2,"kb_used_meta":752381,"kb_avail":91715744,"statfs":{"total":95995035648,"available":93916921856,"internally_reserved":0,"allocated":1307664384,"data_stored":1305751330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":770438197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":2.008,"15min":4.968},"min":{"1min":0.219,"5min":0.191,"15min":0.19},"max":{"1min":10.218999999999999,"5min":22.367000000000001,"15min":161.821},"last":0.317},{"interface":"front","average":{"1min":2.347,"5min":2.1499999999999999,"15min":4.7050000000000001},"min":{"1min":0.26900000000000002,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":9.8710000000000004,"5min":13.443,"15min":163.52000000000001},"last":0.434}]},{"osd":1,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":3.5339999999999998,"5min":4.6109999999999998,"15min":3.952},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":23.719000000000001,"5min":184.012,"15min":184.012},"last":1.268},{"interface":"front","average":{"1min":3.0089999999999999,"5min":5.2629999999999999,"15min":4.774},"min":{"1min":0.26100000000000001,"5min":0.25,"15min":0.20499999999999999},"max":{"1min":23.344999999999999,"5min":184.703,"15min":184.703},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":4.1369999999999996,"15min":3.851},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":6.3949999999999996,"5min":182.44900000000001,"15min":182.44900000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.2809999999999999,"5min":4.194,"15min":4.2800000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":5.319,"5min":182.56399999999999,"15min":182.56399999999999},"last":0.47599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":3.448,"5min":4.1200000000000001,"15min":2.8340000000000001},"min":{"1min":0.42199999999999999,"5min":0.318,"15min":0.26900000000000002},"max":{"1min":19.215,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.2909999999999999,"5min":4.2480000000000002,"15min":2.766},"min":{"1min":0.32400000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":19.16,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.50800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3380000000000001,"5min":5.2290000000000001,"15min":3.0339999999999998},"min":{"1min":0.41299999999999998,"5min":0.35799999999999998,"15min":0.34000000000000002},"max":{"1min":23.648,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.7280000000000002,"5min":4.0819999999999999,"15min":2.407},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":23.683,"5min":177.59399999999999,"15min":177.59399999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":2.1600000000000001,"15min":2.6789999999999998},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.0760000000000005,"5min":16.462,"15min":130.39599999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":2.4790000000000001,"5min":1.7529999999999999,"15min":2.2570000000000001},"min":{"1min":0.39100000000000001,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":9.8179999999999996,"5min":22.260000000000002,"15min":130.67699999999999},"last":2.1640000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:37:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4860000000000002,"5min":2.0289999999999999,"15min":2.7810000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.79,"5min":13.695,"15min":121.613},"last":1.9239999999999999},{"interface":"front","average":{"1min":2.2040000000000002,"5min":3.0009999999999999,"15min":3.109},"min":{"1min":0.372,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.224,"5min":87.001999999999995,"15min":121.559},"last":0.63}]}]},{"osd":4,"up_from":30,"seq":128849019394,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3963052,"kb_used_data":2779620,"kb_used_omap":0,"kb_used_meta":1183360,"kb_avail":89782100,"statfs":{"total":95995035648,"available":91936870400,"internally_reserved":0,"allocated":2846330880,"data_stored":2845282067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1211760640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":10.279,"15min":6.7510000000000003},"min":{"1min":0.247,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.7160000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":2.0430000000000001},{"interface":"front","average":{"1min":2.988,"5min":10.6,"15min":6.9429999999999996},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":9.2110000000000003,"5min":286.36000000000001,"15min":287.57100000000003},"last":5.9870000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2839999999999998,"5min":4.2270000000000003,"15min":3.1699999999999999},"min":{"1min":0.246,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":34.548999999999999,"5min":134.37100000000001,"15min":134.37100000000001},"last":6.3390000000000004},{"interface":"front","average":{"1min":3.262,"5min":4.327,"15min":3.335},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":31.024999999999999,"5min":134.21700000000001,"15min":134.21700000000001},"last":6.0919999999999996}]},{"osd":2,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.423,"5min":5.8159999999999998,"15min":3.8050000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":33.085000000000001,"5min":151.66399999999999,"15min":151.66399999999999},"last":2.9670000000000001},{"interface":"front","average":{"1min":3.569,"5min":5.6989999999999998,"15min":3.7519999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":32.299999999999997,"5min":151.58699999999999,"15min":151.58699999999999},"last":5.8250000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:37:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":4.5220000000000002,"15min":5.1710000000000003},"min":{"1min":0.40999999999999998,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":9.2479999999999993,"5min":144.399,"15min":161.19300000000001},"last":4.3920000000000003},{"interface":"front","average":{"1min":1.3,"5min":4.3700000000000001,"15min":4.8780000000000001},"min":{"1min":0.38,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":7.7300000000000004,"5min":143.84100000000001,"15min":161.934},"last":2.1539999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":0.94299999999999995,"15min":0.94699999999999995},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.252},"max":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":9.1720000000000006},"last":2.931},{"interface":"front","average":{"1min":2.3359999999999999,"5min":0.98699999999999999,"15min":0.99199999999999999},"min":{"1min":0.309,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":9.5009999999999994,"5min":9.5009999999999994,"15min":9.6379999999999999},"last":4.3460000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:37:14 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":2.4900000000000002,"15min":1.8620000000000001},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":9.4239999999999995,"5min":115.70999999999999,"15min":163.36699999999999},"last":6.0590000000000002},{"interface":"front","average":{"1min":3.0819999999999999,"5min":2.7799999999999998,"15min":2.165},"min":{"1min":0.33600000000000002,"5min":0.32200000000000001,"15min":0.27300000000000002},"max":{"1min":10.705,"5min":115.68600000000001,"15min":163.381},"last":5.2839999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:37:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":2.2330000000000001,"15min":1.655},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":5.2110000000000003,"5min":106.801,"15min":106.801},"last":5.2539999999999996},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.448,"15min":1.6599999999999999},"min":{"1min":0.40300000000000002,"5min":0.33200000000000002,"15min":0.32900000000000001},"max":{"1min":9.4749999999999996,"5min":106.827,"15min":106.827},"last":5.2969999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757759,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3488928,"kb_used_data":2595032,"kb_used_omap":0,"kb_used_meta":893504,"kb_avail":90256224,"statfs":{"total":95995035648,"available":92422373376,"internally_reserved":0,"allocated":2657312768,"data_stored":2656272471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914948096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:37:08 2023","interfaces":[{"interface":"back","average":{"1min":10.744,"5min":6.327,"15min":6.3399999999999999},"min":{"1min":0.21099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":147.405,"5min":149.149,"15min":294.89699999999999},"last":6.3150000000000004},{"interface":"front","average":{"1min":15.861000000000001,"5min":6.9119999999999999,"15min":6.4699999999999998},"min":{"1min":0.25600000000000001,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":146.31700000000001,"5min":149.21100000000001,"15min":294.30399999999997},"last":6.0570000000000004}]},{"osd":1,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":21.149000000000001,"5min":6.5359999999999996,"15min":4.3869999999999996},"min":{"1min":0.29699999999999999,"5min":0.22,"15min":0.22},"max":{"1min":188.34100000000001,"5min":188.34100000000001,"15min":188.34100000000001},"last":6.2320000000000002},{"interface":"front","average":{"1min":21.187999999999999,"5min":6.4420000000000002,"15min":4.4930000000000003},"min":{"1min":0.34200000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":188.839,"5min":188.839,"15min":188.839},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:36:54 2023","interfaces":[{"interface":"back","average":{"1min":9.5540000000000003,"5min":4.8109999999999999,"15min":3.2250000000000001},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":80.715999999999994,"5min":120.613,"15min":120.613},"last":8.3490000000000002},{"interface":"front","average":{"1min":7.4580000000000002,"5min":4.4749999999999996,"15min":3.0529999999999999},"min":{"1min":0.315,"5min":0.308,"15min":0.29699999999999999},"max":{"1min":75.640000000000001,"5min":120.962,"15min":120.962},"last":2.2040000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:37:15 2023","interfaces":[{"interface":"back","average":{"1min":7.6399999999999997,"5min":4.8300000000000001,"15min":5.3170000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":78.859999999999999,"5min":85.671000000000006,"15min":144.56399999999999},"last":6.2770000000000001},{"interface":"front","average":{"1min":7.077,"5min":4.3570000000000002,"15min":5.0819999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.28699999999999998},"max":{"1min":78.944999999999993,"5min":85.593999999999994,"15min":144.62799999999999},"last":6.0839999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:36:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4359999999999999,"5min":2.0270000000000001,"15min":1.444},"min":{"1min":0.35499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":79.043000000000006,"5min":79.043000000000006,"15min":79.043000000000006},"last":6.3449999999999998},{"interface":"front","average":{"1min":7.7809999999999997,"5min":2.0419999999999998,"15min":1.385},"min":{"1min":0.35599999999999998,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":79.165999999999997,"5min":79.165999999999997,"15min":79.165999999999997},"last":2.2959999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:37:08 2023","interfaces":[{"interface":"back","average":{"1min":5.2359999999999998,"5min":1.611,"15min":1.673},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":79.013999999999996,"5min":79.013999999999996,"15min":79.013999999999996},"last":2.246},{"interface":"front","average":{"1min":5.5810000000000004,"5min":1.764,"15min":1.764},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":80.781999999999996,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.56399999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:37:30 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":2.1360000000000001,"15min":1.5720000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":9.9659999999999993,"5min":78.890000000000001,"15min":78.890000000000001},"last":11.146000000000001},{"interface":"front","average":{"1min":1.831,"5min":1.913,"15min":1.52},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.359,"5min":79.138999999999996,"15min":79.138999999999996},"last":5.9669999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":942276608,"data_stored":942276608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1541672960,"data_stored":1541668888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2822176768,"data_stored":2822176768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1299709952,"data_stored":1299709952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2831486976,"data_stored":2831482904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2647523328,"data_stored":2647523328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1548943360,"data_stored":1548943360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1602293760,"data_stored":1602289688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:37:49.598 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 26 2023-10-09T06:37:49.678 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:49.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:49.790 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:49.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:49 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:37:49.825 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:49 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:37:49.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:49 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:37:49.831 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:49 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:37:49.935 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:50.075 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:50.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:49 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:37:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:49 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:37:50.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:50.219 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:50.301 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:50.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:50.413 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:50.435 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:37:50.455 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:50.469 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:37:50.503 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:37:50.534 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:50.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:50.560 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:37:50.603 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:37:50.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:50.655 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:37:50.691 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:37:50.725 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:37:50.725 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:37:50.725 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:37:50.725 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:37:50.726 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:37:50.728 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:50.743 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:50.744 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:37:50.777 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.777 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.778 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:37:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:50 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:37:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:50 smithi150 ceph-mon[194178]: pgmap v1433: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 4140/213090 objects degraded (1.943%); 8606/213090 objects misplaced (4.039%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:37:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:50 smithi150 ceph-mon[194178]: 13.15 deep-scrub starts 2023-10-09T06:37:50.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2327964484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:37:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[212750]: pgmap v1433: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 4140/213090 objects degraded (1.943%); 8606/213090 objects misplaced (4.039%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:37:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[212750]: 13.15 deep-scrub starts 2023-10-09T06:37:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2327964484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:37:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[220921]: pgmap v1433: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 4140/213090 objects degraded (1.943%); 8606/213090 objects misplaced (4.039%); 5.4 MiB/s, 87 objects/s recovering 2023-10-09T06:37:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[220921]: 13.15 deep-scrub starts 2023-10-09T06:37:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2327964484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:37:51.805 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:51.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:51.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:37:51.915 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:51.961 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:52.059 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:52.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:52.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:51 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:37:52.068 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:51 smithi150 ceph-mon[194178]: 1.1b deep-scrub starts 2023-10-09T06:37:52.117 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:51 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:37:52.117 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:51 smithi107 ceph-mon[212750]: 1.1b deep-scrub starts 2023-10-09T06:37:52.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:51 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:37:52.117 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:51 smithi107 ceph-mon[220921]: 1.1b deep-scrub starts 2023-10-09T06:37:52.118 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:37:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:37:52] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-10-09T06:37:52.168 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:37:52.245 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:53.090 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:37:53.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:37:53.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[212750]: pgmap v1434: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 4140/211995 objects degraded (1.953%); 8274/211995 objects misplaced (3.903%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:53.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/211995 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:53.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:37:53.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[220921]: pgmap v1434: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 4140/211995 objects degraded (1.953%); 8274/211995 objects misplaced (3.903%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:53.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:52 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/211995 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:53.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:53.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:37:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:52 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:37:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:52 smithi150 ceph-mon[194178]: pgmap v1434: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 4140/211995 objects degraded (1.953%); 8274/211995 objects misplaced (3.903%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:52 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/211995 objects degraded (1.953%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:53.199 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:53.259 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:53.377 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:53.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:53.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:53.456 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:37:53.485 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:53.523 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:53.590 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:53.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:53.700 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:53.738 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:53.821 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:53.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:53.937 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:53.986 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:54.084 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:54.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:54.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:53 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:37:54.097 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:53 smithi150 ceph-mon[194178]: 13.6 deep-scrub starts 2023-10-09T06:37:54.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:53 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:37:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:53 smithi107 ceph-mon[212750]: 13.6 deep-scrub starts 2023-10-09T06:37:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:53 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:37:54.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:53 smithi107 ceph-mon[220921]: 13.6 deep-scrub starts 2023-10-09T06:37:54.199 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.225 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:54.265 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:54.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:54.374 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:54.418 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:54.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:54.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:54.643 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:37:54.697 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:54.726 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:54.779 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:37:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:54.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:37:54.887 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:54.944 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:55.034 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:37:55.035 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[220921]: pgmap v1435: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 4140/201525 objects degraded (2.054%); 8274/201525 objects misplaced (4.106%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:55.035 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:37:55.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:37:55.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[212750]: pgmap v1435: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 4140/201525 objects degraded (2.054%); 8274/201525 objects misplaced (4.106%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:55.035 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:54 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:37:55.142 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:55.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:55.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:54 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:37:55.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:54 smithi150 ceph-mon[194178]: pgmap v1435: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 4140/201525 objects degraded (2.054%); 8274/201525 objects misplaced (4.106%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:55.159 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:54 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:37:55.249 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:37:55.281 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:55.319 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:55.397 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:37:55.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:55.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:37:55.502 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:55.511 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:37:55.560 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:55.624 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:55.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:55.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:55.731 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:55.782 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:55.849 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:55 smithi107 ceph-mon[212750]: 13.15 scrub starts 2023-10-09T06:37:55.850 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:55 smithi107 ceph-mon[220921]: 13.15 scrub starts 2023-10-09T06:37:55.857 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:55.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:55.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:55 smithi150 ceph-mon[194178]: 13.15 scrub starts 2023-10-09T06:37:55.964 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:55.990 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:56.032 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:56.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:56.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:56.140 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:56.189 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:56.691 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:37:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:56.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:37:56.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:56.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:56.888 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 13 pgp_num to 26 2023-10-09T06:37:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:56.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:56.960 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[220921]: pgmap v1436: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4140/199881 objects degraded (2.071%); 8274/199881 objects misplaced (4.139%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:37:56.960 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1390542069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[212750]: pgmap v1436: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4140/199881 objects degraded (2.071%); 8274/199881 objects misplaced (4.139%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1390542069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:56.961 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:56 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:37:57.049 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:57.095 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:56 smithi150 ceph-mon[194178]: pgmap v1436: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 4140/199881 objects degraded (2.071%); 8274/199881 objects misplaced (4.139%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:37:57.095 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1390542069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:57.095 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:56 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T06:37:57.095 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:56 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:37:57.098 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:57.190 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:57.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:57.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:57.300 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:37:57.349 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:57.468 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:37:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:57.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:37:57.579 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:57.667 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[212750]: failed to encode map e472 with expected crc 2023-10-09T06:37:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[212750]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:37:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:37:58.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/199881 objects degraded (2.071%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:58.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[220921]: failed to encode map e472 with expected crc 2023-10-09T06:37:58.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:58.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[220921]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:37:58.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:37:58.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/199881 objects degraded (2.071%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:58.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:57 smithi150 ceph-mon[194178]: failed to encode map e472 with expected crc 2023-10-09T06:37:58.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T06:37:58.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:57 smithi150 ceph-mon[194178]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T06:37:58.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:57 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:37:58.157 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/199881 objects degraded (2.071%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:37:59.071 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:59.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:59.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:59.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:37:58 smithi150 ceph-mon[194178]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 4140/198657 objects degraded (2.084%); 7990/198657 objects misplaced (4.022%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:59.176 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:37:59.176 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:37:58 smithi107 ceph-mon[212750]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 4140/198657 objects degraded (2.084%); 7990/198657 objects misplaced (4.022%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:59.177 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:37:58 smithi107 ceph-mon[220921]: pgmap v1438: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 452 op/s; 4140/198657 objects degraded (2.084%); 7990/198657 objects misplaced (4.022%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T06:37:59.270 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:59.359 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:37:59.401 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:37:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:59.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:37:59.516 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:37:59.565 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:37:59.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:37:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:37:59.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:37:59.750 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:37:59.849 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.182 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125500 11.6 11:66b58078:::benchmark_data_smithi107_418363_object35064:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 536.83675182100001, 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3024929169999999, 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125500 2023-10-09T06:38:00.183 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319867+0000", 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.184 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319865+0000", 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319874+0000", 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9840000000000007e-06 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.185 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319910+0000", 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5954000000000002e-05 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319921+0000", 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1443e-05 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.186 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319979+0000", 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7470999999999999e-05 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.319991+0000", 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2768e-05 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381006+0000", 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0610140829999999 2023-10-09T06:38:00.187 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381014+0000", 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5120000000000008e-06 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.575532+0000", 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1945181199999999 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.188 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.580920+0000", 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053880309999999997 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.581283+0000", 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00036236999999999999 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.189 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.605300+0000", 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024017113 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.612306+0000", 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007006745 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.619553+0000", 2023-10-09T06:38:00.190 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072465910000000001 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.619603+0000", 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0312000000000002e-05 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.622360+0000", 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027564299999999998 2023-10-09T06:38:00.191 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.45989.0:125514 11.6 11:6f06152b:::benchmark_data_smithi107_418363_object35078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 536.80034481200005, 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3080845179999998, 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.192 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.45989", 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/2919158578", 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 125514 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.193 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356274+0000", 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356272+0000", 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.194 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356280+0000", 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0530000000000007e-06 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356291+0000", 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0876000000000001e-05 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356299+0000", 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6820000000000003e-06 2023-10-09T06:38:00.195 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356354+0000", 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5281000000000001e-05 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:03.356364+0000", 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9359999999999994e-06 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.196 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381034+0000", 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0246703669999999 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:04.381040+0000", 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2890000000000003e-06 2023-10-09T06:38:00.197 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626453+0000", 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.245412467 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626634+0000", 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000180667 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.198 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.626806+0000", 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172688 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.636824+0000", 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010017458 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.199 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.651097+0000", 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014273447 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.661502+0000", 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010404958000000001 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.200 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.661553+0000", 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1051999999999998e-05 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:29:05.664358+0000", 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028052569999999998 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.201 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196534 12.13 12:cce4bd0b:::benchmark_data_smithi107_479504_object96923:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.52506195500001, 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0267955550000001, 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196534 2023-10-09T06:38:00.202 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.203 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631555+0000", 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631564+0000", 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9290000000000004e-06 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.204 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631578+0000", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4518000000000001e-05 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631588+0000", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6190000000000006e-06 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631676+0000", 2023-10-09T06:38:00.205 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8369999999999996e-05 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631682+0000", 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.49e-06 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634365+0000", 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026835169999999998 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.206 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634376+0000", 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0621999999999999e-05 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658287+0000", 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.023911 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.207 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658352+0000", 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5395999999999995e-05 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196538 12.13 12:cdcf32ec:::benchmark_data_smithi107_479504_object96927:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.52502241600001, 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0273450609999999, 2023-10-09T06:38:00.208 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196538 2023-10-09T06:38:00.209 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.210 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.210 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.210 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.211 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.212 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631599+0000", 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.411e-06 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631606+0000", 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6889999999999998e-06 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.213 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631611+0000", 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631755+0000", 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014380400000000001 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631760+0000", 2023-10-09T06:38:00.214 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5670000000000002e-06 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634398+0000", 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026376569999999998 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634406+0000", 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6170000000000001e-06 2023-10-09T06:38:00.215 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658849+0000", 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.024443005 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658941+0000", 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2565999999999994e-05 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.216 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196551 12.13 12:cbc1bc89:::benchmark_data_smithi107_479504_object96940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.52214957699999, 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.039733314, 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.217 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196551 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.218 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634468+0000", 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634475+0000", 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0679999999999999e-06 2023-10-09T06:38:00.219 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634487+0000", 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1916000000000001e-05 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634495+0000", 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2039999999999997e-06 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.220 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634673+0000", 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000178653 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634683+0000", 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3039999999999994e-06 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659290+0000", 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0246074949999999 2023-10-09T06:38:00.221 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659410+0000", 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119831 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659844+0000", 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043353199999999998 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.222 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.668999+0000", 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091554749999999997 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671821+0000", 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002821839 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.223 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674089+0000", 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022680840000000001 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674150+0000", 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0591000000000003e-05 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.224 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674202+0000", 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2828000000000003e-05 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196554 12.13 12:cbabae2a:::benchmark_data_smithi107_479504_object96943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.51119008800001, 2023-10-09T06:38:00.225 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0289864500000001, 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196554 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.226 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.227 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645427+0000", 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645435+0000", 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1009999999999992e-06 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645449+0000", 2023-10-09T06:38:00.228 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4063000000000001e-05 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645458+0000", 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4950000000000008e-06 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645541+0000", 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2985999999999995e-05 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.229 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645553+0000", 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2082999999999999e-05 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.665916+0000", 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0203636300000001 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.230 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.666022+0000", 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010592 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.666461+0000", 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000438412 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669034+0000", 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025731090000000001 2023-10-09T06:38:00.231 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674222+0000", 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005187779 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674318+0000", 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6755000000000005e-05 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.232 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674371+0000", 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2889000000000001e-05 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674415+0000", 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.388e-05 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.233 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196590 12.13 12:c878c42d:::benchmark_data_smithi107_479504_object96979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.466922737, 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99367597600000002, 2023-10-09T06:38:00.234 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196590 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.235 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.236 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689695+0000", 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689701+0000", 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2829999999999998e-06 2023-10-09T06:38:00.237 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689713+0000", 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1993e-05 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689721+0000", 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9650000000000008e-06 2023-10-09T06:38:00.238 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689774+0000", 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2966000000000001e-05 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689786+0000", 2023-10-09T06:38:00.239 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1858000000000001e-05 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669169+0000", 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97938338700000005 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669302+0000", 2023-10-09T06:38:00.240 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0001326 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669720+0000", 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041790600000000001 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674273+0000", 2023-10-09T06:38:00.241 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045535749999999998 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674431+0000", 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015736100000000001 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.242 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683264+0000", 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088337339999999993 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683324+0000", 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9746e-05 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.243 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683372+0000", 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7858000000000002e-05 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196596 12.13 12:cc65364a:::benchmark_data_smithi107_479504_object96985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.244 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.46323452199999, 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99598029200000004, 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196596 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.245 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:38:00.246 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693382+0000", 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693389+0000", 2023-10-09T06:38:00.247 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5930000000000002e-06 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693404+0000", 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5085999999999999e-05 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693413+0000", 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8019999999999998e-06 2023-10-09T06:38:00.248 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693622+0000", 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000208634 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693632+0000", 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0638999999999999e-05 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.249 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671865+0000", 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97823273200000005 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671973+0000", 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000107703 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.250 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.672280+0000", 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030727 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674438+0000", 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021582810000000002 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.251 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674519+0000", 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0468000000000003e-05 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689254+0000", 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014735259000000001 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689316+0000", 2023-10-09T06:38:00.252 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1838e-05 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689364+0000", 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8640999999999997e-05 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.253 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196653 12.13 12:cbc848b5:::benchmark_data_smithi107_479504_object97042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.756386+0000", 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.40023219099999, 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.93316870299999999, 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196653 2023-10-09T06:38:00.254 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756386+0000", 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756386+0000", 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.255 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756385+0000", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756393+0000", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4780000000000002e-06 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756404+0000", 2023-10-09T06:38:00.256 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1460999999999999e-05 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756412+0000", 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6450000000000005e-06 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756464+0000", 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2849000000000002e-05 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.257 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.756475+0000", 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0434999999999999e-05 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674564+0000", 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.91808890399999998 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.258 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674707+0000", 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014322799999999999 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.675064+0000", 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035685399999999998 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683390+0000", 2023-10-09T06:38:00.259 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083265130000000007 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683442+0000", 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1799999999999999e-05 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689478+0000", 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006035655 2023-10-09T06:38:00.260 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689518+0000", 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0092000000000002e-05 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689555+0000", 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7218999999999997e-05 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.261 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196720 12.13 12:c8eaa71e:::benchmark_data_smithi107_479504_object97109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.864041+0000", 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.29257774000001, 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83450754900000002, 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.262 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196720 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864041+0000", 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864041+0000", 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.263 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864041+0000", 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864043+0000", 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7829999999999999e-06 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.264 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864050+0000", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2609999999999999e-06 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.864055+0000", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4269999999999999e-06 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.866443+0000", 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002387541 2023-10-09T06:38:00.265 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.866453+0000", 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0937e-05 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.677516+0000", 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.81106242299999998 2023-10-09T06:38:00.266 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.677646+0000", 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000129951 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.677975+0000", 2023-10-09T06:38:00.267 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00032896100000000001 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683452+0000", 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054769370000000003 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689421+0000", 2023-10-09T06:38:00.268 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059690500000000001 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698458+0000", 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090376759999999997 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.269 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698506+0000", 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7108999999999999e-05 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698548+0000", 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2895999999999997e-05 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.270 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196765 12.13 12:ccc2d86c:::benchmark_data_smithi107_479504_object97154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.941949+0000", 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.21466961300001, 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.76371342099999995, 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.271 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196765 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941949+0000", 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.272 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941949+0000", 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941949+0000", 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.273 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941951+0000", 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6510000000000001e-06 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.274 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941962+0000", 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.025e-05 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941968+0000", 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3439999999999997e-06 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.275 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.942025+0000", 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6762999999999998e-05 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.942036+0000", 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1443e-05 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.276 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.680435+0000", 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.738398373 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.680547+0000", 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000112636 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.277 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.680901+0000", 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00035386899999999998 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689433+0000", 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0085317899999999992 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.278 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.692399+0000", 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002965868 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705566+0000", 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013167252000000001 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.279 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705619+0000", 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3009e-05 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705662+0000", 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3538999999999998e-05 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.280 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196768 12.13 12:cb0c4837:::benchmark_data_smithi107_479504_object97157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.941980+0000", 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.21463826199999, 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.76378957999999997, 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.281 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196768 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941980+0000", 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.282 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941980+0000", 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941980+0000", 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.283 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941983+0000", 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.74e-06 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941989+0000", 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9460000000000003e-06 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.941994+0000", 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2639999999999999e-06 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.284 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.942074+0000", 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9791999999999998e-05 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.942092+0000", 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7872000000000001e-05 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.285 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683492+0000", 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.74140016900000005 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683619+0000", 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012741 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683992+0000", 2023-10-09T06:38:00.286 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00037292199999999998 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.692436+0000", 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0084437790000000002 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698409+0000", 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059735029999999998 2023-10-09T06:38:00.287 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705699+0000", 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072897120000000003 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705736+0000", 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6804999999999997e-05 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.288 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705770+0000", 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4100999999999998e-05 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196843 12.13 12:ccdd910f:::benchmark_data_smithi107_479504_object97232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:02.105760+0000", 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 178.050858436, 2023-10-09T06:38:00.289 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.60011683000000005, 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196843 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105760+0000", 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.290 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105760+0000", 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105759+0000", 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.291 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105766+0000", 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8269999999999998e-06 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105783+0000", 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6274999999999999e-05 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.292 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105793+0000", 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0169e-05 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105873+0000", 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0065e-05 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.293 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.105886+0000", 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3152e-05 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.686200+0000", 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.580314196 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.686302+0000", 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101535 2023-10-09T06:38:00.294 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.686695+0000", 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00039285699999999998 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698417+0000", 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011722003999999999 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.295 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698602+0000", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018521499999999999 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705806+0000", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0072038909999999996 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705843+0000", 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7567999999999997e-05 2023-10-09T06:38:00.296 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705877+0000", 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3704999999999997e-05 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196896 12.13 12:ce208052:::benchmark_data_smithi107_479504_object97285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:02.229126+0000", 2023-10-09T06:38:00.297 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 177.927492352, 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.47686181900000002, 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196896 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.298 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229126+0000", 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229126+0000", 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229125+0000", 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:38:00.299 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229132+0000", 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.537e-06 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229148+0000", 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5743000000000002e-05 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.300 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229157+0000", 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1079999999999999e-06 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229220+0000", 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2632000000000006e-05 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.229231+0000", 2023-10-09T06:38:00.301 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.163e-05 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689595+0000", 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.46036379199999999 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689716+0000", 2023-10-09T06:38:00.302 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012130599999999999 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.690108+0000", 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00039160699999999998 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.303 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698560+0000", 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0084517810000000002 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698667+0000", 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010723800000000001 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.304 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705916+0000", 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.007248567 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705955+0000", 2023-10-09T06:38:00.305 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9545999999999997e-05 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.705988+0000", 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3013999999999999e-05 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.306 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196915 12.13 12:c831af66:::benchmark_data_smithi107_479504_object97304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:02.269729+0000", 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 177.886889543, 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.44818395500000002, 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.307 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196915 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269729+0000", 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.308 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269729+0000", 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.309 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269728+0000", 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269736+0000", 2023-10-09T06:38:00.310 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1019999999999997e-06 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269746+0000", 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1060000000000006e-06 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.311 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269752+0000", 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9199999999999998e-06 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269795+0000", 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2070999999999998e-05 2023-10-09T06:38:00.312 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.269803+0000", 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8480000000000007e-06 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.692461+0000", 2023-10-09T06:38:00.313 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.42265726100000001 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.692575+0000", 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114257 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.314 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.692956+0000", 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00038067500000000002 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698565+0000", 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056097630000000002 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.315 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698722+0000", 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000156859 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.717823+0000", 2023-10-09T06:38:00.317 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019100479 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.717874+0000", 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1140999999999997e-05 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.717913+0000", 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9263e-05 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.318 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196933 12.13 12:cea003f0:::benchmark_data_smithi107_479504_object97322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:02.322359+0000", 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 177.83425985700001, 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.39566097099999997, 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:38:00.319 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196933 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322359+0000", 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322359+0000", 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.320 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322357+0000", 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322365+0000", 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3149999999999992e-06 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.321 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322382+0000", 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7283999999999999e-05 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322391+0000", 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3469999999999999e-06 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.322 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322440+0000", 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9586999999999998e-05 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.322457+0000", 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6731e-05 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.695090+0000", 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.37263299500000002 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.323 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.695234+0000", 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014364099999999999 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.695670+0000", 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043608999999999998 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.324 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.698734+0000", 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030647130000000002 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.706460+0000", 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077255980000000002 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.717954+0000", 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011494013000000001 2023-10-09T06:38:00.325 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.717984+0000", 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9567999999999999e-05 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.718020+0000", 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6143999999999998e-05 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.326 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:123520 13.a 13:5a8f5645:::benchmark_data_smithi107_535588_object17413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:37:28.726700+0000", 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 31.429918398000002, 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.284263506, 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:38:00.327 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 123520 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726700+0000", 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726700+0000", 2023-10-09T06:38:00.328 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726699+0000", 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726705+0000", 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4590000000000003e-06 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.329 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726717+0000", 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2136e-05 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.726725+0000", 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3799999999999996e-06 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.330 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.737147+0000", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010421926 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.737284+0000", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013697700000000001 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.737398+0000", 2023-10-09T06:38:00.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114356 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.740511+0000", 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031126690000000002 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745700+0000", 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051897430000000001 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010888+0000", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26518737399999998 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010930+0000", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2398000000000003e-05 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010964+0000", 2023-10-09T06:38:00.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3515e-05 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "MOSDPGPushReply(13.10 467/457 [PushReplyOp(13:0e4c9fb3:::benchmark_data_smithi107_535588_object58895:head)])", 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:37:28.745886+0000", 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 31.410732388, 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26462858500000003, 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:38:00.334 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745886+0000", 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745886+0000", 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745886+0000", 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745893+0000", 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0650000000000001e-06 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745905+0000", 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2473000000000001e-05 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745912+0000", 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3119999999999996e-06 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.745966+0000", 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3433999999999997e-05 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010515+0000", 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26454894200000001 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.338 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:123542 13.10 13:093f2e3e:::benchmark_data_smithi107_535588_object17435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:37:28.746379+0000", 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 31.410239766, 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27605010299999999, 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.339 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 123542 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746379+0000", 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.340 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746379+0000", 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.341 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746378+0000", 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746384+0000", 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0549999999999996e-06 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.342 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746395+0000", 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.145e-05 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746402+0000", 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1319999999999996e-06 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.343 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010548+0000", 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26414567999999999 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010683+0000", 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000135375 2023-10-09T06:38:00.344 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.011065+0000", 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000381803 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.016782+0000", 2023-10-09T06:38:00.345 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0057171189999999997 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.019093+0000", 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023107179999999998 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.346 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.021338+0000", 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022455370000000001 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.021390+0000", 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1709999999999998e-05 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.347 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.022429+0000", 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0010389049999999999 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.348 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:123546 13.c 13:337fe96d:::benchmark_data_smithi107_535588_object17439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:37:28.746411+0000", 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 31.410207702000001, 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26605848300000001, 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:38:00.349 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 123546 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746411+0000", 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746411+0000", 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:38:00.350 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746411+0000", 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:38:00.351 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746413+0000", 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.678e-06 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746419+0000", 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2249999999999999e-06 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746424+0000", 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0560000000000004e-06 2023-10-09T06:38:00.352 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746471+0000", 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7824000000000002e-05 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746587+0000", 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011599099999999999 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.353 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:28.746695+0000", 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000107197 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.354 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010510+0000", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.263815621 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.010673+0000", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016322600000000001 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.012394+0000", 2023-10-09T06:38:00.355 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0017207909999999999 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.012433+0000", 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8574e-05 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:38:00.356 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:37:29.012469+0000", 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6624e-05 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:38:00.357 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:38:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:00.358 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:00.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:00.459 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:00.540 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:00.625 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:00.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:00.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:00.734 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:00.795 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:00.892 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:00.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:00.947 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:38:01.003 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:01.048 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:00 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:38:01.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:00 smithi150 ceph-mon[194178]: pgmap v1439: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/187098 objects degraded (2.213%); 7990/187098 objects misplaced (4.270%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:38:01.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:01.049 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:00 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:38:01.056 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:01.156 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:01.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:01.274 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:01.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:38:01.326 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[212750]: pgmap v1439: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/187098 objects degraded (2.213%); 7990/187098 objects misplaced (4.270%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:38:01.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:01.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:38:01.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:38:01.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[220921]: pgmap v1439: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/187098 objects degraded (2.213%); 7990/187098 objects misplaced (4.270%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:38:01.327 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:01.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:00 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:38:01.339 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:02.324 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:02] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1836449854' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: osdmap e473: 8 total, 8 up, 8 in 2023-10-09T06:38:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: failed to encode map e473 with expected crc 2023-10-09T06:38:02.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:02.599 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:02.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:02.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:02.794 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:02.881 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:03.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:02 smithi107 ceph-mon[212750]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4140/185733 objects degraded (2.229%); 7636/185733 objects misplaced (4.111%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:03.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/185733 objects degraded (2.229%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:02 smithi107 ceph-mon[220921]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4140/185733 objects degraded (2.229%); 7636/185733 objects misplaced (4.111%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:03.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/185733 objects degraded (2.229%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:03.378 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:38:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:02 smithi150 ceph-mon[194178]: pgmap v1441: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 4140/185733 objects degraded (2.229%); 7636/185733 objects misplaced (4.111%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/185733 objects degraded (2.229%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:03.416 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:03.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:03.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:03.537 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:03.653 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:03.811 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:03.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:03.928 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:03.988 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:04.063 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:04.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:04.169 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:04.171 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:38:04.202 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:38:04.212 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:04.236 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:04.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:38:04.294 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:38:04.328 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:04.333 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:38:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:04.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:04.375 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:38:04.405 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:38:04.405 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:38:04.412 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:38:04.443 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:38:04.447 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.470 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:38:04.538 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:04.732 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:04.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:04.845 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:04.882 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:05.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:04 smithi107 ceph-mon[220921]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4140/174447 objects degraded (2.373%); 7636/174447 objects misplaced (4.377%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:05.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1428054997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:05.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:04 smithi107 ceph-mon[212750]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4140/174447 objects degraded (2.373%); 7636/174447 objects misplaced (4.377%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:05.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1428054997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:04 smithi150 ceph-mon[194178]: pgmap v1442: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4140/174447 objects degraded (2.373%); 7636/174447 objects misplaced (4.377%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T06:38:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1428054997' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:05.457 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:38:06.517 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:06.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:06.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:06.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:06.653 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:06.758 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:06.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:06.863 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:06.940 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:07.227 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:07.245 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:07.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:07.255 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:07.347 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:07.382 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:06 smithi150 ceph-mon[194178]: pgmap v1443: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 4140/172755 objects degraded (2.396%); 7636/172755 objects misplaced (4.420%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:38:07.398 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:07.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:06 smithi107 ceph-mon[212750]: pgmap v1443: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 4140/172755 objects degraded (2.396%); 7636/172755 objects misplaced (4.420%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:38:07.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:06 smithi107 ceph-mon[220921]: pgmap v1443: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s; 4140/172755 objects degraded (2.396%); 7636/172755 objects misplaced (4.420%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:38:07.493 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:07.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:07.603 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:07.661 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:07.803 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:07.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:07.867 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:38:07.868 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:38:07.911 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:07.963 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/172755 objects degraded (2.396%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:07.999 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:08.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/172755 objects degraded (2.396%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/172755 objects degraded (2.396%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:08.581 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:08.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:08.685 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:08.737 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:08.808 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:08.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:08.913 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:08.942 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:09.023 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:38:09.042 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:09.051 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:08 smithi150 ceph-mon[194178]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 4140/171465 objects degraded (2.414%); 7314/171465 objects misplaced (4.266%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:09.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:09.156 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:09.207 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:09.307 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:09.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:09.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:09.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:08 smithi107 ceph-mon[220921]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 4140/171465 objects degraded (2.414%); 7314/171465 objects misplaced (4.266%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:08 smithi107 ceph-mon[212750]: pgmap v1444: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 4140/171465 objects degraded (2.414%); 7314/171465 objects misplaced (4.266%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:09.416 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:09.471 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:09.575 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:09.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:09.688 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:09.770 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:10.751 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:38:11.040 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:10 smithi107 ceph-mon[212750]: pgmap v1445: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 4140/159813 objects degraded (2.591%); 7314/159813 objects misplaced (4.577%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:11.041 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:10 smithi107 ceph-mon[220921]: pgmap v1445: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 4140/159813 objects degraded (2.591%); 7314/159813 objects misplaced (4.577%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:11.078 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:11.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:11.197 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:11.218 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:11.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:10 smithi150 ceph-mon[194178]: pgmap v1445: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 4140/159813 objects degraded (2.591%); 7314/159813 objects misplaced (4.577%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:38:11.240 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:11.338 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:11.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:11.446 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:11.488 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:11.573 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:11.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:11.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:11.689 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:11.792 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:12.018 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:12.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:12.127 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:12.184 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:12.209 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:12.256 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:12.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:12.363 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:12.373 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:12] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-10-09T06:38:12.453 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:12.594 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:12.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:12.707 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:12.732 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:12.773 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:12.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:12.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:12.905 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:12.946 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:12.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:12.981 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:38:12.982 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:38:13.052 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: pgmap v1446: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4140/158550 objects degraded (2.611%); 6992/158550 objects misplaced (4.410%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1173117244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3174524012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/158550 objects degraded (2.611%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:13.078 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:13.118 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:13.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:13.225 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:13.306 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: pgmap v1446: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4140/158550 objects degraded (2.611%); 6992/158550 objects misplaced (4.410%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1173117244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3174524012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/158550 objects degraded (2.611%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: pgmap v1446: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4140/158550 objects degraded (2.611%); 6992/158550 objects misplaced (4.410%); 2.9 MiB/s, 47 objects/s recovering 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1173117244' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:38:13.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3174524012' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:13.308 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/158550 objects degraded (2.611%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:13.311 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:14.371 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:14.375 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[220921]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:14.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:13 smithi107 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:38:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:14.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: osdmap e474: 8 total, 8 up, 8 in 2023-10-09T06:38:14.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:13 smithi150 ceph-mon[194178]: failed to encode map e474 with expected crc 2023-10-09T06:38:14.478 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:14.544 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:14.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:14.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:14.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:14.751 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:14.778 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:14 smithi107 ceph-mon[212750]: failed to encode map e474 with expected crc 2023-10-09T06:38:15.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:14 smithi107 ceph-mon[212750]: pgmap v1448: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 4140/148725 objects degraded (2.784%); 6992/148725 objects misplaced (4.701%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:14 smithi107 ceph-mon[220921]: failed to encode map e474 with expected crc 2023-10-09T06:38:15.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:14 smithi107 ceph-mon[220921]: pgmap v1448: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 4140/148725 objects degraded (2.784%); 6992/148725 objects misplaced (4.701%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:15.357 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:15.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:15.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:15.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:14 smithi150 ceph-mon[194178]: failed to encode map e474 with expected crc 2023-10-09T06:38:15.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:14 smithi150 ceph-mon[194178]: pgmap v1448: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 4140/148725 objects degraded (2.784%); 6992/148725 objects misplaced (4.701%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:15.468 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:15.469 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:38:15.529 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:15.616 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:15.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:15.724 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:15.764 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:15.827 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:15.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:15.920 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:38:15.934 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:15.954 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:38:15.981 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:15.988 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:38:16.025 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:38:16.060 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:38:16.076 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:16.085 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:15 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2548870682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:16.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:16.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:16.114 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:38:16.166 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:38:16.188 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:16.209 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.217 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.218 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:38:16.245 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:16.248 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:15 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2548870682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:16.248 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:15 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2548870682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:16.340 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:16.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:16.458 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:16.541 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:17 smithi107 ceph-mon[212750]: pgmap v1449: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/146754 objects degraded (2.821%); 6992/146754 objects misplaced (4.764%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:17.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:17.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:17 smithi107 ceph-mon[220921]: pgmap v1449: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/146754 objects degraded (2.821%); 6992/146754 objects misplaced (4.764%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:17.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:17 smithi150 ceph-mon[194178]: pgmap v1449: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 4140/146754 objects degraded (2.821%); 6992/146754 objects misplaced (4.764%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:38:17.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:17.426 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:38:18.041 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:18.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:18.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:18 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/146754 objects degraded (2.821%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:18 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/146754 objects degraded (2.821%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:18.157 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:18.207 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:18.239 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:18.273 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:18 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/146754 objects degraded (2.821%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:38:18.433 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:18.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:18.538 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:18.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:18.691 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:18.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:18.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:18.862 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:38:18.862 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:38:18.862 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:38:18.862 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:38:18.863 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:38:18.880 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:19.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:19 smithi107 ceph-mon[212750]: pgmap v1450: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 4140/146319 objects degraded (2.829%); 6656/146319 objects misplaced (4.549%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:19 smithi107 ceph-mon[220921]: pgmap v1450: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 4140/146319 objects degraded (2.829%); 6656/146319 objects misplaced (4.549%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:19 smithi150 ceph-mon[194178]: pgmap v1450: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 4140/146319 objects degraded (2.829%); 6656/146319 objects misplaced (4.549%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:20.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:38:21.125 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:21.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:21.228 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:21.255 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:21.319 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:21.333 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:21 smithi150 ceph-mon[194178]: pgmap v1451: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4140/134073 objects degraded (3.088%); 6656/134073 objects misplaced (4.964%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:21.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:21.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:21.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:21 smithi107 ceph-mon[212750]: pgmap v1451: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4140/134073 objects degraded (3.088%); 6656/134073 objects misplaced (4.964%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:21.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:21 smithi107 ceph-mon[220921]: pgmap v1451: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4140/134073 objects degraded (3.088%); 6656/134073 objects misplaced (4.964%); 2.1 MiB/s, 32 objects/s recovering 2023-10-09T06:38:21.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:21.507 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:21.797 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:38:21.853 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:21.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:21.967 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:22.026 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:22.062 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:22 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:22.062 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:22] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-10-09T06:38:22.107 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:22.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:22.203 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:22.219 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:22.290 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:22.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:22 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:22 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:38:22.565 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:22.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:22.671 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:22.745 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:22.836 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:22.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:22.946 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:22.988 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:23.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:23 smithi107 ceph-mon[212750]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/133686 objects degraded (3.097%); 6272/133686 objects misplaced (4.692%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:23.067 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:23 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/133686 objects degraded (3.097%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:23.073 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:23.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:23.115 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:23.179 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:23.311 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:23.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:23 smithi107 ceph-mon[220921]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/133686 objects degraded (3.097%); 6272/133686 objects misplaced (4.692%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:23.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:23 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/133686 objects degraded (3.097%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:23.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:23 smithi150 ceph-mon[194178]: pgmap v1452: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/133686 objects degraded (3.097%); 6272/133686 objects misplaced (4.692%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:23.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:23 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/133686 objects degraded (3.097%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:23.665 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:23.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:23.774 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:23.824 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:24.091 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:38:24.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2727020872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:24.092 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:24.101 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:38:24.124 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:38:24.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:38:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2727020872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:24.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:24.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:38:24.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2727020872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:24.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 4140/123252 objects degraded (3.359%); 6272/123252 objects misplaced (5.089%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:38:25.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[212750]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 4140/123252 objects degraded (3.359%); 6272/123252 objects misplaced (5.089%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:38:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:25 smithi107 ceph-mon[220921]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: pgmap v1453: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 4140/123252 objects degraded (3.359%); 6272/123252 objects misplaced (5.089%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3746324915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: osdmap e475: 8 total, 8 up, 8 in 2023-10-09T06:38:25.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:25 smithi150 ceph-mon[194178]: failed to encode map e475 with expected crc 2023-10-09T06:38:25.674 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:25.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:25.787 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:25.850 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:26.624 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:38:26.631 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:26.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:26.741 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:26.789 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:26.868 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:26.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:26.973 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:27.023 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:27.111 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:27.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[220921]: pgmap v1455: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4140/120900 objects degraded (3.424%); 6272/120900 objects misplaced (5.188%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:27.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:38:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[212750]: pgmap v1455: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4140/120900 objects degraded (3.424%); 6272/120900 objects misplaced (5.188%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:27.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:27 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:38:27.223 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:27.290 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:27.388 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:27.396 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:27 smithi150 ceph-mon[194178]: pgmap v1455: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 4140/120900 objects degraded (3.424%); 6272/120900 objects misplaced (5.188%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T06:38:27.397 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.397 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:27 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:27.397 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:27 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:38:27.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:27.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:27.499 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:27.569 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:27.825 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:38:27.826 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:38:27.826 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:38:27.826 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:38:27.830 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:38:27.865 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:38:27.901 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:38:27.941 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:38:27.957 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:38:27.958 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:38:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:28 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/120900 objects degraded (3.424%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3516556990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:28.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:28 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/120900 objects degraded (3.424%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:28.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3516556990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:28.403 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:28 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/120900 objects degraded (3.424%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:28.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3516556990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:28.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:38:28.874 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:28.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:28.999 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:29.033 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:29.107 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:29.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:29.138 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:29.215 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:29.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:29 smithi107 ceph-mon[212750]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 4140/120519 objects degraded (3.435%); 5900/120519 objects misplaced (4.895%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:29.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:29 smithi107 ceph-mon[220921]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 4140/120519 objects degraded (3.435%); 5900/120519 objects misplaced (4.895%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:29.308 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:29 smithi150 ceph-mon[194178]: pgmap v1456: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 4140/120519 objects degraded (3.435%); 5900/120519 objects misplaced (4.895%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:29.515 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:29.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:38:29.599 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:38:29.622 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:29.630 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:29.725 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:30.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:30.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:30.440 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:30.482 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:30.608 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:30.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:30.721 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:30.777 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:30.993 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:38:31.394 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:31 smithi107 ceph-mon[212750]: pgmap v1457: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/109077 objects degraded (3.795%); 5900/109077 objects misplaced (5.409%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:31.394 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:31 smithi107 ceph-mon[220921]: pgmap v1457: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/109077 objects degraded (3.795%); 5900/109077 objects misplaced (5.409%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:31.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:31 smithi150 ceph-mon[194178]: pgmap v1457: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/109077 objects degraded (3.795%); 5900/109077 objects misplaced (5.409%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T06:38:32.097 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:32.106 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:32] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T06:38:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:32.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:32.115 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:38:32.209 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:32.295 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:38:32.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:32 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:38:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:32 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:38:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:32.605 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:32.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:32.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:32.724 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:32.826 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:33.400 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:33 smithi107 ceph-mon[212750]: pgmap v1458: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/108708 objects degraded (3.808%); 5500/108708 objects misplaced (5.059%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:33.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:33 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/108708 objects degraded (3.808%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:33.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:33 smithi107 ceph-mon[220921]: pgmap v1458: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/108708 objects degraded (3.808%); 5500/108708 objects misplaced (5.059%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:33.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:33 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/108708 objects degraded (3.808%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:33 smithi150 ceph-mon[194178]: pgmap v1458: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 4140/108708 objects degraded (3.808%); 5500/108708 objects misplaced (5.059%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:33 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/108708 objects degraded (3.808%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:33.915 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:33.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:34.023 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:34.147 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:34.210 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:34.296 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:34.390 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:34 smithi107 ceph-mon[220921]: 13.15 scrub starts 2023-10-09T06:38:34.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:34 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:38:34.394 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:34 smithi107 ceph-mon[212750]: 13.15 scrub starts 2023-10-09T06:38:34.394 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:34 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:38:34.403 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:34 smithi150 ceph-mon[194178]: 13.15 scrub starts 2023-10-09T06:38:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:34 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:38:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:34.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:34.510 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:34.588 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:34.960 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:34.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:35.064 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:35.105 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:35.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[220921]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4140/98604 objects degraded (4.199%); 5500/98604 objects misplaced (5.578%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:35.173 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2098143393' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:35.173 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:35.175 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[212750]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4140/98604 objects degraded (4.199%); 5500/98604 objects misplaced (5.578%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:35.175 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2098143393' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:35.176 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:35.181 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:35.185 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:38:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:35.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:35.287 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:35.378 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:35 smithi150 ceph-mon[194178]: pgmap v1459: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 4140/98604 objects degraded (4.199%); 5500/98604 objects misplaced (5.578%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2098143393' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:38:36.083 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:36.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:36.201 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:36.203 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:38:36.271 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:38:36.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1849024315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:38:36.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1849024315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:36.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:36 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: osdmap e476: 8 total, 8 up, 8 in 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: failed to encode map e476 with expected crc 2023-10-09T06:38:36.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1849024315' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:36.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:36 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: pgmap v1461: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/96045 objects degraded (4.310%); 5500/96045 objects misplaced (5.726%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:38:37.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[212750]: failed to encode map e477 with expected crc 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: pgmap v1461: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/96045 objects degraded (4.310%); 5500/96045 objects misplaced (5.726%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: failed to encode map e477 with expected crc 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:38:37.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:37 smithi107 ceph-mon[220921]: failed to encode map e477 with expected crc 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: pgmap v1461: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4140/96045 objects degraded (4.310%); 5500/96045 objects misplaced (5.726%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: failed to encode map e477 with expected crc 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: osdmap e477: 8 total, 8 up, 8 in 2023-10-09T06:38:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:37 smithi150 ceph-mon[194178]: failed to encode map e477 with expected crc 2023-10-09T06:38:38.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:38 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/96045 objects degraded (4.310%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:38.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:38 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/96045 objects degraded (4.310%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:38.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:38 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/96045 objects degraded (4.310%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:39.099 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:38:39.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:39 smithi107 ceph-mon[212750]: pgmap v1463: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 4140/95700 objects degraded (4.326%); 5040/95700 objects misplaced (5.266%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:38:39.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:39 smithi107 ceph-mon[220921]: pgmap v1463: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 4140/95700 objects degraded (4.326%); 5040/95700 objects misplaced (5.266%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:38:39.613 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:39.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:39 smithi150 ceph-mon[194178]: pgmap v1463: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 4140/95700 objects degraded (4.326%); 5040/95700 objects misplaced (5.266%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:38:39.729 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:39.786 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:39.840 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:39.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:39.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:40.068 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:40.230 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:40.345 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[212750]: 1.1b scrub starts 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[220921]: 1.1b scrub starts 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.350 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:40.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:40.380 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:38:40.380 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:38:40.452 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:40.497 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:40 smithi150 ceph-mon[194178]: 1.1b scrub starts 2023-10-09T06:38:40.497 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.497 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:40.504 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:40.598 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:40.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:40.706 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:40.752 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:38:40.795 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:38:40.803 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:40.832 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:38:40.863 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:38:40.905 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:38:40.950 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:38:40.986 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:38:41.023 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:38:41.039 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.040 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: pgmap v1464: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 4140/82998 objects degraded (4.988%); 5040/82998 objects misplaced (6.072%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:41.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1144904894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:41.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: pgmap v1464: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 4140/82998 objects degraded (4.988%); 5040/82998 objects misplaced (6.072%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:38:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1144904894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: pgmap v1464: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 4140/82998 objects degraded (4.988%); 5040/82998 objects misplaced (6.072%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:41.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1144904894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:41.721 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:38:42.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:42] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T06:38:42.321 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:42.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:42.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:42.435 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:42.500 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:42.544 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:42.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:42.649 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:42.680 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:43.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:43 smithi107 ceph-mon[212750]: pgmap v1465: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 4140/82542 objects degraded (5.016%); 4588/82542 objects misplaced (5.558%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:38:43.229 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:43 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/82542 objects degraded (5.016%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:43.336 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:38:43.516 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:43.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:43.529 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:43 smithi107 ceph-mon[220921]: pgmap v1465: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 4140/82542 objects degraded (5.016%); 4588/82542 objects misplaced (5.558%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:38:43.529 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:43 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/82542 objects degraded (5.016%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:43.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:43.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:43.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:43 smithi150 ceph-mon[194178]: pgmap v1465: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 4140/82542 objects degraded (5.016%); 4588/82542 objects misplaced (5.558%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:38:43.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:43 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/82542 objects degraded (5.016%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:43.738 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:43.852 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:43.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:43.924 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:43.967 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:44.036 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:44.147 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:44.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:44.254 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:44.305 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:44.406 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:44.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:44 smithi107 ceph-mon[220921]: 13.6 scrub starts 2023-10-09T06:38:44.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:44 smithi107 ceph-mon[212750]: 13.6 scrub starts 2023-10-09T06:38:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:44.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:44.517 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:44.589 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:44.611 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:44 smithi150 ceph-mon[194178]: 13.6 scrub starts 2023-10-09T06:38:44.819 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:44.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:44.926 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:45.007 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:45.259 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:38:45.638 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:45.649 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[212750]: pgmap v1466: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 4140/72957 objects degraded (5.675%); 4588/72957 objects misplaced (6.289%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[212750]: 13.16 scrub starts 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2005682587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[220921]: pgmap v1466: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 4140/72957 objects degraded (5.675%); 4588/72957 objects misplaced (6.289%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[220921]: 13.16 scrub starts 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2005682587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:45 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:45.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:45.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:45 smithi150 ceph-mon[194178]: pgmap v1466: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 4140/72957 objects degraded (5.675%); 4588/72957 objects misplaced (6.289%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T06:38:45.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:45 smithi150 ceph-mon[194178]: 13.16 scrub starts 2023-10-09T06:38:45.659 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2005682587' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.660 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:45 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:38:45.752 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:45.805 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:45.814 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:38:45.884 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:45.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:45.995 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:46.052 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:46.102 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:46.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:46.207 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:46.249 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:46.324 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:46.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.335 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:46 smithi150 ceph-mon[194178]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.435 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:46.483 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[220921]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:38:46.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: osdmap e478: 8 total, 8 up, 8 in 2023-10-09T06:38:46.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:46 smithi107 ceph-mon[212750]: failed to encode map e478 with expected crc 2023-10-09T06:38:46.734 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:46.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:46.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:46.748 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:38:46.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:46.976 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:47.618 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:47.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:47.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:47.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[212750]: pgmap v1468: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4140/70590 objects degraded (5.865%); 4588/70590 objects misplaced (6.500%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[220921]: pgmap v1468: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4140/70590 objects degraded (5.865%); 4588/70590 objects misplaced (6.500%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:47.636 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:47 smithi150 ceph-mon[194178]: pgmap v1468: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4140/70590 objects degraded (5.865%); 4588/70590 objects misplaced (6.500%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T06:38:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:47 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:38:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:38:47.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:38:47.729 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:47.787 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:48.101 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:48.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:48.208 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:48.232 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:48.256 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:48.352 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:48.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:48.377 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:48.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:38:48.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4140/70590 objects degraded (5.865%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:48.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3506642557' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:38:48.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4140/70590 objects degraded (5.865%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:48.405 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3506642557' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.460 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:48.531 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:48 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:38:48.531 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:48 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4140/70590 objects degraded (5.865%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:48.531 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3506642557' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.531 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:38:48.533 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:48.598 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:38:48.610 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:48.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:48.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:48.722 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:48.778 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:48.875 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:48.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:48.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:48.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:49.063 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:49.529 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:49.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:49.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:49.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[212750]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4140/70257 objects degraded (5.893%); 4140/70257 objects misplaced (5.893%); 2.8 MiB/s, 48 objects/s recovering 2023-10-09T06:38:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[212750]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:38:49.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:38:49.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[220921]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4140/70257 objects degraded (5.893%); 4140/70257 objects misplaced (5.893%); 2.8 MiB/s, 48 objects/s recovering 2023-10-09T06:38:49.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:49.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:49.655 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[220921]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:38:49.656 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:49 smithi107 ceph-mon[220921]: failed to encode map e479 with expected crc 2023-10-09T06:38:49.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:49 smithi150 ceph-mon[194178]: pgmap v1469: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 4140/70257 objects degraded (5.893%); 4140/70257 objects misplaced (5.893%); 2.8 MiB/s, 48 objects/s recovering 2023-10-09T06:38:49.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:49 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:38:49.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:49 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:38:49.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:49 smithi150 ceph-mon[194178]: osdmap e479: 8 total, 8 up, 8 in 2023-10-09T06:38:49.657 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:49 smithi150 ceph-mon[194178]: failed to encode map e479 with expected crc 2023-10-09T06:38:49.698 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:49.791 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:49.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:49.901 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:49.951 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:50.068 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:50.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:50.182 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:50.239 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:50.313 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:50.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:50.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:50 smithi107 ceph-mon[212750]: failed to encode map e479 with expected crc 2023-10-09T06:38:50.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:50 smithi107 ceph-mon[220921]: failed to encode map e479 with expected crc 2023-10-09T06:38:50.420 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:50.446 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:50.520 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:50.530 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:50 smithi150 ceph-mon[194178]: failed to encode map e479 with expected crc 2023-10-09T06:38:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:50.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:50.633 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:38:50.708 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:50.815 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:38:50.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:50.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:38:50.922 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:51.019 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:51.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:51 smithi107 ceph-mon[212750]: pgmap v1471: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 4140/58677 objects degraded (7.056%); 4140/58677 objects misplaced (7.056%); 1.7 MiB/s, 32 objects/s recovering 2023-10-09T06:38:51.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:51 smithi107 ceph-mon[220921]: pgmap v1471: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 4140/58677 objects degraded (7.056%); 4140/58677 objects misplaced (7.056%); 1.7 MiB/s, 32 objects/s recovering 2023-10-09T06:38:51.628 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:38:51.628 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:38:51.628 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:38:51.628 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:38:51.628 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:38:51.651 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:51 smithi150 ceph-mon[194178]: pgmap v1471: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s; 4140/58677 objects degraded (7.056%); 4140/58677 objects misplaced (7.056%); 1.7 MiB/s, 32 objects/s recovering 2023-10-09T06:38:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:51.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:51.758 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:51.803 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:51.927 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:51.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:52.042 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:52.115 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:52.208 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:52.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:52.322 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:52.380 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:38:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:38:52] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-10-09T06:38:52.387 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:52.769 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:52.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:52.878 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:52.947 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:52.977 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:38:53.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:38:53.139 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:53.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:53.238 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:38:53.249 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:53.276 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:38:53.318 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:38:53.366 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:38:53.372 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:53.436 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:38:53.484 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:38:53.526 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:53.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:53.537 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:38:53.594 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:38:53.635 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:38:53.645 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.645 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.645 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.645 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.646 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.646 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.646 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.646 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:38:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[212750]: pgmap v1472: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4028/58263 objects degraded (6.913%); 4028/58263 objects misplaced (6.913%); 1.7 MiB/s, 67 objects/s recovering 2023-10-09T06:38:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4028/58263 objects degraded (6.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:53.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1818839420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[220921]: pgmap v1472: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4028/58263 objects degraded (6.913%); 4028/58263 objects misplaced (6.913%); 1.7 MiB/s, 67 objects/s recovering 2023-10-09T06:38:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4028/58263 objects degraded (6.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:53.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1818839420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:53 smithi150 ceph-mon[194178]: pgmap v1472: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 4028/58263 objects degraded (6.913%); 4028/58263 objects misplaced (6.913%); 1.7 MiB/s, 67 objects/s recovering 2023-10-09T06:38:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:53 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4028/58263 objects degraded (6.913%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:53.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1818839420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:38:53.733 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:53.883 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:38:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:53.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:38:53.996 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:54.032 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:54.074 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:54.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:54.177 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:54.203 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:54.244 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:38:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:54.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:38:54.347 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:54.374 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:54 smithi107 ceph-mon[212750]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 995 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4028/47829 objects degraded (8.422%); 4028/47829 objects misplaced (8.422%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T06:38:54.374 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:38:54.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:54 smithi107 ceph-mon[220921]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 995 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4028/47829 objects degraded (8.422%); 4028/47829 objects misplaced (8.422%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T06:38:54.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:38:54.388 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:38:54.401 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:54.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:54.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:54.719 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:54 smithi150 ceph-mon[194178]: pgmap v1473: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 995 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 4028/47829 objects degraded (8.422%); 4028/47829 objects misplaced (8.422%); 1.7 MiB/s, 64 objects/s recovering 2023-10-09T06:38:54.719 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:38:54.820 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:54.878 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:54.987 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:55.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:55.107 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:55.165 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:55.460 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:55.472 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.473 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.473 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:38:55.473 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:38:55.473 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.473 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[220921]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.474 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.474 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.474 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:38:55.474 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:38:55.474 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:55 smithi107 ceph-mon[212750]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.482 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:55.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:55.494 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:38:55.591 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:55.593 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:38:55.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1767753727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:38:55.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: osdmap e480: 8 total, 8 up, 8 in 2023-10-09T06:38:55.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:55 smithi150 ceph-mon[194178]: failed to encode map e480 with expected crc 2023-10-09T06:38:55.634 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:55.718 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:38:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:55.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:38:55.833 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:38:55.883 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:55.973 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:38:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:55.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:38:56.083 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:56.178 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:56.267 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:56.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:56.373 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:56.391 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:56 smithi150 ceph-mon[194178]: pgmap v1475: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 946 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4028/45465 objects degraded (8.860%); 4028/45465 objects misplaced (8.860%); 0 B/s, 34 objects/s recovering 2023-10-09T06:38:56.394 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:56 smithi107 ceph-mon[212750]: pgmap v1475: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 946 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4028/45465 objects degraded (8.860%); 4028/45465 objects misplaced (8.860%); 0 B/s, 34 objects/s recovering 2023-10-09T06:38:56.394 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:56 smithi107 ceph-mon[220921]: pgmap v1475: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 946 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4028/45465 objects degraded (8.860%); 4028/45465 objects misplaced (8.860%); 0 B/s, 34 objects/s recovering 2023-10-09T06:38:56.429 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:57.222 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:38:57.730 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:57.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:57.842 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:38:57.899 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:57.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 4028/45465 objects degraded (8.860%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:57.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 4028/45465 objects degraded (8.860%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:57.906 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 4028/45465 objects degraded (8.860%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:38:57.960 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:58.119 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:38:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:58.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:38:58.237 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:38:58.291 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:58.385 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:38:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:58.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:38:58.495 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:38:58.504 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:38:58.574 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:58 smithi107 ceph-mon[212750]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 938 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3750/45099 objects degraded (8.315%); 3750/45099 objects misplaced (8.315%); 0 B/s, 62 objects/s recovering 2023-10-09T06:38:58.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:58 smithi107 ceph-mon[220921]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 938 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3750/45099 objects degraded (8.315%); 3750/45099 objects misplaced (8.315%); 0 B/s, 62 objects/s recovering 2023-10-09T06:38:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:58 smithi150 ceph-mon[194178]: pgmap v1476: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 938 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3750/45099 objects degraded (8.315%); 3750/45099 objects misplaced (8.315%); 0 B/s, 62 objects/s recovering 2023-10-09T06:38:59.012 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:38:59.012 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:38:59.294 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:38:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:38:59.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:38:59.408 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:38:59.436 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:38:59.752 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:38:59.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2190675093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:59.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:38:59 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:59.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2190675093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:59.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:38:59 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2190675093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:38:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:38:59 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:00.739 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: pgmap v1477: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 692 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3750/33267 objects degraded (11.272%); 3750/33267 objects misplaced (11.272%); 0 B/s, 55 objects/s recovering 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.740 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[220921]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: pgmap v1477: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 692 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3750/33267 objects degraded (11.272%); 3750/33267 objects misplaced (11.272%); 0 B/s, 55 objects/s recovering 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.741 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:00 smithi107 ceph-mon[212750]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: pgmap v1477: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 127 active+clean; 692 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3750/33267 objects degraded (11.272%); 3750/33267 objects misplaced (11.272%); 0 B/s, 55 objects/s recovering 2023-10-09T06:39:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: osdmap e481: 8 total, 8 up, 8 in 2023-10-09T06:39:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: failed to encode map e481 with expected crc 2023-10-09T06:39:00.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:00 smithi150 ceph-mon[194178]: failed to encode map e481 with expected crc 2023-10-09T06:39:01.123 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:39:01.124 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:01.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:01.233 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:01.323 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:01.460 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:01.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:01.573 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:01.624 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:01.691 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:01.731 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:01.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:01.843 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:01.916 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:02.066 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:02.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:02.176 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:02.220 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:02.386 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:02] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T06:39:02.547 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:02.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[212750]: pgmap v1479: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 684 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3466/32868 objects degraded (10.545%); 3466/32868 objects misplaced (10.545%); 0 B/s, 70 objects/s recovering 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3466/32868 objects degraded (10.545%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[220921]: pgmap v1479: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 684 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3466/32868 objects degraded (10.545%); 3466/32868 objects misplaced (10.545%); 0 B/s, 70 objects/s recovering 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:02.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3466/32868 objects degraded (10.545%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:02.656 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:02.695 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:02.746 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:39:02.771 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:02.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:02.791 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:39:02.839 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:39:02.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:02.879 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:39:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:02 smithi150 ceph-mon[194178]: pgmap v1479: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 684 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3466/32868 objects degraded (10.545%); 3466/32868 objects misplaced (10.545%); 0 B/s, 70 objects/s recovering 2023-10-09T06:39:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:02.906 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3466/32868 objects degraded (10.545%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:02.911 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:39:02.937 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:02.973 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:39:03.021 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:39:03.035 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:03.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:03.058 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.065 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.066 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.066 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:39:03.141 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:03.167 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:39:03.219 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:03.372 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:03.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:03.482 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:03.536 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:03.578 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:03.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:03.627 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3329305345' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:03.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3329305345' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:03.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2183966858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:03.627 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:03.633 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:39:03.682 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:03.709 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2183966858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:03.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3329305345' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:03 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2183966858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:03.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:39:04.117 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:04.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:04.220 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:04.254 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:04.310 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:04.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:04.413 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:04.442 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[212750]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 486 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 3466/23385 objects degraded (14.821%); 3466/23385 objects misplaced (14.821%); 0 B/s, 61 objects/s recovering 2023-10-09T06:39:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[212750]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:39:04.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[212750]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:39:04.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[220921]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 486 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 3466/23385 objects degraded (14.821%); 3466/23385 objects misplaced (14.821%); 0 B/s, 61 objects/s recovering 2023-10-09T06:39:04.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[220921]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[220921]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:39:04.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:04 smithi107 ceph-mon[220921]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:39:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:04 smithi150 ceph-mon[194178]: pgmap v1480: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 486 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 3466/23385 objects degraded (14.821%); 3466/23385 objects misplaced (14.821%); 0 B/s, 61 objects/s recovering 2023-10-09T06:39:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:04 smithi150 ceph-mon[194178]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:04 smithi150 ceph-mon[194178]: failed to encode map e482 with expected crc 2023-10-09T06:39:04.907 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:04 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:39:04.908 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:04 smithi150 ceph-mon[194178]: osdmap e482: 8 total, 8 up, 8 in 2023-10-09T06:39:05.216 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:05.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:05.246 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:39:05.327 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:05.367 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:05.399 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:05.496 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:05.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:05.604 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:05.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[212750]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:39:05.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[212750]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[220921]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[220921]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:39:05.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[220921]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:05 smithi107 ceph-mon[220921]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.679 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:05.725 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:05.725 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:39:05.725 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:39:05.725 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:05.794 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:05.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:05.902 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:05 smithi150 ceph-mon[194178]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:05 smithi150 ceph-mon[194178]: osdmap e483: 8 total, 8 up, 8 in 2023-10-09T06:39:05.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:05 smithi150 ceph-mon[194178]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:05 smithi150 ceph-mon[194178]: failed to encode map e483 with expected crc 2023-10-09T06:39:05.910 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:05.984 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:06.053 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:06.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:06.156 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:06.183 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:06.225 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:06.327 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:06.396 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:06.597 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:39:06.601 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:06.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:06.715 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:06.743 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:06.786 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:06.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:06.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:06.889 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:06 smithi150 ceph-mon[194178]: pgmap v1483: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 85 KiB/s, 48 objects/s recovering 2023-10-09T06:39:06.917 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:06.955 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:39:06.974 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:06.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:06.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:06 smithi107 ceph-mon[212750]: pgmap v1483: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 85 KiB/s, 48 objects/s recovering 2023-10-09T06:39:06.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:06 smithi107 ceph-mon[220921]: pgmap v1483: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 85 KiB/s, 48 objects/s recovering 2023-10-09T06:39:07.039 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:07.079 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:07.164 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:07.436 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:07.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:07.545 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:07.607 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:07.613 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:07.806 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:07.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:07.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:07.821 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:07.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 3047/21132 objects degraded (14.419%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:07.904 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 3047/21132 objects degraded (14.419%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:07.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 3047/21132 objects degraded (14.419%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:39:07.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:07.956 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:08.041 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:08.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:08.145 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:08.244 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:08.540 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:08.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:08.658 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:08.693 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:08.814 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:08 smithi150 ceph-mon[194178]: pgmap v1484: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 65 KiB/s, 37 objects/s recovering 2023-10-09T06:39:08.843 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:08.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:08.886 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:08 smithi107 ceph-mon[220921]: pgmap v1484: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 65 KiB/s, 37 objects/s recovering 2023-10-09T06:39:08.888 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:08 smithi107 ceph-mon[212750]: pgmap v1484: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 127 active+clean; 439 MiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 3047/21132 objects degraded (14.419%); 3457/21132 objects misplaced (16.359%); 65 KiB/s, 37 objects/s recovering 2023-10-09T06:39:08.952 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:08.998 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:09.154 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:09.154 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:39:09.154 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:39:09.154 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:39:09.154 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:09.236 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:09.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:09.345 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:09.407 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:09.487 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:09.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:09.591 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:09.618 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:09.756 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:39:09.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:09 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3152274302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:09 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:09 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3152274302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:09.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:09 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:09.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:09 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3152274302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:09.905 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:09 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:10.024 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:10.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:10.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:10.128 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:39:10.130 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:10.224 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:10.354 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:10.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:10.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:10.473 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:10.528 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:10.614 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:10.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:10.698 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:10.721 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:10.806 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: pgmap v1485: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 42 objects/s recovering 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:39:10.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[212750]: failed to encode map e484 with expected crc 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: pgmap v1485: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 42 objects/s recovering 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: failed to encode map e484 with expected crc 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:39:10.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:10 smithi107 ceph-mon[220921]: failed to encode map e484 with expected crc 2023-10-09T06:39:10.962 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:10.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:10.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: pgmap v1485: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 907 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 42 objects/s recovering 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: failed to encode map e484 with expected crc 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: osdmap e484: 8 total, 8 up, 8 in 2023-10-09T06:39:10.988 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:10 smithi150 ceph-mon[194178]: failed to encode map e484 with expected crc 2023-10-09T06:39:11.068 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:11.113 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:11.304 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:11.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:11.421 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:11.461 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:11.996 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[212750]: 1.10 scrub starts 2023-10-09T06:39:11.996 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[212750]: 1.10 scrub ok 2023-10-09T06:39:11.996 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[212750]: 13.9 scrub starts 2023-10-09T06:39:11.996 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[220921]: 1.10 scrub starts 2023-10-09T06:39:11.996 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[220921]: 1.10 scrub ok 2023-10-09T06:39:11.997 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:11 smithi107 ceph-mon[220921]: 13.9 scrub starts 2023-10-09T06:39:12.091 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:12.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:11 smithi150 ceph-mon[194178]: 1.10 scrub starts 2023-10-09T06:39:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:11 smithi150 ceph-mon[194178]: 1.10 scrub ok 2023-10-09T06:39:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:11 smithi150 ceph-mon[194178]: 13.9 scrub starts 2023-10-09T06:39:12.202 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:12.262 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:12.263 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:12] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T06:39:12.353 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:12.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:12.463 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:12.549 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:12.658 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:12.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:12.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:12.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:12.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:12.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[212750]: pgmap v1487: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 43 objects/s recovering 2023-10-09T06:39:12.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2923/11082 objects degraded (26.376%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:12.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:12.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[220921]: pgmap v1487: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 43 objects/s recovering 2023-10-09T06:39:12.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2923/11082 objects degraded (26.376%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:12.872 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:32545 12.0 12:01fec15d:::benchmark_data_smithi107_479504_object32544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 464.73708893000003, 2023-10-09T06:39:12.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.280403281, 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 32545 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.898 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156734+0000", 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156733+0000", 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156799+0000", 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5947999999999999e-05 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156811+0000", 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2396e-05 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156818+0000", 2023-10-09T06:39:12.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3509999999999998e-06 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156855+0000", 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7357000000000002e-05 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.156935+0000", 2023-10-09T06:39:12.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9637999999999999e-05 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.157020+0000", 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5840000000000005e-05 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158089+0000", 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010683859999999999 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.158849+0000", 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00076036400000000003 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437051+0000", 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27820162700000001 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437100+0000", 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9347e-05 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:28.437137+0000", 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7234000000000003e-05 2023-10-09T06:39:12.905 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:170329 12.2 12:4444f55c:::benchmark_data_smithi107_479504_object70718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:23.677727+0000", 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 289.21609635599998, 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31114080300000002, 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.906 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 170329 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677727+0000", 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677727+0000", 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677726+0000", 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677730+0000", 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5789999999999997e-06 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677740+0000", 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.3610000000000007e-06 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677744+0000", 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2010000000000001e-06 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677800+0000", 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6202999999999997e-05 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.677923+0000", 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000122939 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.678402+0000", 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047929100000000002 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.681089+0000", 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026868819999999998 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.682042+0000", 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00095323000000000001 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.913 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.988808+0000", 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.306765281 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.988842+0000", 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4047000000000001e-05 2023-10-09T06:39:12.914 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:23.988868+0000", 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5955000000000001e-05 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.6 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.74929123499999, 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.4013005, 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.916 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.917 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.919 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.922 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144530+0000", 2023-10-09T06:39:12.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:12.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.922 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.923 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.923 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144569+0000", 2023-10-09T06:39:12.923 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8152000000000003e-05 2023-10-09T06:39:12.923 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.923 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343804+0000", 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19923497000000001 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.924 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.925 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343819+0000", 2023-10-09T06:39:12.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.579e-05 2023-10-09T06:39:12.925 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.925 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.925 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.926 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344902+0000", 2023-10-09T06:39:12.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010829030000000001 2023-10-09T06:39:12.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345087+0000", 2023-10-09T06:39:12.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000184298 2023-10-09T06:39:12.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:12.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345267+0000", 2023-10-09T06:39:12.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000180574 2023-10-09T06:39:12.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.355828+0000", 2023-10-09T06:39:12.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010561034 2023-10-09T06:39:12.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545825+0000", 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.189996794 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545832+0000", 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.16 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:12.931 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.548970349, 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.90554467699999996, 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:39:12.932 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344851+0000", 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344911+0000", 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0288999999999998e-05 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344924+0000", 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3145999999999999e-05 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344930+0000", 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3849999999999999e-06 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545781+0000", 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20085007999999999 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:12.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545788+0000", 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5059999999999998e-06 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244178+0000", 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69839010700000004 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for degraded object", 2023-10-09T06:39:12.937 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244920+0000", 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00074191700000000003 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250315+0000", 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053952330000000001 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250397+0000", 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2120000000000007e-05 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98798 13.d 13:b3d56a11:::benchmark_data_smithi107_535588_object98797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.537707703, 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32981040700000003, 2023-10-09T06:39:12.939 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98798 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:12.940 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356114+0000", 2023-10-09T06:39:12.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3579000000000003e-05 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356182+0000", 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5062000000000001e-05 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356191+0000", 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9409999999999996e-06 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356320+0000", 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012913700000000001 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356328+0000", 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2300000000000002e-06 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578270+0000", 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.221941894 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578431+0000", 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016151199999999999 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:12.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.579090+0000", 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00065903200000000004 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617475+0000", 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.038384553000000002 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617547+0000", 2023-10-09T06:39:12.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1931e-05 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685815+0000", 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.068268076999999996 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685868+0000", 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3386e-05 2023-10-09T06:39:12.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.948 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.948 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.948 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685926+0000", 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7726000000000003e-05 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98801 13.d 13:b7ce87f9:::benchmark_data_smithi107_535588_object98800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.377744+0000", 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.51607872100001, 2023-10-09T06:39:12.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30825868200000001, 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98801 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.950 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377744+0000", 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377744+0000", 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377742+0000", 2023-10-09T06:39:12.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999971 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377807+0000", 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5646000000000001e-05 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377830+0000", 2023-10-09T06:39:12.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2243e-05 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.377842+0000", 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2541000000000001e-05 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.584336+0000", 2023-10-09T06:39:12.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20649357800000001 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.584491+0000", 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000155556 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.584655+0000", 2023-10-09T06:39:12.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016388000000000001 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617500+0000", 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.032844432 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617567+0000", 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7836999999999994e-05 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685936+0000", 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.068368560999999994 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685964+0000", 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7645e-05 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686003+0000", 2023-10-09T06:39:12.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9424000000000001e-05 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98803 13.15 13:abc5b445:::benchmark_data_smithi107_535588_object98802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:39:12.958 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.325836684, 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.67498770900000005, 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98803 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567984+0000", 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568068+0000", 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4116999999999994e-05 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568098+0000", 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9153999999999999e-05 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568108+0000", 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0000000000000001e-05 2023-10-09T06:39:12.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594900+0000", 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026791966 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594910+0000", 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0601000000000001e-05 2023-10-09T06:39:12.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231876+0000", 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.63696583200000001 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231989+0000", 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113386 2023-10-09T06:39:12.965 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.232227+0000", 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00023801399999999999 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.966 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235621+0000", 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033936639999999998 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235647+0000", 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5706000000000001e-05 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.967 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242883+0000", 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072361309999999998 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242933+0000", 2023-10-09T06:39:12.968 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0300000000000003e-05 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242974+0000", 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0917000000000001e-05 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.969 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98804 13.16 13:6bab783a:::benchmark_data_smithi107_535588_object98803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 136.20768922299999, 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57256559900000004, 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.970 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98804 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.971 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686133+0000", 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686232+0000", 2023-10-09T06:39:12.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9147999999999993e-05 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686245+0000", 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3025e-05 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686253+0000", 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0690000000000002e-06 2023-10-09T06:39:12.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686291+0000", 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8095e-05 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686300+0000", 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1460000000000002e-06 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.974 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244979+0000", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.55867869000000003 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245112+0000", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000133514 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245364+0000", 2023-10-09T06:39:12.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000251501 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252198+0000", 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068343839999999998 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252224+0000", 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5789999999999999e-05 2023-10-09T06:39:12.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.255637+0000", 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034128890000000001 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258639+0000", 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003001709 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258699+0000", 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0885999999999999e-05 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:123544 13.d 13:bbc021b8:::benchmark_data_smithi107_535588_object17437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:37:28.746401+0000", 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 104.147421792, 2023-10-09T06:39:12.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27850799799999998, 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 123544 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746401+0000", 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746401+0000", 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746401+0000", 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746404+0000", 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8739999999999999e-06 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746410+0000", 2023-10-09T06:39:12.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2779999999999999e-06 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.746414+0000", 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2100000000000003e-06 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.749465+0000", 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030513369999999999 2023-10-09T06:39:12.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.749570+0000", 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010416000000000001 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.749676+0000", 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106258 2023-10-09T06:39:12.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.752654+0000", 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002977717 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:28.756644+0000", 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0039906519999999999 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:29.024838+0000", 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26819358900000001 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.985 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:29.024875+0000", 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6860999999999998e-05 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:37:29.024909+0000", 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4586999999999999e-05 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:205773 13.6 13:624e0fc4:::benchmark_data_smithi107_535588_object99666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 9.3726435349999999, 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31593875100000002, 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.987 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 205773 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521178+0000", 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521184+0000", 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.366e-06 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521194+0000", 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0015e-05 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521200+0000", 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8919999999999997e-06 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521246+0000", 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6483000000000002e-05 2023-10-09T06:39:12.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521364+0000", 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117922 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:12.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521840+0000", 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047612499999999999 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.524383+0000", 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0025425209999999998 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.525153+0000", 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077040099999999999 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837031+0000", 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31187785499999998 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837079+0000", 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8609999999999997e-05 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837118+0000", 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8887999999999998e-05 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.996 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:205777 13.16 13:69b0e304:::benchmark_data_smithi107_535588_object99670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:03.527630+0000", 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 9.3661927019999993, 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30945920900000001, 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:12.997 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 205777 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527630+0000", 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527630+0000", 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:12.999 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527629+0000", 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527636+0000", 2023-10-09T06:39:13.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4590000000000001e-06 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527652+0000", 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6019e-05 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.001 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527659+0000", 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8430000000000004e-06 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527702+0000", 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3300000000000002e-05 2023-10-09T06:39:13.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.527822+0000", 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119809 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.528255+0000", 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00043275300000000002 2023-10-09T06:39:13.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.530207+0000", 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019523019999999999 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.531570+0000", 2023-10-09T06:39:13.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001363043 2023-10-09T06:39:13.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837002+0000", 2023-10-09T06:39:13.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30543166599999999 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837049+0000", 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6508999999999998e-05 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837090+0000", 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.104e-05 2023-10-09T06:39:13.006 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207722 13.d 13:b874e618:::benchmark_data_smithi107_535588_object101615:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:39:13.007 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.704809+0000", 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 6.1890141009999997, 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25607787700000001, 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.008 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207722 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704809+0000", 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704809+0000", 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704808+0000", 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:13.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704812+0000", 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4839999999999998e-06 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704821+0000", 2023-10-09T06:39:13.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704826+0000", 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9130000000000002e-06 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704877+0000", 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0865999999999997e-05 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.704994+0000", 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116984 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:13.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.705111+0000", 2023-10-09T06:39:13.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000116902 2023-10-09T06:39:13.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.017 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715023+0000", 2023-10-09T06:39:13.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0099123300000000004 2023-10-09T06:39:13.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.019 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.019 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.019 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715283+0000", 2023-10-09T06:39:13.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00025953599999999999 2023-10-09T06:39:13.019 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.960779+0000", 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245496513 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.960835+0000", 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5506e-05 2023-10-09T06:39:13.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.960887+0000", 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2315999999999999e-05 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207724 13.d 13:bc8d3f40:::benchmark_data_smithi107_535588_object101617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.706365+0000", 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 6.1874578920000003, 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25688670899999999, 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.022 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207724 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706365+0000", 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706365+0000", 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706364+0000", 2023-10-09T06:39:13.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706370+0000", 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3409999999999999e-06 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706388+0000", 2023-10-09T06:39:13.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7819e-05 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706395+0000", 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3610000000000003e-06 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.708022+0000", 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0016265120000000001 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.708125+0000", 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103112 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.708217+0000", 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1664999999999997e-05 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715029+0000", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068120730000000001 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715324+0000", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029552000000000002 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.963172+0000", 2023-10-09T06:39:13.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24784811800000001 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.963218+0000", 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.583e-05 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.963252+0000", 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3657e-05 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.029 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207726 13.d 13:b609888f:::benchmark_data_smithi107_535588_object101619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.706407+0000", 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 6.1874162019999996, 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26063185799999999, 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.030 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207726 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706407+0000", 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706407+0000", 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706406+0000", 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706410+0000", 2023-10-09T06:39:13.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.128e-06 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706416+0000", 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9310000000000002e-06 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706421+0000", 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3780000000000003e-06 2023-10-09T06:39:13.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711856+0000", 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005434932 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711986+0000", 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012982099999999999 2023-10-09T06:39:13.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712111+0000", 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000125716 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715292+0000", 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031811719999999999 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715352+0000", 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9799999999999997e-05 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.966936+0000", 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25158379800000003 2023-10-09T06:39:13.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.966995+0000", 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8776000000000001e-05 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.967039+0000", 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3886e-05 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.037 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211407 13.6 13:67af2662:::benchmark_data_smithi107_535588_object105300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.664915+0000", 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.2289084109999999, 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26404735200000001, 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.038 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211407 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664915+0000", 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664915+0000", 2023-10-09T06:39:13.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664914+0000", 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664922+0000", 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1729999999999995e-06 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664932+0000", 2023-10-09T06:39:13.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0597e-05 2023-10-09T06:39:13.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664939+0000", 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8090000000000004e-06 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.664987+0000", 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7898000000000001e-05 2023-10-09T06:39:13.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665102+0000", 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665519+0000", 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041767799999999999 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670744+0000", 2023-10-09T06:39:13.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0052246430000000002 2023-10-09T06:39:13.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670841+0000", 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6911999999999996e-05 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928874+0000", 2023-10-09T06:39:13.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.258032708 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928921+0000", 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7793999999999997e-05 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928962+0000", 2023-10-09T06:39:13.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0670000000000002e-05 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211414 13.6 13:67821ee2:::benchmark_data_smithi107_535588_object105307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.666388+0000", 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.22743512, 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26266919900000002, 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.047 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211414 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666388+0000", 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666388+0000", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666387+0000", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666391+0000", 2023-10-09T06:39:13.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.284e-06 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666399+0000", 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8080000000000005e-06 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666403+0000", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3449999999999999e-06 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.667801+0000", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013977709999999999 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.667922+0000", 2023-10-09T06:39:13.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120597 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668324+0000", 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000402595 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928840+0000", 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26051602200000001 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928847+0000", 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5710000000000003e-06 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928991+0000", 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000144019 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.929027+0000", 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6248000000000003e-05 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.929057+0000", 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9861000000000001e-05 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211415 13.d 13:be3b476b:::benchmark_data_smithi107_535588_object105308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.666411+0000", 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.2274124259999999, 2023-10-09T06:39:13.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26429518299999999, 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211415 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.056 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666411+0000", 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666411+0000", 2023-10-09T06:39:13.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666410+0000", 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666412+0000", 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.024e-06 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666416+0000", 2023-10-09T06:39:13.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.365e-06 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666418+0000", 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2019999999999998e-06 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666450+0000", 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1977000000000002e-05 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666555+0000", 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104769 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:39:13.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666661+0000", 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106703 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.926858+0000", 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26019652799999998 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930604+0000", 2023-10-09T06:39:13.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0037460480000000001 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930627+0000", 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2951000000000001e-05 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930670+0000", 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2653e-05 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930706+0000", 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6236999999999999e-05 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211416 13.15 13:a84b54f6:::benchmark_data_smithi107_535588_object105309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.666425+0000", 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.2273977460000001, 2023-10-09T06:39:13.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26497543699999998, 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211416 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666425+0000", 2023-10-09T06:39:13.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666425+0000", 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666425+0000", 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666427+0000", 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.289e-06 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666433+0000", 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7200000000000003e-06 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666437+0000", 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7120000000000002e-06 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666487+0000", 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0423999999999997e-05 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.666588+0000", 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101103 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.667062+0000", 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047419500000000001 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930548+0000", 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26348589900000002 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.930577+0000", 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8713000000000001e-05 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931328+0000", 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00075084999999999995 2023-10-09T06:39:13.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931367+0000", 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8594999999999999e-05 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931401+0000", 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4282999999999997e-05 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.071 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.46706.0:211411 13.a e484/437 13:57a2c115:::benchmark_data_smithi107_535588_object105304:head v 484'13140, mlcod=484'13138)", 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672569+0000", 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.2212544009999999, 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26744587199999997, 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672569+0000", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672569+0000", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672568+0000", 2023-10-09T06:39:13.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672576+0000", 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5610000000000001e-06 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.927483+0000", 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25490770800000001 2023-10-09T06:39:13.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.927492+0000", 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1789999999999997e-06 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928246+0000", 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00075338599999999999 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928266+0000", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9766e-05 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.939982+0000", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011716236999999999 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.940015+0000", 2023-10-09T06:39:13.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2756000000000003e-05 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211419 13.16 13:6fc85f18:::benchmark_data_smithi107_535588_object105312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672861+0000", 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 1.220962221, 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27010876700000003, 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:13.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211419 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672861+0000", 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672861+0000", 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672860+0000", 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672866+0000", 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2500000000000003e-06 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.926868+0000", 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25400254 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.926883+0000", 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4962e-05 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.927263+0000", 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00038009200000000001 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.927388+0000", 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012455300000000001 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.927947+0000", 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055948499999999997 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.932297+0000", 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043496560000000004 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.932710+0000", 2023-10-09T06:39:13.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041320400000000002 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.942891+0000", 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010180903999999999 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:13.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.942936+0000", 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4706999999999997e-05 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.942970+0000", 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3720000000000002e-05 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:13.085 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:13.086 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:13.086 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:12 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:13.086 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:12 smithi150 ceph-mon[194178]: pgmap v1487: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 230 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 2923/11082 objects degraded (26.376%); 2923/11082 objects misplaced (26.376%); 1.7 MiB/s, 43 objects/s recovering 2023-10-09T06:39:13.086 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2923/11082 objects degraded (26.376%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:13.088 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 106104 objects 2023-10-09T06:39:13.088 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :125.333 2023-10-09T06:39:13.088 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:39:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:13.089 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:13.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:13.090 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:39:13.090 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:39:13.091 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:39:13.091 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:39:13.091 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:39:13.091 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:39:13.100 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:39:13.101 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.101 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.102 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:39:13.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:13.222 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:13.319 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:13.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:13.424 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:13.453 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:13.503 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:13.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:13.607 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:13.639 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:13.729 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:39:13.735 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:13.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:13.741 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:39:13.842 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:13.868 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: 13.18 scrub starts 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: 13.18 scrub ok 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: 13.19 scrub starts 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2764388858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3877313539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:13.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:13.915 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:13.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:14.021 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:14.028 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:39:14.048 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:14.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: 13.18 scrub starts 2023-10-09T06:39:14.053 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: 13.18 scrub ok 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: 13.19 scrub starts 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2764388858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3877313539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: 13.18 scrub starts 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: 13.18 scrub ok 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: 13.19 scrub starts 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2764388858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:14.054 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3877313539' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:14.055 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:39:14.098 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:14.147 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:14.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:14.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:14.281 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:14.590 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:14.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:14.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:14.694 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:14.776 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:14.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:14.893 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 50 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2923/2439 objects degraded (119.844%); 2923/2439 objects misplaced (119.844%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[212750]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 50 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2923/2439 objects degraded (119.844%); 2923/2439 objects misplaced (119.844%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.895 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:14 smithi107 ceph-mon[220921]: failed to encode map e485 with expected crc 2023-10-09T06:39:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:14.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:14.997 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:15.023 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: pgmap v1488: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 50 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 2923/2439 objects degraded (119.844%); 2923/2439 objects misplaced (119.844%); 1.4 MiB/s, 37 objects/s recovering 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: failed to encode map e485 with expected crc 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: osdmap e485: 8 total, 8 up, 8 in 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: failed to encode map e485 with expected crc 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: failed to encode map e485 with expected crc 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: failed to encode map e485 with expected crc 2023-10-09T06:39:15.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:14 smithi150 ceph-mon[194178]: failed to encode map e485 with expected crc 2023-10-09T06:39:15.026 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:15.097 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:15.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:15.200 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:15.230 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:15.289 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:15.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:15.392 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:15.422 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:15.515 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:15.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:15.603 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:15.603 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:15.618 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:15.645 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:15.692 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:15.796 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:15.824 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:15.868 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:15.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:15.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:15 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:39:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:15 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:39:15.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:15 smithi150 ceph-mon[194178]: 13.9 scrub starts 2023-10-09T06:39:15.972 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:15.999 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[212750]: 13.9 scrub starts 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:39:16.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:15 smithi107 ceph-mon[220921]: 13.9 scrub starts 2023-10-09T06:39:16.194 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-10-09T06:39:16.194 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-10-09T06:39:16.278 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:39:16.284 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:16.285 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:39:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:16.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:16.336 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:16.389 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:16.432 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:16.543 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:16.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:16.623 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:16.649 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:16.678 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:16.727 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:16.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:16.820 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:16.820 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:39:16.820 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[212750]: 1.18 scrub ok 2023-10-09T06:39:16.820 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[212750]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 1.6 MiB/s, 74 objects/s recovering 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[220921]: 1.18 scrub ok 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[220921]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 1.6 MiB/s, 74 objects/s recovering 2023-10-09T06:39:16.821 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:16.832 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:16.848 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:16.872 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:17.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:16 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:17.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:16 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:39:17.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:16 smithi150 ceph-mon[194178]: 1.18 scrub ok 2023-10-09T06:39:17.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:16 smithi150 ceph-mon[194178]: pgmap v1490: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 1.6 MiB/s, 74 objects/s recovering 2023-10-09T06:39:17.000 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:17.029 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:17.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:17.132 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:17.245 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:17.848 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2859/861 objects degraded (332.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:18.070 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:18.070 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:18.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2859/861 objects degraded (332.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2859/861 objects degraded (332.056%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:18.226 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:39:18.545 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:18.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:18.649 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:18.654 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:39:18.678 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:18.696 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:18.723 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:18.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:18.830 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:18.861 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:18.903 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:18.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:18 smithi150 ceph-mon[194178]: pgmap v1491: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 0 B/s, 33 objects/s recovering 2023-10-09T06:39:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:18.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:19.009 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:19.033 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:19.038 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:18 smithi107 ceph-mon[220921]: pgmap v1491: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 0 B/s, 33 objects/s recovering 2023-10-09T06:39:19.038 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:18 smithi107 ceph-mon[212750]: pgmap v1491: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 17 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 2859/861 objects degraded (332.056%); 2859/861 objects misplaced (332.056%); 0 B/s, 33 objects/s recovering 2023-10-09T06:39:19.044 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:19.246 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:19.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:19.349 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:19.377 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:19.420 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:19.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:19.524 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:19.554 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:19.716 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:19.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:19.820 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:19.847 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:19 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:19.847 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:19 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:19.850 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:19.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:19 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:19.935 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:19.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:19.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:20.040 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:20.070 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:20.114 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:20.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:20.218 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:20.246 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:20.290 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:20.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:20.394 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:20.421 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:20.528 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:20.528 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:20.681 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:20.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:20.770 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:39:20.785 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:20.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:20 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:20.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:20 smithi150 ceph-mon[194178]: 13.19 scrub starts 2023-10-09T06:39:20.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:20 smithi150 ceph-mon[194178]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:20.810 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:20.813 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:20.856 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:20.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:20.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:20.948 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:20.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[212750]: 13.19 scrub starts 2023-10-09T06:39:20.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[212750]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:20.952 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:20.952 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:20.953 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[220921]: 13.19 scrub starts 2023-10-09T06:39:20.953 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[220921]: pgmap v1492: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:20.953 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:20.959 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:20.987 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:21.082 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:21.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:21.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:21.153 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:39:21.186 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:21.221 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:21.499 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: failed to encode map e486 with expected crc 2023-10-09T06:39:21.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:21 smithi107 ceph-mon[220921]: failed to encode map e486 with expected crc 2023-10-09T06:39:22.030 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:39:22.030 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: failed to encode map e486 with expected crc 2023-10-09T06:39:22.031 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: failed to encode map e486 with expected crc 2023-10-09T06:39:22.031 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1537026236' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:22.031 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: osdmap e486: 8 total, 8 up, 8 in 2023-10-09T06:39:22.031 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: failed to encode map e486 with expected crc 2023-10-09T06:39:22.031 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:21 smithi150 ceph-mon[194178]: failed to encode map e486 with expected crc 2023-10-09T06:39:22.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:22] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-10-09T06:39:22.780 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:22.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:22.886 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:22.894 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:22.895 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:22.936 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:23.042 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:23.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[212750]: failed to encode map e486 with expected crc 2023-10-09T06:39:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[212750]: pgmap v1494: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 99 objects/s recovering 2023-10-09T06:39:23.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2328/552 objects degraded (421.739%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:23.044 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[220921]: failed to encode map e486 with expected crc 2023-10-09T06:39:23.044 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[220921]: pgmap v1494: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 99 objects/s recovering 2023-10-09T06:39:23.044 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2328/552 objects degraded (421.739%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:23.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:23.055 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:22 smithi150 ceph-mon[194178]: failed to encode map e486 with expected crc 2023-10-09T06:39:23.057 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:22 smithi150 ceph-mon[194178]: pgmap v1494: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 80 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 99 objects/s recovering 2023-10-09T06:39:23.057 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2328/552 objects degraded (421.739%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:23.146 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:23.179 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:23.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:23.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:23.481 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:23.486 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:39:23.510 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:23.829 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:24.037 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:23 smithi150 ceph-mon[194178]: 13.1 deep-scrub starts 2023-10-09T06:39:24.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:23 smithi107 ceph-mon[212750]: 13.1 deep-scrub starts 2023-10-09T06:39:24.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:23 smithi107 ceph-mon[220921]: 13.1 deep-scrub starts 2023-10-09T06:39:24.783 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:24.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:24.869 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:24.887 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:24.918 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:25.069 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:25.071 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:24 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:25.071 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:24 smithi150 ceph-mon[194178]: 13.9 scrub starts 2023-10-09T06:39:25.071 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:24 smithi150 ceph-mon[194178]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:25.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[212750]: 13.9 scrub starts 2023-10-09T06:39:25.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[212750]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:25.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[220921]: 13.9 scrub starts 2023-10-09T06:39:25.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:24 smithi107 ceph-mon[220921]: pgmap v1495: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 2328/552 objects degraded (421.739%); 2328/552 objects misplaced (421.739%); 0 B/s, 81 objects/s recovering 2023-10-09T06:39:25.173 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:25.207 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:25.375 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:25.375 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:25.623 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:25.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:25.730 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:25.762 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:25.808 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_8' removed 2023-10-09T06:39:25.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:39:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[212750]: 13.1 scrub starts 2023-10-09T06:39:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2703915743' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[220921]: 13.1 scrub starts 2023-10-09T06:39:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2703915743' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:25 smithi150 ceph-mon[194178]: 13.1 scrub starts 2023-10-09T06:39:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2703915743' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:39:26.287 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:26.788 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:39:26.894 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:39:26.920 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:26.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:26.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:27.025 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 1527/552 objects degraded (276.630%); 1527/552 objects misplaced (276.630%); 0 B/s, 133 objects/s recovering 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1734553960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 1527/552 objects degraded (276.630%); 1527/552 objects misplaced (276.630%); 0 B/s, 133 objects/s recovering 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1734553960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.050 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:26 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.051 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: pgmap v1496: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 11 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 1527/552 objects degraded (276.630%); 1527/552 objects misplaced (276.630%); 0 B/s, 133 objects/s recovering 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: osdmap e487: 8 total, 8 up, 8 in 2023-10-09T06:39:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1734553960' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:26 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:39:27.758 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:27.758 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: failed to encode map e487 with expected crc 2023-10-09T06:39:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: failed to encode map e488 with expected crc 2023-10-09T06:39:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:39:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:39:27.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: failed to encode map e488 with expected crc 2023-10-09T06:39:27.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: failed to encode map e488 with expected crc 2023-10-09T06:39:27.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: 1.1b deep-scrub starts 2023-10-09T06:39:27.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: 1.1b deep-scrub ok 2023-10-09T06:39:27.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: failed to encode map e487 with expected crc 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: 1.1b deep-scrub starts 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: 1.1b deep-scrub ok 2023-10-09T06:39:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: failed to encode map e487 with expected crc 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: osdmap e488: 8 total, 8 up, 8 in 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: failed to encode map e488 with expected crc 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: 1.1b deep-scrub starts 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: 1.1b deep-scrub ok 2023-10-09T06:39:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:39:28.238 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:28.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:28.309 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T06:39:28.309 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T06:39:28.310 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-10-09T06:39:28.310 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-10-09T06:39:28.342 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:28.365 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:39:28.374 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:28.421 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:28.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:28.430 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:39:28.431 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:39:28.488 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:28.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:28.554 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:28.603 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:28.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:28.708 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:28.726 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:28.737 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:28.884 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:28.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:28.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:28 smithi150 ceph-mon[194178]: 1.8 deep-scrub starts 2023-10-09T06:39:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:28 smithi150 ceph-mon[194178]: 1.8 deep-scrub ok 2023-10-09T06:39:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:28 smithi150 ceph-mon[194178]: pgmap v1499: 103 pgs: 103 active+clean; 608 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:28 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:39:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:28 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:39:28.939 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:28.991 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:28.997 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:39:29.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[212750]: 1.8 deep-scrub starts 2023-10-09T06:39:29.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[212750]: 1.8 deep-scrub ok 2023-10-09T06:39:29.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[212750]: pgmap v1499: 103 pgs: 103 active+clean; 608 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:29.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:39:29.016 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:39:29.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[220921]: 1.8 deep-scrub starts 2023-10-09T06:39:29.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[220921]: 1.8 deep-scrub ok 2023-10-09T06:39:29.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[220921]: pgmap v1499: 103 pgs: 103 active+clean; 608 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:29.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1527/552 objects degraded (276.630%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:39:29.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:28 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:39:29.019 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:29.020 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:39:29.040 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:39:29.069 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:29.070 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:39:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:29.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:29.090 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:39:29.109 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:39:29.129 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:39:29.148 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.149 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:39:29.174 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:29.205 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:29.248 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:29.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:29.352 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:29.378 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:29.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:29.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:29.444 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:29.525 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:29.552 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:29.754 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:29.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:29.861 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:29.887 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2026988096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:29.887 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2026988096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:29.890 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:29.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2026988096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:30.194 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:30.194 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:39:30.194 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:39:30.194 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:30.827 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:39:30.882 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:30.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:30.900 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[212750]: pgmap v1500: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[212750]: 1.17 scrub starts 2023-10-09T06:39:30.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[212750]: 1.17 scrub ok 2023-10-09T06:39:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[220921]: pgmap v1500: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[220921]: 1.17 scrub starts 2023-10-09T06:39:30.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:30 smithi107 ceph-mon[220921]: 1.17 scrub ok 2023-10-09T06:39:30.986 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:31.015 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:30 smithi150 ceph-mon[194178]: pgmap v1500: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:30 smithi150 ceph-mon[194178]: 1.17 scrub starts 2023-10-09T06:39:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:30 smithi150 ceph-mon[194178]: 1.17 scrub ok 2023-10-09T06:39:31.177 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:32.074 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:32.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:32.182 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:32.217 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:32.231 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:32] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2023-10-09T06:39:32.269 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:32.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:32.372 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:32.402 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:32.548 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:39:32.551 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:32.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:32.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:32.608 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:32.608 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:39:32.608 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:39:32.609 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:39:32.609 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: 1.1 scrub starts 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: 1.1 scrub ok 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: pgmap v1501: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:32.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:39:32.654 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:32.685 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:32.830 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:32.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: pgmap v1501: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:32.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: 1.1 scrub starts 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: 1.1 scrub ok 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: pgmap v1501: 103 pgs: 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:32.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:39:32.919 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:39:32.919 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_9' created 2023-10-09T06:39:32.934 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:32.943 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:32.960 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:33.071 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:39:33.073 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:39:33.198 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:33.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:33.259 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:39:33.302 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:33.332 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:33.599 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:39:33.780 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:33.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:33.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/86431694' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[212750]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/86431694' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:33 smithi107 ceph-mon[220921]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/86431694' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/230482546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T06:39:33.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: osdmap e489: 8 total, 8 up, 8 in 2023-10-09T06:39:33.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: failed to encode map e489 with expected crc 2023-10-09T06:39:33.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:33 smithi150 ceph-mon[194178]: from='client.47006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:33.912 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:33.961 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:33.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:34.064 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:34.093 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:34.431 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1502,"stamp":"2023-10-09T06:39:32.907467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":319,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9090936,"kb_used_data":383032,"kb_used_omap":20,"kb_used_meta":8707819,"kb_avail":740870280,"statfs":{"total":767960285184,"available":758651166720,"internally_reserved":0,"allocated":392224768,"data_stored":379416794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8916807310},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-10545363,"num_objects":-160,"num_object_clones":0,"num_object_copies":-480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1527,"num_objects_misplaced":-1527,"num_objects_unfound":0,"num_objects_dirty":-160,"num_whiteouts":0,"num_read":-23,"num_read_kb":-42,"num_write":-175703,"num_write_kb":-4725422,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8375,"num_bytes_recovered":-378539038,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.373598"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":782,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106768+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:39:26.106768+0000","last_peered":"2023-10-09T06:39:26.106768+0000","last_clean":"2023-10-09T06:39:26.106768+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:39:26.106768+0000","last_undegraded":"2023-10-09T06:39:26.106768+0000","last_fullsized":"2023-10-09T06:39:26.106768+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":467,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789540+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:39:25.789540+0000","last_peered":"2023-10-09T06:39:25.789540+0000","last_clean":"2023-10-09T06:39:25.789540+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:39:25.789540+0000","last_undegraded":"2023-10-09T06:39:25.789540+0000","last_fullsized":"2023-10-09T06:39:25.789540+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":189,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786399+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:39:25.786399+0000","last_peered":"2023-10-09T06:39:25.786399+0000","last_clean":"2023-10-09T06:39:25.786399+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:39:25.786399+0000","last_undegraded":"2023-10-09T06:39:25.786399+0000","last_fullsized":"2023-10-09T06:39:25.786399+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":743,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.204397+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:39:27.204397+0000","last_peered":"2023-10-09T06:39:27.204397+0000","last_clean":"2023-10-09T06:39:27.204397+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:39:27.204397+0000","last_undegraded":"2023-10-09T06:39:27.204397+0000","last_fullsized":"2023-10-09T06:39:27.204397+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":483,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.814612+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:39:25.814612+0000","last_peered":"2023-10-09T06:39:25.814612+0000","last_clean":"2023-10-09T06:39:25.814612+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:39:25.814612+0000","last_undegraded":"2023-10-09T06:39:25.814612+0000","last_fullsized":"2023-10-09T06:39:25.814612+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":488,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784122+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:39:25.784122+0000","last_peered":"2023-10-09T06:39:25.784122+0000","last_clean":"2023-10-09T06:39:25.784122+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:39:25.784122+0000","last_undegraded":"2023-10-09T06:39:25.784122+0000","last_fullsized":"2023-10-09T06:39:25.784122+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":735,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789462+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:39:25.789462+0000","last_peered":"2023-10-09T06:39:25.789462+0000","last_clean":"2023-10-09T06:39:25.789462+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:39:25.789462+0000","last_undegraded":"2023-10-09T06:39:25.789462+0000","last_fullsized":"2023-10-09T06:39:25.789462+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":207,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785835+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:39:25.785835+0000","last_peered":"2023-10-09T06:39:25.785835+0000","last_clean":"2023-10-09T06:39:25.785835+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:39:25.785835+0000","last_undegraded":"2023-10-09T06:39:25.785835+0000","last_fullsized":"2023-10-09T06:39:25.785835+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":477,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786391+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:39:25.786391+0000","last_peered":"2023-10-09T06:39:25.786391+0000","last_clean":"2023-10-09T06:39:25.786391+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:39:25.786391+0000","last_undegraded":"2023-10-09T06:39:25.786391+0000","last_fullsized":"2023-10-09T06:39:25.786391+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":596,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107048+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:39:26.107048+0000","last_peered":"2023-10-09T06:39:26.107048+0000","last_clean":"2023-10-09T06:39:26.107048+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:39:26.107048+0000","last_undegraded":"2023-10-09T06:39:26.107048+0000","last_fullsized":"2023-10-09T06:39:26.107048+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":163,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784613+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:39:25.784613+0000","last_peered":"2023-10-09T06:39:25.784613+0000","last_clean":"2023-10-09T06:39:25.784613+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:39:25.784613+0000","last_undegraded":"2023-10-09T06:39:25.784613+0000","last_fullsized":"2023-10-09T06:39:25.784613+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":190,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785941+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:39:25.785941+0000","last_peered":"2023-10-09T06:39:25.785941+0000","last_clean":"2023-10-09T06:39:25.785941+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:39:25.785941+0000","last_undegraded":"2023-10-09T06:39:25.785941+0000","last_fullsized":"2023-10-09T06:39:25.785941+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":633,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107369+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:39:26.107369+0000","last_peered":"2023-10-09T06:39:26.107369+0000","last_clean":"2023-10-09T06:39:26.107369+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:39:26.107369+0000","last_undegraded":"2023-10-09T06:39:26.107369+0000","last_fullsized":"2023-10-09T06:39:26.107369+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":175,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.304783+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:39:26.304783+0000","last_peered":"2023-10-09T06:39:26.304783+0000","last_clean":"2023-10-09T06:39:26.304783+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:39:26.304783+0000","last_undegraded":"2023-10-09T06:39:26.304783+0000","last_fullsized":"2023-10-09T06:39:26.304783+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":190,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786393+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:39:25.786393+0000","last_peered":"2023-10-09T06:39:25.786393+0000","last_clean":"2023-10-09T06:39:25.786393+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:39:25.786393+0000","last_undegraded":"2023-10-09T06:39:25.786393+0000","last_fullsized":"2023-10-09T06:39:25.786393+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":603,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784358+0000","last_change":"2023-10-09T06:37:11.071873+0000","last_active":"2023-10-09T06:39:25.784358+0000","last_peered":"2023-10-09T06:39:25.784358+0000","last_clean":"2023-10-09T06:39:25.784358+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:39:25.784358+0000","last_undegraded":"2023-10-09T06:39:25.784358+0000","last_fullsized":"2023-10-09T06:39:25.784358+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:35:31.210137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:35:31.210137+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010104116,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":369,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108478+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:39:26.108478+0000","last_peered":"2023-10-09T06:39:26.108478+0000","last_clean":"2023-10-09T06:39:26.108478+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:39:26.108478+0000","last_undegraded":"2023-10-09T06:39:26.108478+0000","last_fullsized":"2023-10-09T06:39:26.108478+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":425,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.820902+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:39:25.820902+0000","last_peered":"2023-10-09T06:39:25.820902+0000","last_clean":"2023-10-09T06:39:25.820902+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:39:25.820902+0000","last_undegraded":"2023-10-09T06:39:25.820902+0000","last_fullsized":"2023-10-09T06:39:25.820902+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":596,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.822312+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:39:25.822312+0000","last_peered":"2023-10-09T06:39:25.822312+0000","last_clean":"2023-10-09T06:39:25.822312+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:39:25.822312+0000","last_undegraded":"2023-10-09T06:39:25.822312+0000","last_fullsized":"2023-10-09T06:39:25.822312+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":494,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787822+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:39:25.787822+0000","last_peered":"2023-10-09T06:39:25.787822+0000","last_clean":"2023-10-09T06:39:25.787822+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:39:25.787822+0000","last_undegraded":"2023-10-09T06:39:25.787822+0000","last_fullsized":"2023-10-09T06:39:25.787822+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790700+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:39:25.790700+0000","last_peered":"2023-10-09T06:39:25.790700+0000","last_clean":"2023-10-09T06:39:25.790700+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:39:25.790700+0000","last_undegraded":"2023-10-09T06:39:25.790700+0000","last_fullsized":"2023-10-09T06:39:25.790700+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":761,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.783254+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:39:25.783254+0000","last_peered":"2023-10-09T06:39:25.783254+0000","last_clean":"2023-10-09T06:39:25.783254+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:39:25.783254+0000","last_undegraded":"2023-10-09T06:39:25.783254+0000","last_fullsized":"2023-10-09T06:39:25.783254+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":337,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107191+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:39:26.107191+0000","last_peered":"2023-10-09T06:39:26.107191+0000","last_clean":"2023-10-09T06:39:26.107191+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:39:26.107191+0000","last_undegraded":"2023-10-09T06:39:26.107191+0000","last_fullsized":"2023-10-09T06:39:26.107191+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786265+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:39:25.786265+0000","last_peered":"2023-10-09T06:39:25.786265+0000","last_clean":"2023-10-09T06:39:25.786265+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:39:25.786265+0000","last_undegraded":"2023-10-09T06:39:25.786265+0000","last_fullsized":"2023-10-09T06:39:25.786265+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":760,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106864+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:39:26.106864+0000","last_peered":"2023-10-09T06:39:26.106864+0000","last_clean":"2023-10-09T06:39:26.106864+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:39:26.106864+0000","last_undegraded":"2023-10-09T06:39:26.106864+0000","last_fullsized":"2023-10-09T06:39:26.106864+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":486,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784024+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:39:25.784024+0000","last_peered":"2023-10-09T06:39:25.784024+0000","last_clean":"2023-10-09T06:39:25.784024+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:39:25.784024+0000","last_undegraded":"2023-10-09T06:39:25.784024+0000","last_fullsized":"2023-10-09T06:39:25.784024+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":308,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108308+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:39:26.108308+0000","last_peered":"2023-10-09T06:39:26.108308+0000","last_clean":"2023-10-09T06:39:26.108308+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:39:26.108308+0000","last_undegraded":"2023-10-09T06:39:26.108308+0000","last_fullsized":"2023-10-09T06:39:26.108308+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":815,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.814707+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:39:25.814707+0000","last_peered":"2023-10-09T06:39:25.814707+0000","last_clean":"2023-10-09T06:39:25.814707+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:39:25.814707+0000","last_undegraded":"2023-10-09T06:39:25.814707+0000","last_fullsized":"2023-10-09T06:39:25.814707+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":331,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106922+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:39:26.106922+0000","last_peered":"2023-10-09T06:39:26.106922+0000","last_clean":"2023-10-09T06:39:26.106922+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:39:26.106922+0000","last_undegraded":"2023-10-09T06:39:26.106922+0000","last_fullsized":"2023-10-09T06:39:26.106922+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":163,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.783879+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:39:25.783879+0000","last_peered":"2023-10-09T06:39:25.783879+0000","last_clean":"2023-10-09T06:39:25.783879+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:39:25.783879+0000","last_undegraded":"2023-10-09T06:39:25.783879+0000","last_fullsized":"2023-10-09T06:39:25.783879+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":602,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.304275+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:39:26.304275+0000","last_peered":"2023-10-09T06:39:26.304275+0000","last_clean":"2023-10-09T06:39:26.304275+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:39:26.304275+0000","last_undegraded":"2023-10-09T06:39:26.304275+0000","last_fullsized":"2023-10-09T06:39:26.304275+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":566,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786321+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:39:25.786321+0000","last_peered":"2023-10-09T06:39:25.786321+0000","last_clean":"2023-10-09T06:39:25.786321+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:39:25.786321+0000","last_undegraded":"2023-10-09T06:39:25.786321+0000","last_fullsized":"2023-10-09T06:39:25.786321+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":189,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789580+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:39:25.789580+0000","last_peered":"2023-10-09T06:39:25.789580+0000","last_clean":"2023-10-09T06:39:25.789580+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:39:25.789580+0000","last_undegraded":"2023-10-09T06:39:25.789580+0000","last_fullsized":"2023-10-09T06:39:25.789580+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":615,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107153+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:39:26.107153+0000","last_peered":"2023-10-09T06:39:26.107153+0000","last_clean":"2023-10-09T06:39:26.107153+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:39:26.107153+0000","last_undegraded":"2023-10-09T06:39:26.107153+0000","last_fullsized":"2023-10-09T06:39:26.107153+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":373,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.111633+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:39:26.111633+0000","last_peered":"2023-10-09T06:39:26.111633+0000","last_clean":"2023-10-09T06:39:26.111633+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:39:26.111633+0000","last_undegraded":"2023-10-09T06:39:26.111633+0000","last_fullsized":"2023-10-09T06:39:26.111633+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107604+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:39:26.107604+0000","last_peered":"2023-10-09T06:39:26.107604+0000","last_clean":"2023-10-09T06:39:26.107604+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:39:26.107604+0000","last_undegraded":"2023-10-09T06:39:26.107604+0000","last_fullsized":"2023-10-09T06:39:26.107604+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":874,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786038+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:39:25.786038+0000","last_peered":"2023-10-09T06:39:25.786038+0000","last_clean":"2023-10-09T06:39:25.786038+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:39:25.786038+0000","last_undegraded":"2023-10-09T06:39:25.786038+0000","last_fullsized":"2023-10-09T06:39:25.786038+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":335,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.109056+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:39:26.109056+0000","last_peered":"2023-10-09T06:39:26.109056+0000","last_clean":"2023-10-09T06:39:26.109056+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:39:26.109056+0000","last_undegraded":"2023-10-09T06:39:26.109056+0000","last_fullsized":"2023-10-09T06:39:26.109056+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788802+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:39:25.788802+0000","last_peered":"2023-10-09T06:39:25.788802+0000","last_clean":"2023-10-09T06:39:25.788802+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:39:25.788802+0000","last_undegraded":"2023-10-09T06:39:25.788802+0000","last_fullsized":"2023-10-09T06:39:25.788802+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":789,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785745+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:39:25.785745+0000","last_peered":"2023-10-09T06:39:25.785745+0000","last_clean":"2023-10-09T06:39:25.785745+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:39:25.785745+0000","last_undegraded":"2023-10-09T06:39:25.785745+0000","last_fullsized":"2023-10-09T06:39:25.785745+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":391,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107472+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:39:26.107472+0000","last_peered":"2023-10-09T06:39:26.107472+0000","last_clean":"2023-10-09T06:39:26.107472+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:39:26.107472+0000","last_undegraded":"2023-10-09T06:39:26.107472+0000","last_fullsized":"2023-10-09T06:39:26.107472+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":464,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784350+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:39:25.784350+0000","last_peered":"2023-10-09T06:39:25.784350+0000","last_clean":"2023-10-09T06:39:25.784350+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:39:25.784350+0000","last_undegraded":"2023-10-09T06:39:25.784350+0000","last_fullsized":"2023-10-09T06:39:25.784350+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":612,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789702+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:39:25.789702+0000","last_peered":"2023-10-09T06:39:25.789702+0000","last_clean":"2023-10-09T06:39:25.789702+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:39:25.789702+0000","last_undegraded":"2023-10-09T06:39:25.789702+0000","last_fullsized":"2023-10-09T06:39:25.789702+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":58,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786341+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:39:25.786341+0000","last_peered":"2023-10-09T06:39:25.786341+0000","last_clean":"2023-10-09T06:39:25.786341+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:39:25.786341+0000","last_undegraded":"2023-10-09T06:39:25.786341+0000","last_fullsized":"2023-10-09T06:39:25.786341+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":450,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785454+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:39:25.785454+0000","last_peered":"2023-10-09T06:39:25.785454+0000","last_clean":"2023-10-09T06:39:25.785454+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:39:25.785454+0000","last_undegraded":"2023-10-09T06:39:25.785454+0000","last_fullsized":"2023-10-09T06:39:25.785454+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":840,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106842+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:39:26.106842+0000","last_peered":"2023-10-09T06:39:26.106842+0000","last_clean":"2023-10-09T06:39:26.106842+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:39:26.106842+0000","last_undegraded":"2023-10-09T06:39:26.106842+0000","last_fullsized":"2023-10-09T06:39:26.106842+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":216,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788586+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:39:25.788586+0000","last_peered":"2023-10-09T06:39:25.788586+0000","last_clean":"2023-10-09T06:39:25.788586+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:39:25.788586+0000","last_undegraded":"2023-10-09T06:39:25.788586+0000","last_fullsized":"2023-10-09T06:39:25.788586+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":468,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787627+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:39:25.787627+0000","last_peered":"2023-10-09T06:39:25.787627+0000","last_clean":"2023-10-09T06:39:25.787627+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:39:25.787627+0000","last_undegraded":"2023-10-09T06:39:25.787627+0000","last_fullsized":"2023-10-09T06:39:25.787627+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":522,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873656+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:39:26.873656+0000","last_peered":"2023-10-09T06:39:26.873656+0000","last_clean":"2023-10-09T06:39:26.873656+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:39:26.873656+0000","last_undegraded":"2023-10-09T06:39:26.873656+0000","last_fullsized":"2023-10-09T06:39:26.873656+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":445,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784266+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:39:25.784266+0000","last_peered":"2023-10-09T06:39:25.784266+0000","last_clean":"2023-10-09T06:39:25.784266+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:39:25.784266+0000","last_undegraded":"2023-10-09T06:39:25.784266+0000","last_fullsized":"2023-10-09T06:39:25.784266+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":465,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.791045+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:39:25.791045+0000","last_peered":"2023-10-09T06:39:25.791045+0000","last_clean":"2023-10-09T06:39:25.791045+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:39:25.791045+0000","last_undegraded":"2023-10-09T06:39:25.791045+0000","last_fullsized":"2023-10-09T06:39:25.791045+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":824,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.814778+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:39:25.814778+0000","last_peered":"2023-10-09T06:39:25.814778+0000","last_clean":"2023-10-09T06:39:25.814778+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:39:25.814778+0000","last_undegraded":"2023-10-09T06:39:25.814778+0000","last_fullsized":"2023-10-09T06:39:25.814778+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":469,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788574+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:39:25.788574+0000","last_peered":"2023-10-09T06:39:25.788574+0000","last_clean":"2023-10-09T06:39:25.788574+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:39:25.788574+0000","last_undegraded":"2023-10-09T06:39:25.788574+0000","last_fullsized":"2023-10-09T06:39:25.788574+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":448,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.792193+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:39:25.792193+0000","last_peered":"2023-10-09T06:39:25.792193+0000","last_clean":"2023-10-09T06:39:25.792193+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:39:25.792193+0000","last_undegraded":"2023-10-09T06:39:25.792193+0000","last_fullsized":"2023-10-09T06:39:25.792193+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":450,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784215+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:39:25.784215+0000","last_peered":"2023-10-09T06:39:25.784215+0000","last_clean":"2023-10-09T06:39:25.784215+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:39:25.784215+0000","last_undegraded":"2023-10-09T06:39:25.784215+0000","last_fullsized":"2023-10-09T06:39:25.784215+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":190,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785922+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:39:25.785922+0000","last_peered":"2023-10-09T06:39:25.785922+0000","last_clean":"2023-10-09T06:39:25.785922+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:39:25.785922+0000","last_undegraded":"2023-10-09T06:39:25.785922+0000","last_fullsized":"2023-10-09T06:39:25.785922+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":443,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.822777+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:39:25.822777+0000","last_peered":"2023-10-09T06:39:25.822777+0000","last_clean":"2023-10-09T06:39:25.822777+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:39:25.822777+0000","last_undegraded":"2023-10-09T06:39:25.822777+0000","last_fullsized":"2023-10-09T06:39:25.822777+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":780,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786506+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:39:25.786506+0000","last_peered":"2023-10-09T06:39:25.786506+0000","last_clean":"2023-10-09T06:39:25.786506+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:39:25.786506+0000","last_undegraded":"2023-10-09T06:39:25.786506+0000","last_fullsized":"2023-10-09T06:39:25.786506+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":495,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.815537+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:39:25.815537+0000","last_peered":"2023-10-09T06:39:25.815537+0000","last_clean":"2023-10-09T06:39:25.815537+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:39:25.815537+0000","last_undegraded":"2023-10-09T06:39:25.815537+0000","last_fullsized":"2023-10-09T06:39:25.815537+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":58,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786323+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:39:25.786323+0000","last_peered":"2023-10-09T06:39:25.786323+0000","last_clean":"2023-10-09T06:39:25.786323+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:39:25.786323+0000","last_undegraded":"2023-10-09T06:39:25.786323+0000","last_fullsized":"2023-10-09T06:39:25.786323+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":429,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.819459+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:39:25.819459+0000","last_peered":"2023-10-09T06:39:25.819459+0000","last_clean":"2023-10-09T06:39:25.819459+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:39:25.819459+0000","last_undegraded":"2023-10-09T06:39:25.819459+0000","last_fullsized":"2023-10-09T06:39:25.819459+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":655,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.783978+0000","last_change":"2023-10-09T06:37:08.296784+0000","last_active":"2023-10-09T06:39:25.783978+0000","last_peered":"2023-10-09T06:39:25.783978+0000","last_clean":"2023-10-09T06:39:25.783978+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:39:25.783978+0000","last_undegraded":"2023-10-09T06:39:25.783978+0000","last_fullsized":"2023-10-09T06:39:25.783978+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.169826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.169826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":219,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.820763+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:39:25.820763+0000","last_peered":"2023-10-09T06:39:25.820763+0000","last_clean":"2023-10-09T06:39:25.820763+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:39:25.820763+0000","last_undegraded":"2023-10-09T06:39:25.820763+0000","last_fullsized":"2023-10-09T06:39:25.820763+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":469,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790583+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:39:25.790583+0000","last_peered":"2023-10-09T06:39:25.790583+0000","last_clean":"2023-10-09T06:39:25.790583+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:39:25.790583+0000","last_undegraded":"2023-10-09T06:39:25.790583+0000","last_fullsized":"2023-10-09T06:39:25.790583+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":459,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.815479+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:39:25.815479+0000","last_peered":"2023-10-09T06:39:25.815479+0000","last_clean":"2023-10-09T06:39:25.815479+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:39:25.815479+0000","last_undegraded":"2023-10-09T06:39:25.815479+0000","last_fullsized":"2023-10-09T06:39:25.815479+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":830,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106972+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:39:26.106972+0000","last_peered":"2023-10-09T06:39:26.106972+0000","last_clean":"2023-10-09T06:39:26.106972+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:39:26.106972+0000","last_undegraded":"2023-10-09T06:39:26.106972+0000","last_fullsized":"2023-10-09T06:39:26.106972+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":481,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786135+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:39:25.786135+0000","last_peered":"2023-10-09T06:39:25.786135+0000","last_clean":"2023-10-09T06:39:25.786135+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:39:25.786135+0000","last_undegraded":"2023-10-09T06:39:25.786135+0000","last_fullsized":"2023-10-09T06:39:25.786135+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":468,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.820579+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:39:25.820579+0000","last_peered":"2023-10-09T06:39:25.820579+0000","last_clean":"2023-10-09T06:39:25.820579+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:39:25.820579+0000","last_undegraded":"2023-10-09T06:39:25.820579+0000","last_fullsized":"2023-10-09T06:39:25.820579+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":490,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.822823+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:39:25.822823+0000","last_peered":"2023-10-09T06:39:25.822823+0000","last_clean":"2023-10-09T06:39:25.822823+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:39:25.822823+0000","last_undegraded":"2023-10-09T06:39:25.822823+0000","last_fullsized":"2023-10-09T06:39:25.822823+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":297,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107101+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:39:26.107101+0000","last_peered":"2023-10-09T06:39:26.107101+0000","last_clean":"2023-10-09T06:39:26.107101+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:39:26.107101+0000","last_undegraded":"2023-10-09T06:39:26.107101+0000","last_fullsized":"2023-10-09T06:39:26.107101+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":811,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107654+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:39:26.107654+0000","last_peered":"2023-10-09T06:39:26.107654+0000","last_clean":"2023-10-09T06:39:26.107654+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:39:26.107654+0000","last_undegraded":"2023-10-09T06:39:26.107654+0000","last_fullsized":"2023-10-09T06:39:26.107654+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":199,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.791274+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:39:25.791274+0000","last_peered":"2023-10-09T06:39:25.791274+0000","last_clean":"2023-10-09T06:39:25.791274+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:39:25.791274+0000","last_undegraded":"2023-10-09T06:39:25.791274+0000","last_fullsized":"2023-10-09T06:39:25.791274+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":450,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784610+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:39:25.784610+0000","last_peered":"2023-10-09T06:39:25.784610+0000","last_clean":"2023-10-09T06:39:25.784610+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:39:25.784610+0000","last_undegraded":"2023-10-09T06:39:25.784610+0000","last_fullsized":"2023-10-09T06:39:25.784610+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":798,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.814850+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:39:25.814850+0000","last_peered":"2023-10-09T06:39:25.814850+0000","last_clean":"2023-10-09T06:39:25.814850+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:39:25.814850+0000","last_undegraded":"2023-10-09T06:39:25.814850+0000","last_fullsized":"2023-10-09T06:39:25.814850+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":351,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107002+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:39:26.107002+0000","last_peered":"2023-10-09T06:39:26.107002+0000","last_clean":"2023-10-09T06:39:26.107002+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:39:26.107002+0000","last_undegraded":"2023-10-09T06:39:26.107002+0000","last_fullsized":"2023-10-09T06:39:26.107002+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":163,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.783906+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:39:25.783906+0000","last_peered":"2023-10-09T06:39:25.783906+0000","last_clean":"2023-10-09T06:39:25.783906+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:39:25.783906+0000","last_undegraded":"2023-10-09T06:39:25.783906+0000","last_fullsized":"2023-10-09T06:39:25.783906+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":474,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784195+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:39:25.784195+0000","last_peered":"2023-10-09T06:39:25.784195+0000","last_clean":"2023-10-09T06:39:25.784195+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:39:25.784195+0000","last_undegraded":"2023-10-09T06:39:25.784195+0000","last_fullsized":"2023-10-09T06:39:25.784195+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":810,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784597+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:39:25.784597+0000","last_peered":"2023-10-09T06:39:25.784597+0000","last_clean":"2023-10-09T06:39:25.784597+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:39:25.784597+0000","last_undegraded":"2023-10-09T06:39:25.784597+0000","last_fullsized":"2023-10-09T06:39:25.784597+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":351,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108387+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:39:26.108387+0000","last_peered":"2023-10-09T06:39:26.108387+0000","last_clean":"2023-10-09T06:39:26.108387+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:39:26.108387+0000","last_undegraded":"2023-10-09T06:39:26.108387+0000","last_fullsized":"2023-10-09T06:39:26.108387+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":485,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.815393+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:39:25.815393+0000","last_peered":"2023-10-09T06:39:25.815393+0000","last_clean":"2023-10-09T06:39:25.815393+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:39:25.815393+0000","last_undegraded":"2023-10-09T06:39:25.815393+0000","last_fullsized":"2023-10-09T06:39:25.815393+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":472,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785683+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:39:25.785683+0000","last_peered":"2023-10-09T06:39:25.785683+0000","last_clean":"2023-10-09T06:39:25.785683+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:39:25.785683+0000","last_undegraded":"2023-10-09T06:39:25.785683+0000","last_fullsized":"2023-10-09T06:39:25.785683+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":811,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788042+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:39:25.788042+0000","last_peered":"2023-10-09T06:39:25.788042+0000","last_clean":"2023-10-09T06:39:25.788042+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:39:25.788042+0000","last_undegraded":"2023-10-09T06:39:25.788042+0000","last_fullsized":"2023-10-09T06:39:25.788042+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":220,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789752+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:39:25.789752+0000","last_peered":"2023-10-09T06:39:25.789752+0000","last_clean":"2023-10-09T06:39:25.789752+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:39:25.789752+0000","last_undegraded":"2023-10-09T06:39:25.789752+0000","last_fullsized":"2023-10-09T06:39:25.789752+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":453,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.822506+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:39:25.822506+0000","last_peered":"2023-10-09T06:39:25.822506+0000","last_clean":"2023-10-09T06:39:25.822506+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:39:25.822506+0000","last_undegraded":"2023-10-09T06:39:25.822506+0000","last_fullsized":"2023-10-09T06:39:25.822506+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":208,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785855+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:39:25.785855+0000","last_peered":"2023-10-09T06:39:25.785855+0000","last_clean":"2023-10-09T06:39:25.785855+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:39:25.785855+0000","last_undegraded":"2023-10-09T06:39:25.785855+0000","last_fullsized":"2023-10-09T06:39:25.785855+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":842,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786428+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:39:25.786428+0000","last_peered":"2023-10-09T06:39:25.786428+0000","last_clean":"2023-10-09T06:39:25.786428+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:39:25.786428+0000","last_undegraded":"2023-10-09T06:39:25.786428+0000","last_fullsized":"2023-10-09T06:39:25.786428+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":472,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786641+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:39:25.786641+0000","last_peered":"2023-10-09T06:39:25.786641+0000","last_clean":"2023-10-09T06:39:25.786641+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:39:25.786641+0000","last_undegraded":"2023-10-09T06:39:25.786641+0000","last_fullsized":"2023-10-09T06:39:25.786641+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":455,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784041+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:39:25.784041+0000","last_peered":"2023-10-09T06:39:25.784041+0000","last_clean":"2023-10-09T06:39:25.784041+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:39:25.784041+0000","last_undegraded":"2023-10-09T06:39:25.784041+0000","last_fullsized":"2023-10-09T06:39:25.784041+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":451,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788696+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:39:25.788696+0000","last_peered":"2023-10-09T06:39:25.788696+0000","last_clean":"2023-10-09T06:39:25.788696+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:39:25.788696+0000","last_undegraded":"2023-10-09T06:39:25.788696+0000","last_fullsized":"2023-10-09T06:39:25.788696+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":768,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785942+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:39:25.785942+0000","last_peered":"2023-10-09T06:39:25.785942+0000","last_clean":"2023-10-09T06:39:25.785942+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:39:25.785942+0000","last_undegraded":"2023-10-09T06:39:25.785942+0000","last_fullsized":"2023-10-09T06:39:25.785942+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":456,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785600+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:39:25.785600+0000","last_peered":"2023-10-09T06:39:25.785600+0000","last_clean":"2023-10-09T06:39:25.785600+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:39:25.785600+0000","last_undegraded":"2023-10-09T06:39:25.785600+0000","last_fullsized":"2023-10-09T06:39:25.785600+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":478,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.815328+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:39:25.815328+0000","last_peered":"2023-10-09T06:39:25.815328+0000","last_clean":"2023-10-09T06:39:25.815328+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:39:25.815328+0000","last_undegraded":"2023-10-09T06:39:25.815328+0000","last_fullsized":"2023-10-09T06:39:25.815328+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":699,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787336+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:39:25.787336+0000","last_peered":"2023-10-09T06:39:25.787336+0000","last_clean":"2023-10-09T06:39:25.787336+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:39:25.787336+0000","last_undegraded":"2023-10-09T06:39:25.787336+0000","last_fullsized":"2023-10-09T06:39:25.787336+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":166,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787719+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:39:25.787719+0000","last_peered":"2023-10-09T06:39:25.787719+0000","last_clean":"2023-10-09T06:39:25.787719+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:39:25.787719+0000","last_undegraded":"2023-10-09T06:39:25.787719+0000","last_fullsized":"2023-10-09T06:39:25.787719+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":495,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790507+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:39:25.790507+0000","last_peered":"2023-10-09T06:39:25.790507+0000","last_clean":"2023-10-09T06:39:25.790507+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:39:25.790507+0000","last_undegraded":"2023-10-09T06:39:25.790507+0000","last_fullsized":"2023-10-09T06:39:25.790507+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":166,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.791135+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:39:25.791135+0000","last_peered":"2023-10-09T06:39:25.791135+0000","last_clean":"2023-10-09T06:39:25.791135+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:39:25.791135+0000","last_undegraded":"2023-10-09T06:39:25.791135+0000","last_fullsized":"2023-10-09T06:39:25.791135+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":505,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789603+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:39:25.789603+0000","last_peered":"2023-10-09T06:39:25.789603+0000","last_clean":"2023-10-09T06:39:25.789603+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:39:25.789603+0000","last_undegraded":"2023-10-09T06:39:25.789603+0000","last_fullsized":"2023-10-09T06:39:25.789603+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":844,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107378+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:39:26.107378+0000","last_peered":"2023-10-09T06:39:26.107378+0000","last_clean":"2023-10-09T06:39:26.107378+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:39:26.107378+0000","last_undegraded":"2023-10-09T06:39:26.107378+0000","last_fullsized":"2023-10-09T06:39:26.107378+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":190,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787508+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:39:25.787508+0000","last_peered":"2023-10-09T06:39:25.787508+0000","last_clean":"2023-10-09T06:39:25.787508+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:39:25.787508+0000","last_undegraded":"2023-10-09T06:39:25.787508+0000","last_fullsized":"2023-10-09T06:39:25.787508+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":207,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786325+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:39:25.786325+0000","last_peered":"2023-10-09T06:39:25.786325+0000","last_clean":"2023-10-09T06:39:25.786325+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:39:25.786325+0000","last_undegraded":"2023-10-09T06:39:25.786325+0000","last_fullsized":"2023-10-09T06:39:25.786325+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":610,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.823087+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:39:25.823087+0000","last_peered":"2023-10-09T06:39:25.823087+0000","last_clean":"2023-10-09T06:39:25.823087+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:39:25.823087+0000","last_undegraded":"2023-10-09T06:39:25.823087+0000","last_fullsized":"2023-10-09T06:39:25.823087+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":485,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784536+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:39:25.784536+0000","last_peered":"2023-10-09T06:39:25.784536+0000","last_clean":"2023-10-09T06:39:25.784536+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:39:25.784536+0000","last_undegraded":"2023-10-09T06:39:25.784536+0000","last_fullsized":"2023-10-09T06:39:25.784536+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":448,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789425+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:39:25.789425+0000","last_peered":"2023-10-09T06:39:25.789425+0000","last_clean":"2023-10-09T06:39:25.789425+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:39:25.789425+0000","last_undegraded":"2023-10-09T06:39:25.789425+0000","last_fullsized":"2023-10-09T06:39:25.789425+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234507,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1513108,"kb_used_data":12108,"kb_used_omap":0,"kb_used_meta":1500992,"kb_avail":92232044,"statfs":{"total":95995035648,"available":94445613056,"internally_reserved":0,"allocated":12398592,"data_stored":11333589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537015808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":1.4510000000000001,"15min":4.3040000000000003},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.218},"max":{"1min":8.0050000000000008,"5min":8.8819999999999997,"15min":215.97800000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.4870000000000001,"15min":4.3499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":10.449999999999999,"5min":10.449999999999999,"15min":215.577},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":3.0350000000000001,"5min":6.9569999999999999,"15min":5.3869999999999996},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23200000000000001},"max":{"1min":9.7919999999999998,"5min":208.626,"15min":215.61699999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":3.1589999999999998,"5min":5.0759999999999996,"15min":4.5380000000000003},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":9.6679999999999993,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":20.891999999999999,"5min":9.2080000000000002,"15min":7.4980000000000002},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":228.24000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":21,"5min":9.1869999999999994,"15min":7.5990000000000002},"min":{"1min":0.33100000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":228.291,"5min":228.291,"15min":228.291},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":10.587999999999999,"5min":5.6100000000000003,"15min":5.4560000000000004},"min":{"1min":0.33300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":88.897999999999996,"5min":175.40600000000001,"15min":215.922},"last":0.59199999999999997},{"interface":"front","average":{"1min":11.07,"5min":5.6189999999999998,"15min":5.4779999999999998},"min":{"1min":0.46400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":88.926000000000002,"5min":175.30099999999999,"15min":215.94900000000001},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.4489999999999998,"15min":2.8239999999999998},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.0429999999999993,"5min":73.688999999999993,"15min":218.446},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.1640000000000001,"15min":2.6360000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":3.7770000000000001,"5min":71.831000000000003,"15min":216.172},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":3.2210000000000001,"15min":2.7530000000000001},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":12.499000000000001,"5min":71.116,"15min":216.37200000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.712,"5min":2.9809999999999999,"15min":2.7530000000000001},"min":{"1min":0.41499999999999998,"5min":0.312,"15min":0.28999999999999998},"max":{"1min":8.1839999999999993,"5min":70.984999999999999,"15min":218.15899999999999},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":3.2690000000000001,"15min":2.9409999999999998},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":8.3409999999999993,"5min":71.087999999999994,"15min":218.19999999999999},"last":0.623},{"interface":"front","average":{"1min":1.1699999999999999,"5min":3.1190000000000002,"15min":2.6739999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1420000000000003,"5min":71.149000000000001,"15min":218.05500000000001},"last":0.63200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496143,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899900,"kb_used_data":4980,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92845252,"statfs":{"total":95995035648,"available":95073538048,"internally_reserved":0,"allocated":5099520,"data_stored":4057815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.2170000000000001,"15min":2.5009999999999999},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":5.9569999999999999,"5min":66.867000000000004,"15min":75.061999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":13.164,"5min":4.8049999999999997,"15min":3.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":212.096,"5min":212.096,"15min":212.096},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.738,"5min":4.6589999999999998,"15min":5.0199999999999996},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":20.050000000000001,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.5369999999999999,"5min":4.6719999999999997,"15min":5.024},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.676,"5min":219.94800000000001,"15min":292.904},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":4.6600000000000001,"15min":3.5720000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":10.032999999999999,"5min":185.654,"15min":185.654},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.024,"5min":4.5049999999999999,"15min":3.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.29399999999999998,"15min":0.27100000000000002},"max":{"1min":6.0960000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":3.5939999999999999,"15min":3.7570000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":8.5380000000000003,"5min":121.749,"15min":121.749},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.5049999999999999,"15min":3.6829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.4770000000000003,"5min":120.70399999999999,"15min":120.70399999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.3359999999999999,"15min":1.5629999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":10.009,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.609,"15min":1.581},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.223},"max":{"1min":8.2759999999999998,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.696,"15min":1.3500000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":9.984,"5min":9.984,"15min":12.747999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.6970000000000001,"5min":2.1560000000000001,"15min":1.599},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":9.0060000000000002,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":3.0070000000000001,"15min":2.798},"min":{"1min":0.307,"5min":0.307,"15min":0.26600000000000001},"max":{"1min":8.8059999999999992,"5min":165.37,"15min":165.37},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.5670000000000002,"15min":3.0880000000000001},"min":{"1min":0.36399999999999999,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":8.5980000000000008,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.433}]}]},{"osd":1,"up_from":110,"seq":472446402879,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967436,"kb_used_data":5828,"kb_used_omap":4,"kb_used_meta":961595,"kb_avail":92777716,"statfs":{"total":95995035648,"available":95004381184,"internally_reserved":0,"allocated":5967872,"data_stored":3852473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":984674190},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":6.851,"5min":2.9369999999999998,"15min":5.992},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":110.479,"5min":110.479,"15min":269.79199999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":8.1400000000000006,"5min":3.1259999999999999,"15min":6.4020000000000001},"min":{"1min":0.23599999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":110.59999999999999,"5min":110.59999999999999,"15min":271.20999999999998},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.625,"15min":3.1850000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":6.6020000000000003,"5min":13.217000000000001,"15min":139.75399999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.1509999999999998,"5min":2.0870000000000002,"15min":3.8940000000000001},"min":{"1min":0.307,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":8.968,"5min":12.244,"15min":140.29900000000001},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":2.202,"15min":3.1509999999999998},"min":{"1min":0.23599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.329000000000001,"5min":12.329000000000001,"15min":77.331999999999994},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.907,"5min":1.7130000000000001,"15min":2.2240000000000002},"min":{"1min":0.33400000000000002,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":7.0350000000000001,"5min":13.129,"15min":77.197000000000003},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":3.4569999999999999,"15min":3.597},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27500000000000002},"max":{"1min":5.4820000000000002,"5min":120.309,"15min":120.309},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":3.3530000000000002,"15min":3.3519999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":7.9900000000000002,"5min":120.297,"15min":120.297},"last":0.55000000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.9369999999999998,"15min":2.9359999999999999},"min":{"1min":0.39600000000000002,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":15.83,"5min":119.408,"15min":119.408},"last":0.52400000000000002},{"interface":"front","average":{"1min":3.3250000000000002,"5min":4.1779999999999999,"15min":3.2959999999999998},"min":{"1min":0.40600000000000003,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":7.9470000000000001,"5min":119.483,"15min":119.483},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.9820000000000002,"15min":3.403},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":8.0129999999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":0.5},{"interface":"front","average":{"1min":2.0089999999999999,"5min":3.1070000000000002,"15min":3.6419999999999999},"min":{"1min":0.42099999999999999,"5min":0.379,"15min":0.316},"max":{"1min":5.5750000000000002,"5min":121.559,"15min":121.559},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":4.3049999999999997,"15min":4.2549999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":6.7000000000000002,"5min":123.20699999999999,"15min":123.20699999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.0379999999999998,"5min":3.9540000000000002,"15min":3.649},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":6.718,"5min":123.48999999999999,"15min":123.48999999999999},"last":0.65000000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024313,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1028572,"kb_used_data":6548,"kb_used_omap":5,"kb_used_meta":1022010,"kb_avail":92716580,"statfs":{"total":95995035648,"available":94941777920,"internally_reserved":0,"allocated":6705152,"data_stored":4235245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1046538874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:38:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2859999999999996,"5min":3.8889999999999998,"15min":3.1179999999999999},"min":{"1min":0.23200000000000001,"5min":0.184,"15min":0.183},"max":{"1min":42.613,"5min":94.808999999999997,"15min":191.446},"last":0.23300000000000001},{"interface":"front","average":{"1min":3.8700000000000001,"5min":4.4880000000000004,"15min":3.552},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.187},"max":{"1min":43.963000000000001,"5min":94.876999999999995,"15min":191.393},"last":0.29199999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:38:56 2023","interfaces":[{"interface":"back","average":{"1min":7.7439999999999998,"5min":6.1040000000000001,"15min":3.8999999999999999},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":97.650000000000006,"5min":100.425,"15min":100.425},"last":0.249},{"interface":"front","average":{"1min":8.4260000000000002,"5min":6.1600000000000001,"15min":4.226},"min":{"1min":0.26200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":96.082999999999998,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.32700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":7.5549999999999997,"5min":3.9510000000000001,"15min":5.0010000000000003},"min":{"1min":0.313,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":95.555000000000007,"5min":95.555000000000007,"15min":208.11199999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":7.7720000000000002,"5min":4.133,"15min":5.0060000000000002},"min":{"1min":0.32800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":97.605000000000004,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":5.9390000000000001,"15min":3.02},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":11.519,"5min":200.654,"15min":200.654},"last":0.55600000000000005},{"interface":"front","average":{"1min":3.2050000000000001,"5min":6.6100000000000003,"15min":3.5059999999999998},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.0790000000000006,"5min":200.86600000000001,"15min":200.86600000000001},"last":0.44700000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":2.6989999999999998,"15min":2.0129999999999999},"min":{"1min":0.32900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.036,"5min":80.885999999999996,"15min":80.885999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.875,"5min":2.605,"15min":2.2210000000000001},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.2089999999999996,"5min":80.438999999999993,"15min":80.438999999999993},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":3.3069999999999999,"15min":3.2559999999999998},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":7.1660000000000004,"5min":118.056,"15min":120.572},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.448,"5min":3.1509999999999998,"15min":2.8929999999999998},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.563000000000001,"5min":118.274,"15min":120.64400000000001},"last":0.51700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":2.2200000000000002,"15min":2.5670000000000002},"min":{"1min":0.497,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":7.0270000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.49299999999999999},{"interface":"front","average":{"1min":3.601,"5min":2.6579999999999999,"15min":2.9009999999999998},"min":{"1min":0.54100000000000004,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9190000000000005,"5min":22.683,"15min":122.32299999999999},"last":0.54800000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814182,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888784,"kb_used_data":6024,"kb_used_omap":7,"kb_used_meta":882744,"kb_avail":92856368,"statfs":{"total":95995035648,"available":95084920832,"internally_reserved":0,"allocated":6168576,"data_stored":4113483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":903929937},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":7.5289999999999999,"15min":5.8550000000000004},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.19400000000000001},"max":{"1min":15.731999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.552,"5min":7.056,"15min":5.1829999999999998},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":13.795,"5min":156.922,"15min":168.49700000000001},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":5.952,"15min":5.101},"min":{"1min":0.21299999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2370000000000001,"5min":140.452,"15min":205.148},"last":0.438},{"interface":"front","average":{"1min":1.958,"5min":6.1539999999999999,"15min":5.2329999999999997},"min":{"1min":0.34699999999999998,"5min":0.224,"15min":0.183},"max":{"1min":7.6470000000000002,"5min":139.75299999999999,"15min":205.267},"last":0.373}]},{"osd":3,"last update":"Mon Oct 9 06:39:29 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.3029999999999999,"15min":3.548},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":7.6870000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.004,"5min":3.3860000000000001,"15min":3.899},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":8.8510000000000009,"5min":79.614000000000004,"15min":91.685000000000002},"last":0.46899999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":3.4820000000000002,"15min":2.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.407,"5min":79.643000000000001,"15min":90.938999999999993},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.25,"5min":3.7240000000000002,"15min":3.1520000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.26900000000000002},"max":{"1min":2.9039999999999999,"5min":82.872,"15min":90.950000000000003},"last":0.53300000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":3.75,"15min":2.4729999999999999},"min":{"1min":0.51500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4320000000000004,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.339,"5min":3.8809999999999998,"15min":2.5630000000000002},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.8620000000000001,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":3.8149999999999999,"15min":3.9529999999999998},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.30199999999999999},"max":{"1min":3.7919999999999998,"5min":82.899000000000001,"15min":138.291},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.0640000000000001,"5min":3.8900000000000001,"15min":3.891},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":8.1349999999999998,"5min":80.364999999999995,"15min":138.01599999999999},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":4.0190000000000001,"15min":3.1230000000000002},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.1809999999999992,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.403,"5min":3.387,"15min":2.7759999999999998},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":4.4530000000000003,"5min":82.798000000000002,"15min":82.798000000000002},"last":0.54400000000000004}]}]},{"osd":3,"up_from":149,"seq":639950127379,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":784624,"kb_used_data":5992,"kb_used_omap":2,"kb_used_meta":778621,"kb_avail":92960528,"statfs":{"total":95995035648,"available":95191580672,"internally_reserved":0,"allocated":6135808,"data_stored":4169537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":797307957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.0419999999999998,"15min":4.6059999999999999},"min":{"1min":0.216,"5min":0.191,"15min":0.19},"max":{"1min":11.317,"5min":22.367000000000001,"15min":161.821},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.8100000000000001,"5min":2.3260000000000001,"15min":4.3579999999999997},"min":{"1min":0.25700000000000001,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":11.233000000000001,"5min":13.443,"15min":163.52000000000001},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":3.4849999999999999,"5min":4.8120000000000003,"15min":3.9159999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":11.279999999999999,"5min":184.012,"15min":184.012},"last":0.32900000000000001},{"interface":"front","average":{"1min":3.2029999999999998,"5min":5.141,"15min":4.7000000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":11.442,"5min":184.703,"15min":184.703},"last":0.42899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":4.0960000000000001,"15min":3.7040000000000002},"min":{"1min":0.26400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.725,"5min":182.44900000000001,"15min":182.44900000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.8879999999999999,"5min":4.3079999999999998,"15min":4.173},"min":{"1min":0.25,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":11.41,"5min":182.56399999999999,"15min":182.56399999999999},"last":0.32000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":4.3360000000000003,"15min":2.9369999999999998},"min":{"1min":0.32000000000000001,"5min":0.318,"15min":0.26900000000000002},"max":{"1min":8.6690000000000005,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.7829999999999999,"5min":4.3819999999999997,"15min":2.9049999999999998},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.416,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.503}]},{"osd":5,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":5.3419999999999996,"15min":3.165},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":10.472,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.0609999999999999,"5min":4.1390000000000002,"15min":2.4900000000000002},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":6.657,"5min":177.59399999999999,"15min":177.59399999999999},"last":0.495}]},{"osd":6,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.2370000000000001,"15min":2.806},"min":{"1min":0.39000000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.505000000000001,"5min":16.462,"15min":130.39599999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.0870000000000002,"5min":1.8149999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":7.5949999999999998,"5min":22.260000000000002,"15min":130.67699999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":2.2829999999999999,"15min":2.8759999999999999},"min":{"1min":0.375,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7110000000000003,"5min":13.695,"15min":121.613},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.9390000000000001,"5min":3.1539999999999999,"15min":3.2559999999999998},"min":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.33400000000000002},"max":{"1min":10.532,"5min":87.001999999999995,"15min":121.559},"last":0.55700000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019416,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1320388,"kb_used_data":149308,"kb_used_omap":0,"kb_used_meta":1171072,"kb_avail":92424764,"statfs":{"total":95995035648,"available":94642958336,"internally_reserved":0,"allocated":152891392,"data_stored":151841872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1199177728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":4.7210000000000001,"15min":6.1859999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":3.0750000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":0.496},{"interface":"front","average":{"1min":1.298,"5min":5.3399999999999999,"15min":6.4199999999999999},"min":{"1min":0.317,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":4.202,"5min":286.36000000000001,"15min":287.57100000000003},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":3.5230000000000001,"15min":3.2389999999999999},"min":{"1min":0.247,"5min":0.246,"15min":0.22900000000000001},"max":{"1min":2.786,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":3.5350000000000001,"15min":3.3570000000000002},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":3.4180000000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":6.5839999999999996,"5min":6.0540000000000003,"15min":4.298},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":54.767000000000003,"5min":148.36099999999999,"15min":151.66399999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":6.2439999999999998,"5min":6.1929999999999996,"15min":4.2759999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":54.636000000000003,"5min":145.755,"15min":151.58699999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.5409999999999999,"15min":5.6559999999999997},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":6.8920000000000003,"5min":94.984999999999999,"15min":161.19300000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.411,"5min":3.1480000000000001,"15min":5.3979999999999997},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.524,"5min":94.966999999999999,"15min":161.934},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.3620000000000001,"15min":1.0589999999999999},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":4.2679999999999998,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.595,"5min":1.526,"15min":1.131},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.26600000000000001},"max":{"1min":4.1340000000000003,"5min":9.5009999999999994,"15min":9.5009999999999994},"last":0.34000000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.7400000000000002,"15min":1.958},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.2879999999999998,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.073,"5min":3.0670000000000002,"15min":2.258},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":4.3200000000000003,"5min":115.68600000000001,"15min":163.381},"last":0.40100000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4820000000000002,"15min":1.6950000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":3.8140000000000001,"5min":106.801,"15min":106.801},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.6030000000000002,"15min":1.722},"min":{"1min":0.40000000000000002,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":3.3300000000000001,"5min":106.827,"15min":106.827},"last":0.46899999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757781,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1688124,"kb_used_data":192244,"kb_used_omap":0,"kb_used_meta":1495872,"kb_avail":92057028,"statfs":{"total":95995035648,"available":94266396672,"internally_reserved":0,"allocated":196857856,"data_stored":195812780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1531772928},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":6.8890000000000002,"15min":6.0650000000000004},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.5899999999999999,"5min":149.149,"15min":294.89699999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.214,"5min":7.9660000000000002,"15min":6.202},"min":{"1min":0.24099999999999999,"5min":0.215,"15min":0.215},"max":{"1min":6.9029999999999996,"5min":149.21100000000001,"15min":294.30399999999997},"last":0.70599999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":26.379000000000001,"5min":14.6,"15min":7.0659999999999998},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":153.91300000000001,"5min":188.34100000000001,"15min":188.34100000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":25.495999999999999,"5min":14.301,"15min":7.0369999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":154.023,"5min":188.839,"15min":188.839},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":14.561999999999999,"5min":7.6950000000000003,"15min":4.3739999999999997},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":100.46599999999999,"5min":120.613,"15min":120.613},"last":0.58399999999999996},{"interface":"front","average":{"1min":15.493,"5min":7.4169999999999998,"15min":4.2469999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":100.56999999999999,"5min":120.962,"15min":120.962},"last":0.629}]},{"osd":3,"last update":"Mon Oct 9 06:39:18 2023","interfaces":[{"interface":"back","average":{"1min":15.433,"5min":8.0549999999999997,"15min":6.6719999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":220.41800000000001,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":15.595000000000001,"5min":7.508,"15min":6.3470000000000004},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":220.45500000000001,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.59999999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":2.472,"15min":1.536},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":4.5330000000000004,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.907,"5min":2.6030000000000002,"15min":1.5109999999999999},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.25800000000000001},"max":{"1min":5.5170000000000003,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8480000000000001,"15min":1.655},"min":{"1min":0.35299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.5810000000000004,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1.9419999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0819999999999999,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":2.4929999999999999,"15min":1.6850000000000001},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":11.146000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.1560000000000001,"15min":1.573},"min":{"1min":0.38900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8070000000000004,"5min":79.138999999999996,"15min":79.138999999999996},"last":0.72899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:39:34.440 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:39:34.491 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:34.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:34.595 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:34.626 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:34.720 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:34.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:34.823 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:34.853 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:35.000 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:35.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:35.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6275 12.d 12:b8b44611:::benchmark_data_smithi107_479504_object6274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 501.01295738900001, 2023-10-09T06:39:35.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267782297, 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6275 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025392+0000", 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025391+0000", 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025438+0000", 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7531999999999998e-05 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025454+0000", 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5452e-05 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.044 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025461+0000", 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1380000000000001e-06 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025514+0000", 2023-10-09T06:39:35.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2778999999999997e-05 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025621+0000", 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010712 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025721+0000", 2023-10-09T06:39:35.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000100147 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027384+0000", 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0016627790000000001 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027404+0000", 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0047999999999998e-05 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293102+0000", 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265697933 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293139+0000", 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7493000000000002e-05 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293175+0000", 2023-10-09T06:39:35.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5145999999999999e-05 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6278 12.d 12:be6c2ec3:::benchmark_data_smithi107_479504_object6277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 501.01285499900001, 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267768063, 2023-10-09T06:39:35.050 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6278 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025495+0000", 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025494+0000", 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025557+0000", 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2922000000000005e-05 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025575+0000", 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7739000000000001e-05 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025579+0000", 2023-10-09T06:39:35.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0400000000000003e-06 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026133+0000", 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055439499999999995 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026261+0000", 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127372 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026428+0000", 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016770500000000001 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027443+0000", 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001015099 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027452+0000", 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.6920000000000008e-06 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293213+0000", 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265760789 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293242+0000", 2023-10-09T06:39:35.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9085e-05 2023-10-09T06:39:35.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293263+0000", 2023-10-09T06:39:35.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0744999999999999e-05 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:6279 12.d 12:bf6b6632:::benchmark_data_smithi107_479504_object6278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 501.01276331999998, 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26777375199999998, 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.059 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6279 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025586+0000", 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025640+0000", 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4079999999999997e-05 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025645+0000", 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0640000000000001e-06 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.025648+0000", 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0460000000000001e-06 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026860+0000", 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001212252 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.026986+0000", 2023-10-09T06:39:35.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012579999999999999 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027075+0000", 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8715999999999996e-05 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027532+0000", 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045738199999999999 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.027923+0000", 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00039128200000000001 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293289+0000", 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26536579300000002 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293328+0000", 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8287000000000001e-05 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:14.293360+0000", 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2518000000000003e-05 2023-10-09T06:39:35.066 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:37240 12.d 12:bb21ad69:::benchmark_data_smithi107_479504_object37239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 483.190305309, 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25148841100000002, 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.067 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 37240 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848044+0000", 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848043+0000", 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848103+0000", 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9973999999999999e-05 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848112+0000", 2023-10-09T06:39:35.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7090000000000008e-06 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848117+0000", 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3640000000000001e-06 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848144+0000", 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7644000000000002e-05 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848210+0000", 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5368000000000002e-05 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.848283+0000", 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3858999999999999e-05 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849143+0000", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000859705 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097694+0000", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24855127599999999 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099429+0000", 2023-10-09T06:39:35.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001734759 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099492+0000", 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2785000000000003e-05 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.099533+0000", 2023-10-09T06:39:35.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0816999999999999e-05 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46340.0:37240 12.d e367/361 ondisk, result = 0)", 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 483.18858178400001, 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24792936400000001, 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:39:35.075 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849768+0000", 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849767+0000", 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849771+0000", 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.523e-06 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849788+0000", 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6719e-05 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:31.849792+0000", 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9400000000000004e-06 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097677+0000", 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24788569099999999 2023-10-09T06:39:35.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097687+0000", 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7059999999999999e-06 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:31:32.097697+0000", 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0288e-05 2023-10-09T06:39:35.080 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:76840 12.d 12:be36faa0:::benchmark_data_smithi107_479504_object76839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 443.289322175, 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.232798802, 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.081 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 76840 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749027+0000", 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749026+0000", 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749082+0000", 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5448000000000002e-05 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749094+0000", 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2517999999999999e-05 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749100+0000", 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4060000000000004e-06 2023-10-09T06:39:35.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749151+0000", 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1034e-05 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749306+0000", 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015525700000000001 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.085 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749503+0000", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00019740799999999999 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.756545+0000", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0070414650000000002 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.955708+0000", 2023-10-09T06:39:35.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.199163272 2023-10-09T06:39:35.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981744+0000", 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026035632999999999 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981787+0000", 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3210000000000001e-05 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.981826+0000", 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9178999999999999e-05 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:76844 12.d 12:b571eb92:::benchmark_data_smithi107_479504_object76843:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 443.289104033, 2023-10-09T06:39:35.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23556622999999999, 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 76844 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:39:35.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749246+0000", 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749245+0000", 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749318+0000", 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2859000000000002e-05 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749325+0000", 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9809999999999997e-06 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.749328+0000", 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5949999999999999e-06 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754650+0000", 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053215629999999996 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754748+0000", 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8474999999999995e-05 2023-10-09T06:39:35.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.754858+0000", 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000109648 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.756692+0000", 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001834086 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.957451+0000", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20075878699999999 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984727+0000", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.027275995000000001 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984764+0000", 2023-10-09T06:39:35.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7248e-05 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:11.984812+0000", 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7599999999999998e-05 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:153016 12.d 12:b3683356:::benchmark_data_smithi107_479504_object53405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-10-09T06:39:35.096 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 336.03344933, 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23512572800000001, 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 153016 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004900+0000", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004899+0000", 2023-10-09T06:39:35.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004906+0000", 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3010000000000001e-06 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004921+0000", 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5142e-05 2023-10-09T06:39:35.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004928+0000", 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0589999999999997e-06 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.004982+0000", 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3878999999999997e-05 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005090+0000", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010767799999999999 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.005531+0000", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044087999999999999 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.009097+0000", 2023-10-09T06:39:35.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0035658479999999999 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.012729+0000", 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036324439999999999 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.239962+0000", 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22723316399999999 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.239999+0000", 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6426e-05 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:33:59.240026+0000", 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7291999999999999e-05 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180927 12.d 12:b6ff457f:::benchmark_data_smithi107_479504_object81316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 296.28993803700001, 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.35203043499999997, 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180927 2023-10-09T06:39:35.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748411+0000", 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748415+0000", 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6779999999999999e-06 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748424+0000", 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748429+0000", 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3270000000000002e-06 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748471+0000", 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2400000000000001e-05 2023-10-09T06:39:35.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748563+0000", 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2088000000000003e-05 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748643+0000", 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9326000000000006e-05 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.750567+0000", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019239070000000001 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.751954+0000", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013873869999999999 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100346+0000", 2023-10-09T06:39:35.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.34839218799999999 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100401+0000", 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4484999999999998e-05 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100442+0000", 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1430000000000001e-05 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.110 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180945 12.d 12:b29197fc:::benchmark_data_smithi107_479504_object81334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 296.25898948600002, 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32532718199999999, 2023-10-09T06:39:35.111 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180945 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779359+0000", 2023-10-09T06:39:35.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779366+0000", 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8349999999999998e-06 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779380+0000", 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3978999999999999e-05 2023-10-09T06:39:35.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779386+0000", 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2960000000000004e-06 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779426+0000", 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0438000000000001e-05 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779510+0000", 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3070000000000003e-05 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779582+0000", 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2076999999999994e-05 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.782438+0000", 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002856408 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.786572+0000", 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041337539999999999 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104626+0000", 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31805433399999999 2023-10-09T06:39:35.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104660+0000", 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3751000000000003e-05 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104687+0000", 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7487999999999998e-05 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.118 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98783 13.e 13:7e44f156:::benchmark_data_smithi107_535588_object98782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 158.89944061599999, 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23612880999999999, 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.119 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98783 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138907+0000", 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138985+0000", 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8372000000000001e-05 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138998+0000", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.254e-05 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139004+0000", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1290000000000001e-06 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139052+0000", 2023-10-09T06:39:35.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8241000000000001e-05 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139170+0000", 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117379 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:39:35.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139302+0000", 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013261800000000001 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145027+0000", 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057249759999999997 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148343+0000", 2023-10-09T06:39:35.126 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033158129999999999 2023-10-09T06:39:35.126 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.126 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.126 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.127 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374933+0000", 2023-10-09T06:39:35.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22659017100000001 2023-10-09T06:39:35.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.127 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374992+0000", 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8931000000000002e-05 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.375038+0000", 2023-10-09T06:39:35.135 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5528000000000002e-05 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:39:35.136 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 158.8993265, 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30779935400000002, 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.137 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.138 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.138 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:39:35.138 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.138 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.139 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.139 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.140 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139054+0000", 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000001e-05 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139062+0000", 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.141 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139066+0000", 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140814+0000", 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001748793 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.142 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140929+0000", 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.141055+0000", 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126563 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.146811+0000", 2023-10-09T06:39:35.143 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005755418 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446811+0000", 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29999972899999999 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446822+0000", 2023-10-09T06:39:35.144 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1856e-05 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 158.783884175, 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93419914299999995, 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:39:35.145 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.146 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254463+0000", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254519+0000", 2023-10-09T06:39:35.147 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6208999999999997e-05 2023-10-09T06:39:35.148 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.148 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.148 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.149 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254547+0000", 2023-10-09T06:39:35.149 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8109000000000001e-05 2023-10-09T06:39:35.149 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.149 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.149 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254562+0000", 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5073e-05 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.150 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254724+0000", 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161974 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254729+0000", 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497899+0000", 2023-10-09T06:39:35.151 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24316990899999999 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497910+0000", 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0874e-05 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188593+0000", 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69068296699999998 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.152 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188665+0000", 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1802000000000001e-05 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98796 13.7 13:e44dc813:::benchmark_data_smithi107_535588_object98795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 158.68223456800001, 2023-10-09T06:39:35.153 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84800129700000004, 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98796 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:35.154 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356113+0000", 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.155 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4719000000000002e-05 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356184+0000", 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6277e-05 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.156 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356192+0000", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9789999999999993e-06 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498051+0000", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14185953700000001 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498058+0000", 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2799999999999998e-06 2023-10-09T06:39:35.157 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188683+0000", 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.690624601 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.158 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188792+0000", 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108497 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.159 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188939+0000", 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014728699999999999 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.191074+0000", 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021353840000000002 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.160 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.203068+0000", 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011993634 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204019+0000", 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000951625 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.161 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204076+0000", 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6569e-05 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204116+0000", 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0346999999999998e-05 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.162 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98805 13.0 13:054831b9:::benchmark_data_smithi107_535588_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 158.35213062400001, 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.528813278, 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98805 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.163 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.164 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686214+0000", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999952 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686248+0000", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4001000000000002e-05 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686269+0000", 2023-10-09T06:39:35.165 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0539000000000001e-05 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686278+0000", 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2469999999999997e-06 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686339+0000", 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0735e-05 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.166 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686348+0000", 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181836+0000", 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49548772400000002 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.167 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181965+0000", 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000129448 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.182343+0000", 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037732500000000002 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.184651+0000", 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023081030000000002 2023-10-09T06:39:35.168 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.199956+0000", 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015305375 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214937+0000", 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014981213 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.169 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214993+0000", 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5884999999999998e-05 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.215032+0000", 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9060999999999998e-05 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.170 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207733 13.0 13:0665e03e:::benchmark_data_smithi107_535588_object101626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 28.322748182000002, 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26422807999999998, 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.171 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207733 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.172 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715600+0000", 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715606+0000", 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.173 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715622+0000", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6038000000000001e-05 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715630+0000", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2320000000000004e-06 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715686+0000", 2023-10-09T06:39:35.174 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6475999999999999e-05 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715825+0000", 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013932800000000001 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716377+0000", 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055127400000000003 2023-10-09T06:39:35.175 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.722227+0000", 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0058503299999999999 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.964701+0000", 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24247376800000001 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.176 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979740+0000", 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015039132 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979785+0000", 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5225999999999997e-05 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.177 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979830+0000", 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4431000000000002e-05 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:209742 13.e 13:70f84e2f:::benchmark_data_smithi107_535588_object103635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:09.469812+0000", 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 25.568537797000001, 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23086913000000001, 2023-10-09T06:39:35.178 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 209742 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469812+0000", 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.179 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469812+0000", 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469810+0000", 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.180 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469817+0000", 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9589999999999998e-06 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469830+0000", 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2588999999999999e-05 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.181 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469836+0000", 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3990000000000001e-06 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.469887+0000", 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0192999999999999e-05 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.470002+0000", 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115709 2023-10-09T06:39:35.182 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.470110+0000", 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010807900000000001 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.473638+0000", 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003527386 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.183 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.478378+0000", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047398320000000002 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.700596+0000", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22221796299999999 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.700645+0000", 2023-10-09T06:39:35.184 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9252999999999998e-05 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:09.700681+0000", 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6159999999999999e-05 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211410 13.7 13:e70c994c:::benchmark_data_smithi107_535588_object105303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:35.185 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 23.368903408000001, 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267478086, 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211410 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.186 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.187 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669444+0000", 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669454+0000", 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8539999999999992e-06 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669466+0000", 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1838000000000001e-05 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.188 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669472+0000", 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0059999999999998e-06 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669508+0000", 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.189 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669585+0000", 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7124000000000002e-05 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670141+0000", 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055599200000000005 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.674991+0000", 2023-10-09T06:39:35.190 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048495869999999998 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936590+0000", 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159882800000001 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936837+0000", 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024747099999999998 2023-10-09T06:39:35.191 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936885+0000", 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8071000000000001e-05 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936924+0000", 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8995000000000002e-05 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.192 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211418 13.17 13:e9af5116:::benchmark_data_smithi107_535588_object105311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 23.365944433999999, 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.264762895, 2023-10-09T06:39:35.193 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211418 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.194 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672404+0000", 2023-10-09T06:39:35.195 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672414+0000", 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0522e-05 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672428+0000", 2023-10-09T06:39:35.196 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3747999999999999e-05 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672435+0000", 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.878e-06 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672490+0000", 2023-10-09T06:39:35.197 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5414999999999997e-05 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672574+0000", 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4127000000000003e-05 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673020+0000", 2023-10-09T06:39:35.198 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044588000000000001 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676720+0000", 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036994749999999998 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936560+0000", 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25984011699999998 2023-10-09T06:39:35.199 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937086+0000", 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000526558 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937128+0000", 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1659999999999998e-05 2023-10-09T06:39:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937168+0000", 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9950999999999997e-05 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211420 13.17 13:ef772928:::benchmark_data_smithi107_535588_object105313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:39:35.201 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 23.365108495000001, 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265498495, 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211420 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.202 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:39:35.203 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673244+0000", 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7500000000000001e-06 2023-10-09T06:39:35.204 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673253+0000", 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2239999999999995e-06 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673257+0000", 2023-10-09T06:39:35.205 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4170000000000002e-06 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675010+0000", 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017534130000000001 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.206 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.207 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:39:35.207 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675083+0000", 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2701999999999996e-05 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675377+0000", 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029354000000000001 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:39:35.208 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676887+0000", 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015100999999999999 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936617+0000", 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.259729969 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938661+0000", 2023-10-09T06:39:35.209 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020442260000000001 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938704+0000", 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3467000000000002e-05 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938740+0000", 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5256999999999998e-05 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.210 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.211 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.211 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:39:35.211 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:39:35.211 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:39:35.211 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:35.212 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: pgmap v1503: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:35.213 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.213 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:39:35.213 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.214 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.214 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.214 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[212750]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.214 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: pgmap v1503: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:35.214 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.215 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:39:35.215 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.218 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.218 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.218 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:34 smithi150 ceph-mon[194178]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.218 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: pgmap v1503: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: osdmap e490: 8 total, 8 up, 8 in 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.219 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:34 smithi107 ceph-mon[220921]: failed to encode map e490 with expected crc 2023-10-09T06:39:35.246 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:35.291 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:35.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:35.295 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-10-09T06:39:35.395 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:35.430 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:35.450 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:35.522 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:35.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:35.627 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:35.655 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:35.659 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:39:35.860 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:39:35.880 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:39:35.906 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:39:35.909 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:35.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:35.929 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:39:35.955 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:39:35.981 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:39:36.000 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:39:36.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:36.019 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.020 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.021 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.021 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:39:36.055 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:36.101 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:36.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:36.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:36.105 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[212750]: failed to encode map e491 with expected crc 2023-10-09T06:39:36.105 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[212750]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:39:36.105 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/439458839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:36.106 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[220921]: failed to encode map e491 with expected crc 2023-10-09T06:39:36.106 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[220921]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:39:36.106 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/439458839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:36.205 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:36.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:35 smithi150 ceph-mon[194178]: failed to encode map e491 with expected crc 2023-10-09T06:39:36.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:35 smithi150 ceph-mon[194178]: osdmap e491: 8 total, 8 up, 8 in 2023-10-09T06:39:36.231 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/439458839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:36.234 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:36.278 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:36.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:36.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:36.384 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:36.391 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:39:36.411 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:36.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:36.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:36.762 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:36.793 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:36 smithi107 ceph-mon[220921]: pgmap v1506: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:37.083 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:37.138 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:37.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:37.243 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:37.269 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:36 smithi150 ceph-mon[194178]: pgmap v1506: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:37.272 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:37.291 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:37.291 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:37.315 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:37.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:37.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:36 smithi107 ceph-mon[212750]: pgmap v1506: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:37.421 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:37.447 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:37.489 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:37.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:37.594 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:37.624 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:37.981 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:39:37.983 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:39:38.487 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:38.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:38.592 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:38.625 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:38.707 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:38.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:38.808 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:38.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:38.881 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:38.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:38.986 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.018 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:39.055 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:39:39.057 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1507,"stamp":"2023-10-09T06:39:37.536920+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":339,"acting":339,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":312,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9091004,"kb_used_data":383100,"kb_used_omap":20,"kb_used_meta":8707819,"kb_avail":740870212,"statfs":{"total":767960285184,"available":758651097088,"internally_reserved":0,"allocated":392294400,"data_stored":379468105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8916807310},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001167"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":782,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106768+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:39:26.106768+0000","last_peered":"2023-10-09T06:39:26.106768+0000","last_clean":"2023-10-09T06:39:26.106768+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:39:26.106768+0000","last_undegraded":"2023-10-09T06:39:26.106768+0000","last_fullsized":"2023-10-09T06:39:26.106768+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":467,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789540+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:39:25.789540+0000","last_peered":"2023-10-09T06:39:25.789540+0000","last_clean":"2023-10-09T06:39:25.789540+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:39:25.789540+0000","last_undegraded":"2023-10-09T06:39:25.789540+0000","last_fullsized":"2023-10-09T06:39:25.789540+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":190,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873433+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:39:26.873433+0000","last_peered":"2023-10-09T06:39:26.873433+0000","last_clean":"2023-10-09T06:39:26.873433+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:39:26.873433+0000","last_undegraded":"2023-10-09T06:39:26.873433+0000","last_fullsized":"2023-10-09T06:39:26.873433+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":743,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.204397+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:39:27.204397+0000","last_peered":"2023-10-09T06:39:27.204397+0000","last_clean":"2023-10-09T06:39:27.204397+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:39:27.204397+0000","last_undegraded":"2023-10-09T06:39:27.204397+0000","last_fullsized":"2023-10-09T06:39:27.204397+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":484,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233455+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:39:27.233455+0000","last_peered":"2023-10-09T06:39:27.233455+0000","last_clean":"2023-10-09T06:39:27.233455+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:39:27.233455+0000","last_undegraded":"2023-10-09T06:39:27.233455+0000","last_fullsized":"2023-10-09T06:39:27.233455+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":490,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913747+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:39:32.913747+0000","last_peered":"2023-10-09T06:39:32.913747+0000","last_clean":"2023-10-09T06:39:32.913747+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:39:32.913747+0000","last_undegraded":"2023-10-09T06:39:32.913747+0000","last_fullsized":"2023-10-09T06:39:32.913747+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":735,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789462+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:39:25.789462+0000","last_peered":"2023-10-09T06:39:25.789462+0000","last_clean":"2023-10-09T06:39:25.789462+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:39:25.789462+0000","last_undegraded":"2023-10-09T06:39:25.789462+0000","last_fullsized":"2023-10-09T06:39:25.789462+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":208,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.872842+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:39:26.872842+0000","last_peered":"2023-10-09T06:39:26.872842+0000","last_clean":"2023-10-09T06:39:26.872842+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:39:26.872842+0000","last_undegraded":"2023-10-09T06:39:26.872842+0000","last_fullsized":"2023-10-09T06:39:26.872842+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":477,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786391+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:39:25.786391+0000","last_peered":"2023-10-09T06:39:25.786391+0000","last_clean":"2023-10-09T06:39:25.786391+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:39:25.786391+0000","last_undegraded":"2023-10-09T06:39:25.786391+0000","last_fullsized":"2023-10-09T06:39:25.786391+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":596,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107048+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:39:26.107048+0000","last_peered":"2023-10-09T06:39:26.107048+0000","last_clean":"2023-10-09T06:39:26.107048+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:39:26.107048+0000","last_undegraded":"2023-10-09T06:39:26.107048+0000","last_fullsized":"2023-10-09T06:39:26.107048+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913677+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:39:32.913677+0000","last_peered":"2023-10-09T06:39:32.913677+0000","last_clean":"2023-10-09T06:39:32.913677+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:39:32.913677+0000","last_undegraded":"2023-10-09T06:39:32.913677+0000","last_fullsized":"2023-10-09T06:39:32.913677+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.872953+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:39:26.872953+0000","last_peered":"2023-10-09T06:39:26.872953+0000","last_clean":"2023-10-09T06:39:26.872953+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:39:26.872953+0000","last_undegraded":"2023-10-09T06:39:26.872953+0000","last_fullsized":"2023-10-09T06:39:26.872953+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":633,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107369+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:39:26.107369+0000","last_peered":"2023-10-09T06:39:26.107369+0000","last_clean":"2023-10-09T06:39:26.107369+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:39:26.107369+0000","last_undegraded":"2023-10-09T06:39:26.107369+0000","last_fullsized":"2023-10-09T06:39:26.107369+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":175,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.304783+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:39:26.304783+0000","last_peered":"2023-10-09T06:39:26.304783+0000","last_clean":"2023-10-09T06:39:26.304783+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:39:26.304783+0000","last_undegraded":"2023-10-09T06:39:26.304783+0000","last_fullsized":"2023-10-09T06:39:26.304783+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":191,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873222+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:39:26.873222+0000","last_peered":"2023-10-09T06:39:26.873222+0000","last_clean":"2023-10-09T06:39:26.873222+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:39:26.873222+0000","last_undegraded":"2023-10-09T06:39:26.873222+0000","last_fullsized":"2023-10-09T06:39:26.873222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":618,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.915102+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:39:33.915102+0000","last_peered":"2023-10-09T06:39:33.915102+0000","last_clean":"2023-10-09T06:39:33.915102+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:39:33.915102+0000","last_undegraded":"2023-10-09T06:39:33.915102+0000","last_fullsized":"2023-10-09T06:39:33.915102+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":369,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108478+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:39:26.108478+0000","last_peered":"2023-10-09T06:39:26.108478+0000","last_clean":"2023-10-09T06:39:26.108478+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:39:26.108478+0000","last_undegraded":"2023-10-09T06:39:26.108478+0000","last_fullsized":"2023-10-09T06:39:26.108478+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":426,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233781+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:39:27.233781+0000","last_peered":"2023-10-09T06:39:27.233781+0000","last_clean":"2023-10-09T06:39:27.233781+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:39:27.233781+0000","last_undegraded":"2023-10-09T06:39:27.233781+0000","last_fullsized":"2023-10-09T06:39:27.233781+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":597,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233831+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:39:27.233831+0000","last_peered":"2023-10-09T06:39:27.233831+0000","last_clean":"2023-10-09T06:39:27.233831+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:39:27.233831+0000","last_undegraded":"2023-10-09T06:39:27.233831+0000","last_fullsized":"2023-10-09T06:39:27.233831+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":494,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787822+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:39:25.787822+0000","last_peered":"2023-10-09T06:39:25.787822+0000","last_clean":"2023-10-09T06:39:25.787822+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:39:25.787822+0000","last_undegraded":"2023-10-09T06:39:25.787822+0000","last_fullsized":"2023-10-09T06:39:25.787822+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790700+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:39:25.790700+0000","last_peered":"2023-10-09T06:39:25.790700+0000","last_clean":"2023-10-09T06:39:25.790700+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:39:25.790700+0000","last_undegraded":"2023-10-09T06:39:25.790700+0000","last_fullsized":"2023-10-09T06:39:25.790700+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":763,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.911311+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:39:32.911311+0000","last_peered":"2023-10-09T06:39:32.911311+0000","last_clean":"2023-10-09T06:39:32.911311+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:39:32.911311+0000","last_undegraded":"2023-10-09T06:39:32.911311+0000","last_fullsized":"2023-10-09T06:39:32.911311+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":337,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107191+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:39:26.107191+0000","last_peered":"2023-10-09T06:39:26.107191+0000","last_clean":"2023-10-09T06:39:26.107191+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:39:26.107191+0000","last_undegraded":"2023-10-09T06:39:26.107191+0000","last_fullsized":"2023-10-09T06:39:26.107191+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":191,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873380+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:39:26.873380+0000","last_peered":"2023-10-09T06:39:26.873380+0000","last_clean":"2023-10-09T06:39:26.873380+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:39:26.873380+0000","last_undegraded":"2023-10-09T06:39:26.873380+0000","last_fullsized":"2023-10-09T06:39:26.873380+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":760,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106864+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:39:26.106864+0000","last_peered":"2023-10-09T06:39:26.106864+0000","last_clean":"2023-10-09T06:39:26.106864+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:39:26.106864+0000","last_undegraded":"2023-10-09T06:39:26.106864+0000","last_fullsized":"2023-10-09T06:39:26.106864+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":488,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.911759+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:39:32.911759+0000","last_peered":"2023-10-09T06:39:32.911759+0000","last_clean":"2023-10-09T06:39:32.911759+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:39:32.911759+0000","last_undegraded":"2023-10-09T06:39:32.911759+0000","last_fullsized":"2023-10-09T06:39:32.911759+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":308,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108308+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:39:26.108308+0000","last_peered":"2023-10-09T06:39:26.108308+0000","last_clean":"2023-10-09T06:39:26.108308+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:39:26.108308+0000","last_undegraded":"2023-10-09T06:39:26.108308+0000","last_fullsized":"2023-10-09T06:39:26.108308+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":816,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233532+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:39:27.233532+0000","last_peered":"2023-10-09T06:39:27.233532+0000","last_clean":"2023-10-09T06:39:27.233532+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:39:27.233532+0000","last_undegraded":"2023-10-09T06:39:27.233532+0000","last_fullsized":"2023-10-09T06:39:27.233532+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":331,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106922+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:39:26.106922+0000","last_peered":"2023-10-09T06:39:26.106922+0000","last_clean":"2023-10-09T06:39:26.106922+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:39:26.106922+0000","last_undegraded":"2023-10-09T06:39:26.106922+0000","last_fullsized":"2023-10-09T06:39:26.106922+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913497+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:39:32.913497+0000","last_peered":"2023-10-09T06:39:32.913497+0000","last_clean":"2023-10-09T06:39:32.913497+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:39:32.913497+0000","last_undegraded":"2023-10-09T06:39:32.913497+0000","last_fullsized":"2023-10-09T06:39:32.913497+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":602,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.304275+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:39:26.304275+0000","last_peered":"2023-10-09T06:39:26.304275+0000","last_clean":"2023-10-09T06:39:26.304275+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:39:26.304275+0000","last_undegraded":"2023-10-09T06:39:26.304275+0000","last_fullsized":"2023-10-09T06:39:26.304275+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":566,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786321+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:39:25.786321+0000","last_peered":"2023-10-09T06:39:25.786321+0000","last_clean":"2023-10-09T06:39:25.786321+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:39:25.786321+0000","last_undegraded":"2023-10-09T06:39:25.786321+0000","last_fullsized":"2023-10-09T06:39:25.786321+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873747+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:39:26.873747+0000","last_peered":"2023-10-09T06:39:26.873747+0000","last_clean":"2023-10-09T06:39:26.873747+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:39:26.873747+0000","last_undegraded":"2023-10-09T06:39:26.873747+0000","last_fullsized":"2023-10-09T06:39:26.873747+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":615,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107153+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:39:26.107153+0000","last_peered":"2023-10-09T06:39:26.107153+0000","last_clean":"2023-10-09T06:39:26.107153+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:39:26.107153+0000","last_undegraded":"2023-10-09T06:39:26.107153+0000","last_fullsized":"2023-10-09T06:39:26.107153+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":373,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.111633+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:39:26.111633+0000","last_peered":"2023-10-09T06:39:26.111633+0000","last_clean":"2023-10-09T06:39:26.111633+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:39:26.111633+0000","last_undegraded":"2023-10-09T06:39:26.111633+0000","last_fullsized":"2023-10-09T06:39:26.111633+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107604+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:39:26.107604+0000","last_peered":"2023-10-09T06:39:26.107604+0000","last_clean":"2023-10-09T06:39:26.107604+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:39:26.107604+0000","last_undegraded":"2023-10-09T06:39:26.107604+0000","last_fullsized":"2023-10-09T06:39:26.107604+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.926580+0000","last_change":"2023-10-09T06:39:33.923263+0000","last_active":"2023-10-09T06:39:33.926580+0000","last_peered":"2023-10-09T06:39:33.926580+0000","last_clean":"2023-10-09T06:39:33.926580+0000","last_became_active":"2023-10-09T06:39:33.922687+0000","last_became_peered":"2023-10-09T06:39:33.922687+0000","last_unstale":"2023-10-09T06:39:33.926580+0000","last_undegraded":"2023-10-09T06:39:33.926580+0000","last_fullsized":"2023-10-09T06:39:33.926580+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:59.579546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":875,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873099+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:39:26.873099+0000","last_peered":"2023-10-09T06:39:26.873099+0000","last_clean":"2023-10-09T06:39:26.873099+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:39:26.873099+0000","last_undegraded":"2023-10-09T06:39:26.873099+0000","last_fullsized":"2023-10-09T06:39:26.873099+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":335,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.109056+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:39:26.109056+0000","last_peered":"2023-10-09T06:39:26.109056+0000","last_clean":"2023-10-09T06:39:26.109056+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:39:26.109056+0000","last_undegraded":"2023-10-09T06:39:26.109056+0000","last_fullsized":"2023-10-09T06:39:26.109056+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":485,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788802+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:39:25.788802+0000","last_peered":"2023-10-09T06:39:25.788802+0000","last_clean":"2023-10-09T06:39:25.788802+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:39:25.788802+0000","last_undegraded":"2023-10-09T06:39:25.788802+0000","last_fullsized":"2023-10-09T06:39:25.788802+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.928890+0000","last_change":"2023-10-09T06:39:33.927681+0000","last_active":"2023-10-09T06:39:33.928890+0000","last_peered":"2023-10-09T06:39:33.928890+0000","last_clean":"2023-10-09T06:39:33.928890+0000","last_became_active":"2023-10-09T06:39:33.927408+0000","last_became_peered":"2023-10-09T06:39:33.927408+0000","last_unstale":"2023-10-09T06:39:33.928890+0000","last_undegraded":"2023-10-09T06:39:33.928890+0000","last_fullsized":"2023-10-09T06:39:33.928890+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:59.401749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":789,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.785745+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:39:25.785745+0000","last_peered":"2023-10-09T06:39:25.785745+0000","last_clean":"2023-10-09T06:39:25.785745+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:39:25.785745+0000","last_undegraded":"2023-10-09T06:39:25.785745+0000","last_fullsized":"2023-10-09T06:39:25.785745+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":391,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107472+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:39:26.107472+0000","last_peered":"2023-10-09T06:39:26.107472+0000","last_clean":"2023-10-09T06:39:26.107472+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:39:26.107472+0000","last_undegraded":"2023-10-09T06:39:26.107472+0000","last_fullsized":"2023-10-09T06:39:26.107472+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":464,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784350+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:39:25.784350+0000","last_peered":"2023-10-09T06:39:25.784350+0000","last_clean":"2023-10-09T06:39:25.784350+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:39:25.784350+0000","last_undegraded":"2023-10-09T06:39:25.784350+0000","last_fullsized":"2023-10-09T06:39:25.784350+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":1,"reported_epoch":489,"state":"creating+peering","last_fresh":"2023-10-09T06:39:32.916619+0000","last_change":"2023-10-09T06:39:32.916619+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.916619+0000","last_undegraded":"2023-10-09T06:39:32.916619+0000","last_fullsized":"2023-10-09T06:39:32.916619+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:35.001011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":612,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789702+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:39:25.789702+0000","last_peered":"2023-10-09T06:39:25.789702+0000","last_clean":"2023-10-09T06:39:25.789702+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:39:25.789702+0000","last_undegraded":"2023-10-09T06:39:25.789702+0000","last_fullsized":"2023-10-09T06:39:25.789702+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":59,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873439+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:39:26.873439+0000","last_peered":"2023-10-09T06:39:26.873439+0000","last_clean":"2023-10-09T06:39:26.873439+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:39:26.873439+0000","last_undegraded":"2023-10-09T06:39:26.873439+0000","last_fullsized":"2023-10-09T06:39:26.873439+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":452,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.912585+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:39:32.912585+0000","last_peered":"2023-10-09T06:39:32.912585+0000","last_clean":"2023-10-09T06:39:32.912585+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:39:32.912585+0000","last_undegraded":"2023-10-09T06:39:32.912585+0000","last_fullsized":"2023-10-09T06:39:32.912585+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.923505+0000","last_change":"2023-10-09T06:39:33.922928+0000","last_active":"2023-10-09T06:39:33.923505+0000","last_peered":"2023-10-09T06:39:33.923505+0000","last_clean":"2023-10-09T06:39:33.923505+0000","last_became_active":"2023-10-09T06:39:33.922653+0000","last_became_peered":"2023-10-09T06:39:33.922653+0000","last_unstale":"2023-10-09T06:39:33.923505+0000","last_undegraded":"2023-10-09T06:39:33.923505+0000","last_fullsized":"2023-10-09T06:39:33.923505+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:31.453742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":840,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106842+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:39:26.106842+0000","last_peered":"2023-10-09T06:39:26.106842+0000","last_clean":"2023-10-09T06:39:26.106842+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:39:26.106842+0000","last_undegraded":"2023-10-09T06:39:26.106842+0000","last_fullsized":"2023-10-09T06:39:26.106842+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":217,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873058+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:39:26.873058+0000","last_peered":"2023-10-09T06:39:26.873058+0000","last_clean":"2023-10-09T06:39:26.873058+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:39:26.873058+0000","last_undegraded":"2023-10-09T06:39:26.873058+0000","last_fullsized":"2023-10-09T06:39:26.873058+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":468,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787627+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:39:25.787627+0000","last_peered":"2023-10-09T06:39:25.787627+0000","last_clean":"2023-10-09T06:39:25.787627+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:39:25.787627+0000","last_undegraded":"2023-10-09T06:39:25.787627+0000","last_fullsized":"2023-10-09T06:39:25.787627+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":523,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873753+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:39:26.873753+0000","last_peered":"2023-10-09T06:39:26.873753+0000","last_clean":"2023-10-09T06:39:26.873753+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:39:26.873753+0000","last_undegraded":"2023-10-09T06:39:26.873753+0000","last_fullsized":"2023-10-09T06:39:26.873753+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":445,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784266+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:39:25.784266+0000","last_peered":"2023-10-09T06:39:25.784266+0000","last_clean":"2023-10-09T06:39:25.784266+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:39:25.784266+0000","last_undegraded":"2023-10-09T06:39:25.784266+0000","last_fullsized":"2023-10-09T06:39:25.784266+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":465,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.791045+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:39:25.791045+0000","last_peered":"2023-10-09T06:39:25.791045+0000","last_clean":"2023-10-09T06:39:25.791045+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:39:25.791045+0000","last_undegraded":"2023-10-09T06:39:25.791045+0000","last_fullsized":"2023-10-09T06:39:25.791045+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":825,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233582+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:39:27.233582+0000","last_peered":"2023-10-09T06:39:27.233582+0000","last_clean":"2023-10-09T06:39:27.233582+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:39:27.233582+0000","last_undegraded":"2023-10-09T06:39:27.233582+0000","last_fullsized":"2023-10-09T06:39:27.233582+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":469,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788574+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:39:25.788574+0000","last_peered":"2023-10-09T06:39:25.788574+0000","last_clean":"2023-10-09T06:39:25.788574+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:39:25.788574+0000","last_undegraded":"2023-10-09T06:39:25.788574+0000","last_fullsized":"2023-10-09T06:39:25.788574+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":448,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.792193+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:39:25.792193+0000","last_peered":"2023-10-09T06:39:25.792193+0000","last_clean":"2023-10-09T06:39:25.792193+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:39:25.792193+0000","last_undegraded":"2023-10-09T06:39:25.792193+0000","last_fullsized":"2023-10-09T06:39:25.792193+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.923471+0000","last_change":"2023-10-09T06:39:33.922114+0000","last_active":"2023-10-09T06:39:33.923471+0000","last_peered":"2023-10-09T06:39:33.923471+0000","last_clean":"2023-10-09T06:39:33.923471+0000","last_became_active":"2023-10-09T06:39:33.921838+0000","last_became_peered":"2023-10-09T06:39:33.921838+0000","last_unstale":"2023-10-09T06:39:33.923471+0000","last_undegraded":"2023-10-09T06:39:33.923471+0000","last_fullsized":"2023-10-09T06:39:33.923471+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:43.308955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":450,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784215+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:39:25.784215+0000","last_peered":"2023-10-09T06:39:25.784215+0000","last_clean":"2023-10-09T06:39:25.784215+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:39:25.784215+0000","last_undegraded":"2023-10-09T06:39:25.784215+0000","last_fullsized":"2023-10-09T06:39:25.784215+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.872951+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:39:26.872951+0000","last_peered":"2023-10-09T06:39:26.872951+0000","last_clean":"2023-10-09T06:39:26.872951+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:39:26.872951+0000","last_undegraded":"2023-10-09T06:39:26.872951+0000","last_fullsized":"2023-10-09T06:39:26.872951+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":444,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233879+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:39:27.233879+0000","last_peered":"2023-10-09T06:39:27.233879+0000","last_clean":"2023-10-09T06:39:27.233879+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:39:27.233879+0000","last_undegraded":"2023-10-09T06:39:27.233879+0000","last_fullsized":"2023-10-09T06:39:27.233879+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.923478+0000","last_change":"2023-10-09T06:39:33.922898+0000","last_active":"2023-10-09T06:39:33.923478+0000","last_peered":"2023-10-09T06:39:33.923478+0000","last_clean":"2023-10-09T06:39:33.923478+0000","last_became_active":"2023-10-09T06:39:33.922619+0000","last_became_peered":"2023-10-09T06:39:33.922619+0000","last_unstale":"2023-10-09T06:39:33.923478+0000","last_undegraded":"2023-10-09T06:39:33.923478+0000","last_fullsized":"2023-10-09T06:39:33.923478+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.252997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":780,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786506+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:39:25.786506+0000","last_peered":"2023-10-09T06:39:25.786506+0000","last_clean":"2023-10-09T06:39:25.786506+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:39:25.786506+0000","last_undegraded":"2023-10-09T06:39:25.786506+0000","last_fullsized":"2023-10-09T06:39:25.786506+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":496,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233835+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:39:27.233835+0000","last_peered":"2023-10-09T06:39:27.233835+0000","last_clean":"2023-10-09T06:39:27.233835+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:39:27.233835+0000","last_undegraded":"2023-10-09T06:39:27.233835+0000","last_fullsized":"2023-10-09T06:39:27.233835+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":59,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873380+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:39:26.873380+0000","last_peered":"2023-10-09T06:39:26.873380+0000","last_clean":"2023-10-09T06:39:26.873380+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:39:26.873380+0000","last_undegraded":"2023-10-09T06:39:26.873380+0000","last_fullsized":"2023-10-09T06:39:26.873380+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":430,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233600+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:39:27.233600+0000","last_peered":"2023-10-09T06:39:27.233600+0000","last_clean":"2023-10-09T06:39:27.233600+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:39:27.233600+0000","last_undegraded":"2023-10-09T06:39:27.233600+0000","last_fullsized":"2023-10-09T06:39:27.233600+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":669,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.911563+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:39:32.911563+0000","last_peered":"2023-10-09T06:39:32.911563+0000","last_clean":"2023-10-09T06:39:32.911563+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:39:32.911563+0000","last_undegraded":"2023-10-09T06:39:32.911563+0000","last_fullsized":"2023-10-09T06:39:32.911563+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":220,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233603+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:39:27.233603+0000","last_peered":"2023-10-09T06:39:27.233603+0000","last_clean":"2023-10-09T06:39:27.233603+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:39:27.233603+0000","last_undegraded":"2023-10-09T06:39:27.233603+0000","last_fullsized":"2023-10-09T06:39:27.233603+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":469,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790583+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:39:25.790583+0000","last_peered":"2023-10-09T06:39:25.790583+0000","last_clean":"2023-10-09T06:39:25.790583+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:39:25.790583+0000","last_undegraded":"2023-10-09T06:39:25.790583+0000","last_fullsized":"2023-10-09T06:39:25.790583+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":460,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233824+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:39:27.233824+0000","last_peered":"2023-10-09T06:39:27.233824+0000","last_clean":"2023-10-09T06:39:27.233824+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:39:27.233824+0000","last_undegraded":"2023-10-09T06:39:27.233824+0000","last_fullsized":"2023-10-09T06:39:27.233824+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":830,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.106972+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:39:26.106972+0000","last_peered":"2023-10-09T06:39:26.106972+0000","last_clean":"2023-10-09T06:39:26.106972+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:39:26.106972+0000","last_undegraded":"2023-10-09T06:39:26.106972+0000","last_fullsized":"2023-10-09T06:39:26.106972+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":482,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.872995+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:39:26.872995+0000","last_peered":"2023-10-09T06:39:26.872995+0000","last_clean":"2023-10-09T06:39:26.872995+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:39:26.872995+0000","last_undegraded":"2023-10-09T06:39:26.872995+0000","last_fullsized":"2023-10-09T06:39:26.872995+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":469,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233581+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:39:27.233581+0000","last_peered":"2023-10-09T06:39:27.233581+0000","last_clean":"2023-10-09T06:39:27.233581+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:39:27.233581+0000","last_undegraded":"2023-10-09T06:39:27.233581+0000","last_fullsized":"2023-10-09T06:39:27.233581+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":491,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233976+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:39:27.233976+0000","last_peered":"2023-10-09T06:39:27.233976+0000","last_clean":"2023-10-09T06:39:27.233976+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:39:27.233976+0000","last_undegraded":"2023-10-09T06:39:27.233976+0000","last_fullsized":"2023-10-09T06:39:27.233976+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T06:39:32.904263+0000","last_change":"2023-10-09T06:39:32.904263+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.904263+0000","last_undegraded":"2023-10-09T06:39:32.904263+0000","last_fullsized":"2023-10-09T06:39:32.904263+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":297,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107101+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:39:26.107101+0000","last_peered":"2023-10-09T06:39:26.107101+0000","last_clean":"2023-10-09T06:39:26.107101+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:39:26.107101+0000","last_undegraded":"2023-10-09T06:39:26.107101+0000","last_fullsized":"2023-10-09T06:39:26.107101+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":811,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107654+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:39:26.107654+0000","last_peered":"2023-10-09T06:39:26.107654+0000","last_clean":"2023-10-09T06:39:26.107654+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:39:26.107654+0000","last_undegraded":"2023-10-09T06:39:26.107654+0000","last_fullsized":"2023-10-09T06:39:26.107654+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":200,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873138+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:39:26.873138+0000","last_peered":"2023-10-09T06:39:26.873138+0000","last_clean":"2023-10-09T06:39:26.873138+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:39:26.873138+0000","last_undegraded":"2023-10-09T06:39:26.873138+0000","last_fullsized":"2023-10-09T06:39:26.873138+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":450,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784610+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:39:25.784610+0000","last_peered":"2023-10-09T06:39:25.784610+0000","last_clean":"2023-10-09T06:39:25.784610+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:39:25.784610+0000","last_undegraded":"2023-10-09T06:39:25.784610+0000","last_fullsized":"2023-10-09T06:39:25.784610+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.920154+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:39:33.920154+0000","last_peered":"2023-10-09T06:39:33.920154+0000","last_clean":"2023-10-09T06:39:33.920154+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:39:33.920154+0000","last_undegraded":"2023-10-09T06:39:33.920154+0000","last_fullsized":"2023-10-09T06:39:33.920154+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":799,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233606+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:39:27.233606+0000","last_peered":"2023-10-09T06:39:27.233606+0000","last_clean":"2023-10-09T06:39:27.233606+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:39:27.233606+0000","last_undegraded":"2023-10-09T06:39:27.233606+0000","last_fullsized":"2023-10-09T06:39:27.233606+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":351,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107002+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:39:26.107002+0000","last_peered":"2023-10-09T06:39:26.107002+0000","last_clean":"2023-10-09T06:39:26.107002+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:39:26.107002+0000","last_undegraded":"2023-10-09T06:39:26.107002+0000","last_fullsized":"2023-10-09T06:39:26.107002+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":165,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.911484+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:39:32.911484+0000","last_peered":"2023-10-09T06:39:32.911484+0000","last_clean":"2023-10-09T06:39:32.911484+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:39:32.911484+0000","last_undegraded":"2023-10-09T06:39:32.911484+0000","last_fullsized":"2023-10-09T06:39:32.911484+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":476,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913816+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:39:32.913816+0000","last_peered":"2023-10-09T06:39:32.913816+0000","last_clean":"2023-10-09T06:39:32.913816+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:39:32.913816+0000","last_undegraded":"2023-10-09T06:39:32.913816+0000","last_fullsized":"2023-10-09T06:39:32.913816+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":1,"reported_epoch":489,"state":"creating+peering","last_fresh":"2023-10-09T06:39:32.916151+0000","last_change":"2023-10-09T06:39:32.916151+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.916151+0000","last_undegraded":"2023-10-09T06:39:32.916151+0000","last_fullsized":"2023-10-09T06:39:32.916151+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:18.487828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":810,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.784597+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:39:25.784597+0000","last_peered":"2023-10-09T06:39:25.784597+0000","last_clean":"2023-10-09T06:39:25.784597+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:39:25.784597+0000","last_undegraded":"2023-10-09T06:39:25.784597+0000","last_fullsized":"2023-10-09T06:39:25.784597+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":351,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.108387+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:39:26.108387+0000","last_peered":"2023-10-09T06:39:26.108387+0000","last_clean":"2023-10-09T06:39:26.108387+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:39:26.108387+0000","last_undegraded":"2023-10-09T06:39:26.108387+0000","last_fullsized":"2023-10-09T06:39:26.108387+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":486,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233742+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:39:27.233742+0000","last_peered":"2023-10-09T06:39:27.233742+0000","last_clean":"2023-10-09T06:39:27.233742+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:39:27.233742+0000","last_undegraded":"2023-10-09T06:39:27.233742+0000","last_fullsized":"2023-10-09T06:39:27.233742+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":474,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.912784+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:39:32.912784+0000","last_peered":"2023-10-09T06:39:32.912784+0000","last_clean":"2023-10-09T06:39:32.912784+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:39:32.912784+0000","last_undegraded":"2023-10-09T06:39:32.912784+0000","last_fullsized":"2023-10-09T06:39:32.912784+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":1,"reported_epoch":489,"state":"creating+peering","last_fresh":"2023-10-09T06:39:32.914170+0000","last_change":"2023-10-09T06:39:32.914170+0000","last_active":"2023-10-09T06:39:32.904263+0000","last_peered":"2023-10-09T06:39:32.904263+0000","last_clean":"2023-10-09T06:39:32.904263+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T06:39:32.914170+0000","last_undegraded":"2023-10-09T06:39:32.914170+0000","last_fullsized":"2023-10-09T06:39:32.914170+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":811,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788042+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:39:25.788042+0000","last_peered":"2023-10-09T06:39:25.788042+0000","last_clean":"2023-10-09T06:39:25.788042+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:39:25.788042+0000","last_undegraded":"2023-10-09T06:39:25.788042+0000","last_fullsized":"2023-10-09T06:39:25.788042+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":221,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873818+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:39:26.873818+0000","last_peered":"2023-10-09T06:39:26.873818+0000","last_clean":"2023-10-09T06:39:26.873818+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:39:26.873818+0000","last_undegraded":"2023-10-09T06:39:26.873818+0000","last_fullsized":"2023-10-09T06:39:26.873818+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":454,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233962+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:39:27.233962+0000","last_peered":"2023-10-09T06:39:27.233962+0000","last_clean":"2023-10-09T06:39:27.233962+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:39:27.233962+0000","last_undegraded":"2023-10-09T06:39:27.233962+0000","last_fullsized":"2023-10-09T06:39:27.233962+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":209,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.872836+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:39:26.872836+0000","last_peered":"2023-10-09T06:39:26.872836+0000","last_clean":"2023-10-09T06:39:26.872836+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:39:26.872836+0000","last_undegraded":"2023-10-09T06:39:26.872836+0000","last_fullsized":"2023-10-09T06:39:26.872836+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":10,"reported_epoch":490,"state":"active+clean","last_fresh":"2023-10-09T06:39:33.928705+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:39:33.928705+0000","last_peered":"2023-10-09T06:39:33.928705+0000","last_clean":"2023-10-09T06:39:33.928705+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:39:33.928705+0000","last_undegraded":"2023-10-09T06:39:33.928705+0000","last_fullsized":"2023-10-09T06:39:33.928705+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":843,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873492+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:39:26.873492+0000","last_peered":"2023-10-09T06:39:26.873492+0000","last_clean":"2023-10-09T06:39:26.873492+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:39:26.873492+0000","last_undegraded":"2023-10-09T06:39:26.873492+0000","last_fullsized":"2023-10-09T06:39:26.873492+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":472,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.786641+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:39:25.786641+0000","last_peered":"2023-10-09T06:39:25.786641+0000","last_clean":"2023-10-09T06:39:25.786641+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:39:25.786641+0000","last_undegraded":"2023-10-09T06:39:25.786641+0000","last_fullsized":"2023-10-09T06:39:25.786641+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":457,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913662+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:39:32.913662+0000","last_peered":"2023-10-09T06:39:32.913662+0000","last_clean":"2023-10-09T06:39:32.913662+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:39:32.913662+0000","last_undegraded":"2023-10-09T06:39:32.913662+0000","last_fullsized":"2023-10-09T06:39:32.913662+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":451,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.788696+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:39:25.788696+0000","last_peered":"2023-10-09T06:39:25.788696+0000","last_clean":"2023-10-09T06:39:25.788696+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:39:25.788696+0000","last_undegraded":"2023-10-09T06:39:25.788696+0000","last_fullsized":"2023-10-09T06:39:25.788696+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":770,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.917010+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:39:32.917010+0000","last_peered":"2023-10-09T06:39:32.917010+0000","last_clean":"2023-10-09T06:39:32.917010+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:39:32.917010+0000","last_undegraded":"2023-10-09T06:39:32.917010+0000","last_fullsized":"2023-10-09T06:39:32.917010+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":458,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.912738+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:39:32.912738+0000","last_peered":"2023-10-09T06:39:32.912738+0000","last_clean":"2023-10-09T06:39:32.912738+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:39:32.912738+0000","last_undegraded":"2023-10-09T06:39:32.912738+0000","last_fullsized":"2023-10-09T06:39:32.912738+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":479,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233739+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:39:27.233739+0000","last_peered":"2023-10-09T06:39:27.233739+0000","last_clean":"2023-10-09T06:39:27.233739+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:39:27.233739+0000","last_undegraded":"2023-10-09T06:39:27.233739+0000","last_fullsized":"2023-10-09T06:39:27.233739+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":700,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873168+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:39:26.873168+0000","last_peered":"2023-10-09T06:39:26.873168+0000","last_clean":"2023-10-09T06:39:26.873168+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:39:26.873168+0000","last_undegraded":"2023-10-09T06:39:26.873168+0000","last_fullsized":"2023-10-09T06:39:26.873168+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":166,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.787719+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:39:25.787719+0000","last_peered":"2023-10-09T06:39:25.787719+0000","last_clean":"2023-10-09T06:39:25.787719+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:39:25.787719+0000","last_undegraded":"2023-10-09T06:39:25.787719+0000","last_fullsized":"2023-10-09T06:39:25.787719+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":495,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.790507+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:39:25.790507+0000","last_peered":"2023-10-09T06:39:25.790507+0000","last_clean":"2023-10-09T06:39:25.790507+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:39:25.790507+0000","last_undegraded":"2023-10-09T06:39:25.790507+0000","last_fullsized":"2023-10-09T06:39:25.790507+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":166,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.791135+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:39:25.791135+0000","last_peered":"2023-10-09T06:39:25.791135+0000","last_clean":"2023-10-09T06:39:25.791135+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:39:25.791135+0000","last_undegraded":"2023-10-09T06:39:25.791135+0000","last_fullsized":"2023-10-09T06:39:25.791135+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":505,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789603+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:39:25.789603+0000","last_peered":"2023-10-09T06:39:25.789603+0000","last_clean":"2023-10-09T06:39:25.789603+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:39:25.789603+0000","last_undegraded":"2023-10-09T06:39:25.789603+0000","last_fullsized":"2023-10-09T06:39:25.789603+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":844,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.107378+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:39:26.107378+0000","last_peered":"2023-10-09T06:39:26.107378+0000","last_clean":"2023-10-09T06:39:26.107378+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:39:26.107378+0000","last_undegraded":"2023-10-09T06:39:26.107378+0000","last_fullsized":"2023-10-09T06:39:26.107378+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":191,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873629+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:39:26.873629+0000","last_peered":"2023-10-09T06:39:26.873629+0000","last_clean":"2023-10-09T06:39:26.873629+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:39:26.873629+0000","last_undegraded":"2023-10-09T06:39:26.873629+0000","last_fullsized":"2023-10-09T06:39:26.873629+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":208,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:26.873128+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:39:26.873128+0000","last_peered":"2023-10-09T06:39:26.873128+0000","last_clean":"2023-10-09T06:39:26.873128+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:39:26.873128+0000","last_undegraded":"2023-10-09T06:39:26.873128+0000","last_fullsized":"2023-10-09T06:39:26.873128+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":611,"reported_epoch":488,"state":"active+clean","last_fresh":"2023-10-09T06:39:27.233911+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:39:27.233911+0000","last_peered":"2023-10-09T06:39:27.233911+0000","last_clean":"2023-10-09T06:39:27.233911+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:39:27.233911+0000","last_undegraded":"2023-10-09T06:39:27.233911+0000","last_fullsized":"2023-10-09T06:39:27.233911+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":487,"reported_epoch":489,"state":"active+clean","last_fresh":"2023-10-09T06:39:32.913621+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:39:32.913621+0000","last_peered":"2023-10-09T06:39:32.913621+0000","last_clean":"2023-10-09T06:39:32.913621+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:39:32.913621+0000","last_undegraded":"2023-10-09T06:39:32.913621+0000","last_fullsized":"2023-10-09T06:39:32.913621+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":448,"reported_epoch":487,"state":"active+clean","last_fresh":"2023-10-09T06:39:25.789425+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:39:25.789425+0000","last_peered":"2023-10-09T06:39:25.789425+0000","last_clean":"2023-10-09T06:39:25.789425+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:39:25.789425+0000","last_undegraded":"2023-10-09T06:39:25.789425+0000","last_fullsized":"2023-10-09T06:39:25.789425+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":30,"acting":30,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234507,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1513108,"kb_used_data":12108,"kb_used_omap":0,"kb_used_meta":1500992,"kb_avail":92232044,"statfs":{"total":95995035648,"available":94445613056,"internally_reserved":0,"allocated":12398592,"data_stored":11333589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537015808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":1.4510000000000001,"15min":4.3040000000000003},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.218},"max":{"1min":8.0050000000000008,"5min":8.8819999999999997,"15min":215.97800000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.4870000000000001,"15min":4.3499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":10.449999999999999,"5min":10.449999999999999,"15min":215.577},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":3.0350000000000001,"5min":6.9569999999999999,"15min":5.3869999999999996},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23200000000000001},"max":{"1min":9.7919999999999998,"5min":208.626,"15min":215.61699999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":3.1589999999999998,"5min":5.0759999999999996,"15min":4.5380000000000003},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":9.6679999999999993,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":20.891999999999999,"5min":9.2080000000000002,"15min":7.4980000000000002},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":228.24000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":21,"5min":9.1869999999999994,"15min":7.5990000000000002},"min":{"1min":0.33100000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":228.291,"5min":228.291,"15min":228.291},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":10.587999999999999,"5min":5.6100000000000003,"15min":5.4560000000000004},"min":{"1min":0.33300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":88.897999999999996,"5min":175.40600000000001,"15min":215.922},"last":0.59199999999999997},{"interface":"front","average":{"1min":11.07,"5min":5.6189999999999998,"15min":5.4779999999999998},"min":{"1min":0.46400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":88.926000000000002,"5min":175.30099999999999,"15min":215.94900000000001},"last":0.61299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":2.4489999999999998,"15min":2.8239999999999998},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.0429999999999993,"5min":73.688999999999993,"15min":218.446},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.1640000000000001,"15min":2.6360000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.23699999999999999},"max":{"1min":3.7770000000000001,"5min":71.831000000000003,"15min":216.172},"last":0.55500000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":3.2210000000000001,"15min":2.7530000000000001},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":12.499000000000001,"5min":71.116,"15min":216.37200000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.712,"5min":2.9809999999999999,"15min":2.7530000000000001},"min":{"1min":0.41499999999999998,"5min":0.312,"15min":0.28999999999999998},"max":{"1min":8.1839999999999993,"5min":70.984999999999999,"15min":218.15899999999999},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":3.2690000000000001,"15min":2.9409999999999998},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":8.3409999999999993,"5min":71.087999999999994,"15min":218.19999999999999},"last":0.623},{"interface":"front","average":{"1min":1.1699999999999999,"5min":3.1190000000000002,"15min":2.6739999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1420000000000003,"5min":71.149000000000001,"15min":218.05500000000001},"last":0.63200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496143,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899900,"kb_used_data":4980,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92845252,"statfs":{"total":95995035648,"available":95073538048,"internally_reserved":0,"allocated":5099520,"data_stored":4057815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.2170000000000001,"15min":2.5009999999999999},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":5.9569999999999999,"5min":66.867000000000004,"15min":75.061999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":13.164,"5min":4.8049999999999997,"15min":3.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":212.096,"5min":212.096,"15min":212.096},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.738,"5min":4.6589999999999998,"15min":5.0199999999999996},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":20.050000000000001,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.5369999999999999,"5min":4.6719999999999997,"15min":5.024},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.676,"5min":219.94800000000001,"15min":292.904},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":4.6600000000000001,"15min":3.5720000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":10.032999999999999,"5min":185.654,"15min":185.654},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.024,"5min":4.5049999999999999,"15min":3.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.29399999999999998,"15min":0.27100000000000002},"max":{"1min":6.0960000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":3.5939999999999999,"15min":3.7570000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":8.5380000000000003,"5min":121.749,"15min":121.749},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.5049999999999999,"15min":3.6829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.4770000000000003,"5min":120.70399999999999,"15min":120.70399999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.3359999999999999,"15min":1.5629999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":10.009,"5min":86.171999999999997,"15min":86.171999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.609,"15min":1.581},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.223},"max":{"1min":8.2759999999999998,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.32800000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.696,"15min":1.3500000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":9.984,"5min":9.984,"15min":12.747999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.6970000000000001,"5min":2.1560000000000001,"15min":1.599},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":9.0060000000000002,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.944,"5min":3.0070000000000001,"15min":2.798},"min":{"1min":0.307,"5min":0.307,"15min":0.26600000000000001},"max":{"1min":8.8059999999999992,"5min":165.37,"15min":165.37},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.5670000000000002,"15min":3.0880000000000001},"min":{"1min":0.36399999999999999,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":8.5980000000000008,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.433}]}]},{"osd":1,"up_from":110,"seq":472446402880,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":967448,"kb_used_data":5840,"kb_used_omap":4,"kb_used_meta":961595,"kb_avail":92777704,"statfs":{"total":95995035648,"available":95004368896,"internally_reserved":0,"allocated":5980160,"data_stored":3860607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":984674190},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":6.851,"5min":2.9369999999999998,"15min":5.992},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":110.479,"5min":110.479,"15min":269.79199999999997},"last":0.251},{"interface":"front","average":{"1min":8.1400000000000006,"5min":3.1259999999999999,"15min":6.4020000000000001},"min":{"1min":0.23599999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":110.59999999999999,"5min":110.59999999999999,"15min":271.20999999999998},"last":0.42599999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.625,"15min":3.1850000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":6.6020000000000003,"5min":13.217000000000001,"15min":139.75399999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":3.1509999999999998,"5min":2.0870000000000002,"15min":3.8940000000000001},"min":{"1min":0.307,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":8.968,"5min":12.244,"15min":140.29900000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.6839999999999999,"15min":2.6779999999999999},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":3.1139999999999999,"5min":12.329000000000001,"15min":31.062000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.266,"15min":1.76},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":1.972,"5min":10.228999999999999,"15min":27.657},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":3.4569999999999999,"15min":3.597},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27500000000000002},"max":{"1min":5.4820000000000002,"5min":120.309,"15min":120.309},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":3.3530000000000002,"15min":3.3519999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":7.9900000000000002,"5min":120.297,"15min":120.297},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.9369999999999998,"15min":2.9359999999999999},"min":{"1min":0.39600000000000002,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":15.83,"5min":119.408,"15min":119.408},"last":0.64300000000000002},{"interface":"front","average":{"1min":3.3250000000000002,"5min":4.1779999999999999,"15min":3.2959999999999998},"min":{"1min":0.40600000000000003,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":7.9470000000000001,"5min":119.483,"15min":119.483},"last":0.65700000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.9820000000000002,"15min":3.403},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":8.0129999999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.0089999999999999,"5min":3.1070000000000002,"15min":3.6419999999999999},"min":{"1min":0.42099999999999999,"5min":0.379,"15min":0.316},"max":{"1min":5.5750000000000002,"5min":121.559,"15min":121.559},"last":0.49099999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":4.3049999999999997,"15min":4.2549999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":6.7000000000000002,"5min":123.20699999999999,"15min":123.20699999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.0379999999999998,"5min":3.9540000000000002,"15min":3.649},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":6.718,"5min":123.48999999999999,"15min":123.48999999999999},"last":0.47499999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024314,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1028584,"kb_used_data":6560,"kb_used_omap":5,"kb_used_meta":1022010,"kb_avail":92716568,"statfs":{"total":95995035648,"available":94941765632,"internally_reserved":0,"allocated":6717440,"data_stored":4244095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1046538874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:38:44 2023","interfaces":[{"interface":"back","average":{"1min":4.2859999999999996,"5min":3.8889999999999998,"15min":3.1179999999999999},"min":{"1min":0.23200000000000001,"5min":0.184,"15min":0.183},"max":{"1min":42.613,"5min":94.808999999999997,"15min":191.446},"last":0.39600000000000002},{"interface":"front","average":{"1min":3.8700000000000001,"5min":4.4880000000000004,"15min":3.552},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.187},"max":{"1min":43.963000000000001,"5min":94.876999999999995,"15min":191.393},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 06:38:56 2023","interfaces":[{"interface":"back","average":{"1min":7.7439999999999998,"5min":6.1040000000000001,"15min":3.8999999999999999},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":97.650000000000006,"5min":100.425,"15min":100.425},"last":0.32600000000000001},{"interface":"front","average":{"1min":8.4260000000000002,"5min":6.1600000000000001,"15min":4.226},"min":{"1min":0.26200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":96.082999999999998,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":7.5549999999999997,"5min":3.9510000000000001,"15min":5.0010000000000003},"min":{"1min":0.313,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":95.555000000000007,"5min":95.555000000000007,"15min":208.11199999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":7.7720000000000002,"5min":4.133,"15min":5.0060000000000002},"min":{"1min":0.32800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":97.605000000000004,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":5.9390000000000001,"15min":3.02},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":11.519,"5min":200.654,"15min":200.654},"last":0.57899999999999996},{"interface":"front","average":{"1min":3.2050000000000001,"5min":6.6100000000000003,"15min":3.5059999999999998},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.0790000000000006,"5min":200.86600000000001,"15min":200.86600000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":2.6989999999999998,"15min":2.0129999999999999},"min":{"1min":0.32900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.036,"5min":80.885999999999996,"15min":80.885999999999996},"last":0.621},{"interface":"front","average":{"1min":1.875,"5min":2.605,"15min":2.2210000000000001},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.2089999999999996,"5min":80.438999999999993,"15min":80.438999999999993},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":3.3069999999999999,"15min":3.2559999999999998},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":7.1660000000000004,"5min":118.056,"15min":120.572},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.448,"5min":3.1509999999999998,"15min":2.8929999999999998},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.563000000000001,"5min":118.274,"15min":120.64400000000001},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":2.2200000000000002,"15min":2.5670000000000002},"min":{"1min":0.497,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":7.0270000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.601,"5min":2.6579999999999999,"15min":2.9009999999999998},"min":{"1min":0.54100000000000004,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9190000000000005,"5min":22.683,"15min":122.32299999999999},"last":0.497}]}]},{"osd":2,"up_from":128,"seq":549755814183,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888820,"kb_used_data":6060,"kb_used_omap":7,"kb_used_meta":882744,"kb_avail":92856332,"statfs":{"total":95995035648,"available":95084883968,"internally_reserved":0,"allocated":6205440,"data_stored":4139672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":903929937},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":7.5289999999999999,"15min":5.8550000000000004},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.19400000000000001},"max":{"1min":15.731999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":0.317},{"interface":"front","average":{"1min":2.552,"5min":7.056,"15min":5.1829999999999998},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":13.795,"5min":156.922,"15min":168.49700000000001},"last":0.35399999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":5.952,"15min":5.101},"min":{"1min":0.21299999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2370000000000001,"5min":140.452,"15min":205.148},"last":0.439},{"interface":"front","average":{"1min":1.958,"5min":6.1539999999999999,"15min":5.2329999999999997},"min":{"1min":0.34699999999999998,"5min":0.224,"15min":0.183},"max":{"1min":7.6470000000000002,"5min":139.75299999999999,"15min":205.267},"last":0.27800000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:29 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.3029999999999999,"15min":3.548},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":7.6870000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.004,"5min":3.3860000000000001,"15min":3.899},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":8.8510000000000009,"5min":79.614000000000004,"15min":91.685000000000002},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":3.4820000000000002,"15min":2.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.407,"5min":79.643000000000001,"15min":90.938999999999993},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.25,"5min":3.7240000000000002,"15min":3.1520000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.26900000000000002},"max":{"1min":2.9039999999999999,"5min":82.872,"15min":90.950000000000003},"last":0.69499999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":3.75,"15min":2.4729999999999999},"min":{"1min":0.51500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4320000000000004,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.339,"5min":3.8809999999999998,"15min":2.5630000000000002},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.8620000000000001,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.65200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":3.8149999999999999,"15min":3.9529999999999998},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.30199999999999999},"max":{"1min":3.7919999999999998,"5min":82.899000000000001,"15min":138.291},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.0640000000000001,"5min":3.8900000000000001,"15min":3.891},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":8.1349999999999998,"5min":80.364999999999995,"15min":138.01599999999999},"last":0.61899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":4.0190000000000001,"15min":3.1230000000000002},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.1809999999999992,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.403,"5min":3.387,"15min":2.7759999999999998},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":4.4530000000000003,"5min":82.798000000000002,"15min":82.798000000000002},"last":0.56999999999999995}]}]},{"osd":3,"up_from":149,"seq":639950127380,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":784632,"kb_used_data":6000,"kb_used_omap":2,"kb_used_meta":778621,"kb_avail":92960520,"statfs":{"total":95995035648,"available":95191572480,"internally_reserved":0,"allocated":6144000,"data_stored":4177675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":797307957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8700000000000001,"5min":2.0419999999999998,"15min":4.6059999999999999},"min":{"1min":0.216,"5min":0.191,"15min":0.19},"max":{"1min":11.317,"5min":22.367000000000001,"15min":161.821},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.8100000000000001,"5min":2.3260000000000001,"15min":4.3579999999999997},"min":{"1min":0.25700000000000001,"5min":0.20599999999999999,"15min":0.17799999999999999},"max":{"1min":11.233000000000001,"5min":13.443,"15min":163.52000000000001},"last":0.24399999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":3.4849999999999999,"5min":4.8120000000000003,"15min":3.9159999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":11.279999999999999,"5min":184.012,"15min":184.012},"last":0.38200000000000001},{"interface":"front","average":{"1min":3.2029999999999998,"5min":5.141,"15min":4.7000000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":11.442,"5min":184.703,"15min":184.703},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":4.0960000000000001,"15min":3.7040000000000002},"min":{"1min":0.26400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":11.725,"5min":182.44900000000001,"15min":182.44900000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.8879999999999999,"5min":4.3079999999999998,"15min":4.173},"min":{"1min":0.25,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":11.41,"5min":182.56399999999999,"15min":182.56399999999999},"last":0.35599999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":4.3360000000000003,"15min":2.9369999999999998},"min":{"1min":0.32000000000000001,"5min":0.318,"15min":0.26900000000000002},"max":{"1min":8.6690000000000005,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.432},{"interface":"front","average":{"1min":2.7829999999999999,"5min":4.3819999999999997,"15min":2.9049999999999998},"min":{"1min":0.35099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.416,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":5.3419999999999996,"15min":3.165},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":10.472,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.0609999999999999,"5min":4.1390000000000002,"15min":2.4900000000000002},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":6.657,"5min":177.59399999999999,"15min":177.59399999999999},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.2370000000000001,"15min":2.806},"min":{"1min":0.39000000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.505000000000001,"5min":16.462,"15min":130.39599999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.0870000000000002,"5min":1.8149999999999999,"15min":2.355},"min":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":7.5949999999999998,"5min":22.260000000000002,"15min":130.67699999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":2.2829999999999999,"15min":2.8759999999999999},"min":{"1min":0.375,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.7110000000000003,"5min":13.695,"15min":121.613},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.9390000000000001,"5min":3.1539999999999999,"15min":3.2559999999999998},"min":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.33400000000000002},"max":{"1min":10.532,"5min":87.001999999999995,"15min":121.559},"last":0.53700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019416,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1320388,"kb_used_data":149308,"kb_used_omap":0,"kb_used_meta":1171072,"kb_avail":92424764,"statfs":{"total":95995035648,"available":94642958336,"internally_reserved":0,"allocated":152891392,"data_stored":151841872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1199177728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":4.7210000000000001,"15min":6.1859999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":3.0750000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":0.496},{"interface":"front","average":{"1min":1.298,"5min":5.3399999999999999,"15min":6.4199999999999999},"min":{"1min":0.317,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":4.202,"5min":286.36000000000001,"15min":287.57100000000003},"last":0.38800000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":3.5230000000000001,"15min":3.2389999999999999},"min":{"1min":0.247,"5min":0.246,"15min":0.22900000000000001},"max":{"1min":2.786,"5min":134.37100000000001,"15min":134.37100000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":3.5350000000000001,"15min":3.3570000000000002},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":3.4180000000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.372}]},{"osd":2,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":6.5839999999999996,"5min":6.0540000000000003,"15min":4.298},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":54.767000000000003,"5min":148.36099999999999,"15min":151.66399999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":6.2439999999999998,"5min":6.1929999999999996,"15min":4.2759999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":54.636000000000003,"5min":145.755,"15min":151.58699999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.5409999999999999,"15min":5.6559999999999997},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":6.8920000000000003,"5min":94.984999999999999,"15min":161.19300000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.411,"5min":3.1480000000000001,"15min":5.3979999999999997},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.524,"5min":94.966999999999999,"15min":161.934},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.3620000000000001,"15min":1.0589999999999999},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":4.2679999999999998,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.595,"5min":1.526,"15min":1.131},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.26600000000000001},"max":{"1min":4.1340000000000003,"5min":9.5009999999999994,"15min":9.5009999999999994},"last":0.34000000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.7400000000000002,"15min":1.958},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.2879999999999998,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.073,"5min":3.0670000000000002,"15min":2.258},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":4.3200000000000003,"5min":115.68600000000001,"15min":163.381},"last":0.40100000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4820000000000002,"15min":1.6950000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":3.8140000000000001,"5min":106.801,"15min":106.801},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.6030000000000002,"15min":1.722},"min":{"1min":0.40000000000000002,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":3.3300000000000001,"5min":106.827,"15min":106.827},"last":0.46899999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757781,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1688124,"kb_used_data":192244,"kb_used_omap":0,"kb_used_meta":1495872,"kb_avail":92057028,"statfs":{"total":95995035648,"available":94266396672,"internally_reserved":0,"allocated":196857856,"data_stored":195812780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1531772928},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":6.8890000000000002,"15min":6.0650000000000004},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.5899999999999999,"5min":149.149,"15min":294.89699999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.214,"5min":7.9660000000000002,"15min":6.202},"min":{"1min":0.24099999999999999,"5min":0.215,"15min":0.215},"max":{"1min":6.9029999999999996,"5min":149.21100000000001,"15min":294.30399999999997},"last":0.70599999999999996}]},{"osd":1,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":26.379000000000001,"5min":14.6,"15min":7.0659999999999998},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":153.91300000000001,"5min":188.34100000000001,"15min":188.34100000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":25.495999999999999,"5min":14.301,"15min":7.0369999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":154.023,"5min":188.839,"15min":188.839},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":14.561999999999999,"5min":7.6950000000000003,"15min":4.3739999999999997},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":100.46599999999999,"5min":120.613,"15min":120.613},"last":0.58399999999999996},{"interface":"front","average":{"1min":15.493,"5min":7.4169999999999998,"15min":4.2469999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":100.56999999999999,"5min":120.962,"15min":120.962},"last":0.629}]},{"osd":3,"last update":"Mon Oct 9 06:39:18 2023","interfaces":[{"interface":"back","average":{"1min":15.433,"5min":8.0549999999999997,"15min":6.6719999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":220.41800000000001,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":15.595000000000001,"5min":7.508,"15min":6.3470000000000004},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":220.45500000000001,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.59999999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":2.472,"15min":1.536},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":4.5330000000000004,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.907,"5min":2.6030000000000002,"15min":1.5109999999999999},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.25800000000000001},"max":{"1min":5.5170000000000003,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8480000000000001,"15min":1.655},"min":{"1min":0.35299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.5810000000000004,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1.9419999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0819999999999999,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.67400000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":2.4929999999999999,"15min":1.6850000000000001},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":11.146000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.1560000000000001,"15min":1.573},"min":{"1min":0.38900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.8070000000000004,"5min":79.138999999999996,"15min":79.138999999999996},"last":0.72899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:39:39.069 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:39:39.070 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:39.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:39.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:38 smithi150 ceph-mon[194178]: pgmap v1507: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:38 smithi150 ceph-mon[194178]: from='client.47018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:39.172 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.203 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:38 smithi107 ceph-mon[220921]: pgmap v1507: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:39.204 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:38 smithi107 ceph-mon[220921]: from='client.47018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:39.205 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:38 smithi107 ceph-mon[212750]: pgmap v1507: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:39.205 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:38 smithi107 ceph-mon[212750]: from='client.47018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:39.208 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:39.661 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:39.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:39.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:39.765 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:39.797 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:39.847 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:39.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:39.954 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:39.982 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:40.278 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:40.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:40.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:40.382 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:40.412 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:40.612 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:40.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:40.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:40.677 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:40.716 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:40.750 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:40.801 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:40.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:40.910 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:40.924 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:40.942 INFO:teuthology.orchestra.run.smithi107.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-10-09T06:39:40.962 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:40 smithi150 ceph-mon[194178]: pgmap v1508: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/375227456' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:41.173 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:39:41.174 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:39:41.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[212750]: pgmap v1508: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:41.197 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/375227456' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[220921]: pgmap v1508: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/375227456' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:39:41.198 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:39:41.945 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:39:42.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:42] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2136901110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2136901110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T06:39:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: osdmap e492: 8 total, 8 up, 8 in 2023-10-09T06:39:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: failed to encode map e492 with expected crc 2023-10-09T06:39:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2136901110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:39:42.622 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:42.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:42.727 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:42.755 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:42.850 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:42.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:42.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:42.986 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:43.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:39:43.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:42 smithi150 ceph-mon[194178]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:39:43.234 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: pgmap v1510: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: osdmap e493: 8 total, 8 up, 8 in 2023-10-09T06:39:43.235 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:42 smithi107 ceph-mon[220921]: failed to encode map e493 with expected crc 2023-10-09T06:39:43.567 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:39:43.991 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:43.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:44.096 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:44.123 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:44.146 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[220921]: failed to encode map e493 with expected crc 2023-10-09T06:39:44.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[212750]: failed to encode map e493 with expected crc 2023-10-09T06:39:44.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.147 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.167 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:44.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:44.272 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:44.300 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:43 smithi150 ceph-mon[194178]: failed to encode map e493 with expected crc 2023-10-09T06:39:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:44.501 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:44.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:44.606 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:44.615 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:44.633 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:44.676 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:44.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:44.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:44.814 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:44.832 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:39:44.832 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1997,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":493,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":8,"osd_in_since":1696832954,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":119}],"num_pgs":119,"num_pools":5,"num_objects":24,"data_bytes":595872,"bytes_used":9262776320,"bytes_avail":758697508864,"bytes_total":767960285184},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":29,"modified":"2023-10-09T06:39:15.531037+0000","services":{}},"progress_events":{"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:39:44.863 DEBUG:teuthology.orchestra.run.smithi107:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-10-09T06:39:44.866 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T06:39:44.866 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T06:39:44.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[212750]: pgmap v1512: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:44.954 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2513752571' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:39:44.998 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:39:44.998 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:39:44.999 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:39:45.019 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:39:45.039 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:39:45.058 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:39:45.076 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:39:45.091 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:hints = 1 2023-10-09T06:39:45.091 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T06:39:45.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object prefix: benchmark_data_smithi107_590976 2023-10-09T06:39:45.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:39:45.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 0 0 0 0 0 0 - 0 2023-10-09T06:39:45.109 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:39:45.115 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.115 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.115 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.116 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.116 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.116 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.116 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.116 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:39:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/416991940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[220921]: pgmap v1512: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2513752571' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:39:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/416991940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:44 smithi150 ceph-mon[194178]: pgmap v1512: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2513752571' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:39:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/416991940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:39:46.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 1 16 2294 2278 142.367 142.375 0.00326779 0.00700335 2023-10-09T06:39:46.235 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:46.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:46.341 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:46.397 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:46.426 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:39:47.038 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:47.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:47.061 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T06:39:47.062 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:39:47.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 2 16 4661 4645 145.145 147.938 0.00311322 0.00688009 2023-10-09T06:39:47.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[220921]: pgmap v1513: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:47.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:47.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:47.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[212750]: pgmap v1513: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:47.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:47.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:47.146 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:47.186 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:47.243 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:47.246 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:46 smithi150 ceph-mon[194178]: pgmap v1513: 119 pgs: 119 active+clean; 582 KiB data, 8.6 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:47.246 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:47.246 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:39:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:47.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:47.348 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:47.427 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:48.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 3 16 6740 6724 140.072 129.938 0.00131096 0.00656067 2023-10-09T06:39:48.589 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:48.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:48.695 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:48.734 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:48.762 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:48.854 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:48.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:48.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:48.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:48 smithi107 ceph-mon[220921]: pgmap v1514: 119 pgs: 119 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:48.861 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:48.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:48 smithi107 ceph-mon[212750]: pgmap v1514: 119 pgs: 119 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:48.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:48 smithi150 ceph-mon[194178]: pgmap v1514: 119 pgs: 119 active+clean; 582 KiB data, 8.7 GiB used, 707 GiB / 715 GiB avail 2023-10-09T06:39:48.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:39:48.956 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:39:48.962 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:48.976 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:48.993 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:49.051 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:49.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:49.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 4 16 9275 9259 144.66 158.438 0.00265215 0.00690311 2023-10-09T06:39:49.157 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:39:49.221 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:49.824 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:39:49.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:49.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:39:49.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:49.970 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:50.029 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:50.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:50.092 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 5 16 11485 11469 143.351 138.125 0.00157107 0.00696044 2023-10-09T06:39:50.134 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:50.168 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:50.221 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:50.223 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:39:50.223 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:39:50.223 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:39:50.223 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:39:50.223 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:39:50.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:50.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:50.327 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.361 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:50.427 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:50.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:50.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:50.535 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:50.577 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:50.648 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:50.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:50 smithi107 ceph-mon[212750]: pgmap v1515: 119 pgs: 119 active+clean; 492 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T06:39:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:50.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:50.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:50.800 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:50.866 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:39:50.870 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:50 smithi150 ceph-mon[194178]: pgmap v1515: 119 pgs: 119 active+clean; 492 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T06:39:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:50.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:39:50.973 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:39:51.021 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:50 smithi107 ceph-mon[220921]: pgmap v1515: 119 pgs: 119 active+clean; 492 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-10-09T06:39:51.031 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:51.093 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 6 16 14048 14032 146.145 160.188 0.00388243 0.00683772 2023-10-09T06:39:52.094 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 7 16 15548 15532 138.659 93.75 0.00781231 0.00720492 2023-10-09T06:39:52.274 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:39:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:39:52] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.43.0" 2023-10-09T06:39:52.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:52 smithi150 ceph-mon[194178]: pgmap v1516: 119 pgs: 119 active+clean; 492 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:39:52.950 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:39:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:52.957 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:52 smithi107 ceph-mon[212750]: pgmap v1516: 119 pgs: 119 active+clean; 492 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:39:52.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:52 smithi107 ceph-mon[220921]: pgmap v1516: 119 pgs: 119 active+clean; 492 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T06:39:52.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:39:53.059 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:53.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 8 16 17164 17148 133.948 101 0.00699976 0.00745983 2023-10-09T06:39:53.131 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:54.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:39:54.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 9 16 19001 18985 131.797 114.812 0.00622241 0.0075815 2023-10-09T06:39:54.146 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:54.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:54.260 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:54.420 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:54.709 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:54.836 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:54.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:54.942 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:39:54.972 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:54 smithi150 ceph-mon[194178]: pgmap v1517: 119 pgs: 119 active+clean; 772 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:39:54.979 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:55.036 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:39:55.039 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:54 smithi107 ceph-mon[212750]: pgmap v1517: 119 pgs: 119 active+clean; 772 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:39:55.040 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:54 smithi107 ceph-mon[220921]: pgmap v1517: 119 pgs: 119 active+clean; 772 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:39:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:55.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:39:55.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 10 16 20717 20701 129.342 107.25 0.00409948 0.0077237 2023-10-09T06:39:55.143 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:55.233 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:55.690 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:55.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:55.709 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:39:55.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:55.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:55.896 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:55.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:56.002 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:39:56.014 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:39:56.032 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:39:56.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2080514408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.039 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:56.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2080514408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.060 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.060 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2080514408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.060 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:39:56.096 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 11 16 22471 22455 127.549 109.625 0.0090911 0.00783265 2023-10-09T06:39:56.098 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:39:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:56.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:39:56.205 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:56.234 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:56.291 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:56.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:56.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:39:56.464 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: pgmap v1518: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[212750]: from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: pgmap v1518: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: failed to encode map e494 with expected crc 2023-10-09T06:39:56.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:56 smithi107 ceph-mon[220921]: from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:57.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 12 16 24377 24361 126.846 119.125 0.00998766 0.00787886 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: pgmap v1518: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: failed to encode map e494 with expected crc 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: failed to encode map e494 with expected crc 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:39:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: osdmap e494: 8 total, 8 up, 8 in 2023-10-09T06:39:57.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: failed to encode map e494 with expected crc 2023-10-09T06:39:57.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: failed to encode map e494 with expected crc 2023-10-09T06:39:57.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:56 smithi150 ceph-mon[194178]: from='client.47060 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:39:58.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 13 16 26194 26178 125.823 113.562 0.00158468 0.00791108 2023-10-09T06:39:59.058 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:39:58 smithi107 ceph-mon[220921]: pgmap v1520: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:39:59.059 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:39:58 smithi107 ceph-mon[212750]: pgmap v1520: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:39:59.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 14 16 27658 27642 123.371 91.5 0.0174347 0.00809635 2023-10-09T06:39:59.115 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:39:59.144 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:39:59.153 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-10-09T06:39:55.683435+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1168119712,"num_objects":17839,"num_object_clones":0,"num_object_copies":53517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17876,"num_write_kb":1140189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22861,"ondisk_log_size":22861,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12154568,"kb_used_data":3400712,"kb_used_omap":20,"kb_used_meta":8753771,"kb_avail":737806648,"statfs":{"total":767960285184,"available":755514007552,"internally_reserved":0,"allocated":3482329088,"data_stored":3469454150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":8963862158},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,2,1,1,0,0,0,13],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1167523840,"num_objects":17815,"num_object_clones":0,"num_object_copies":53445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17815,"num_write_kb":1140160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.144258"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":787,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001374+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:39:41.001374+0000","last_peered":"2023-10-09T06:39:41.001374+0000","last_clean":"2023-10-09T06:39:41.001374+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:39:41.001374+0000","last_undegraded":"2023-10-09T06:39:41.001374+0000","last_fullsized":"2023-10-09T06:39:41.001374+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":472,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932907+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:39:40.932907+0000","last_peered":"2023-10-09T06:39:40.932907+0000","last_clean":"2023-10-09T06:39:40.932907+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:39:40.932907+0000","last_undegraded":"2023-10-09T06:39:40.932907+0000","last_fullsized":"2023-10-09T06:39:40.932907+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":194,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.997967+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:39:40.997967+0000","last_peered":"2023-10-09T06:39:40.997967+0000","last_clean":"2023-10-09T06:39:40.997967+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:39:40.997967+0000","last_undegraded":"2023-10-09T06:39:40.997967+0000","last_fullsized":"2023-10-09T06:39:40.997967+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":748,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932938+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:39:40.932938+0000","last_peered":"2023-10-09T06:39:40.932938+0000","last_clean":"2023-10-09T06:39:40.932938+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:39:40.932938+0000","last_undegraded":"2023-10-09T06:39:40.932938+0000","last_fullsized":"2023-10-09T06:39:40.932938+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":489,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.997915+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:39:40.997915+0000","last_peered":"2023-10-09T06:39:40.997915+0000","last_clean":"2023-10-09T06:39:40.997915+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:39:40.997915+0000","last_undegraded":"2023-10-09T06:39:40.997915+0000","last_fullsized":"2023-10-09T06:39:40.997915+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":493,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930443+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:39:40.930443+0000","last_peered":"2023-10-09T06:39:40.930443+0000","last_clean":"2023-10-09T06:39:40.930443+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:39:40.930443+0000","last_undegraded":"2023-10-09T06:39:40.930443+0000","last_fullsized":"2023-10-09T06:39:40.930443+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":740,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932858+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:39:40.932858+0000","last_peered":"2023-10-09T06:39:40.932858+0000","last_clean":"2023-10-09T06:39:40.932858+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:39:40.932858+0000","last_undegraded":"2023-10-09T06:39:40.932858+0000","last_fullsized":"2023-10-09T06:39:40.932858+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":213,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.598042+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:39:42.598042+0000","last_peered":"2023-10-09T06:39:42.598042+0000","last_clean":"2023-10-09T06:39:42.598042+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:39:42.598042+0000","last_undegraded":"2023-10-09T06:39:42.598042+0000","last_fullsized":"2023-10-09T06:39:42.598042+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":482,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003464+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:39:41.003464+0000","last_peered":"2023-10-09T06:39:41.003464+0000","last_clean":"2023-10-09T06:39:41.003464+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:39:41.003464+0000","last_undegraded":"2023-10-09T06:39:41.003464+0000","last_fullsized":"2023-10-09T06:39:41.003464+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":601,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002087+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:39:41.002087+0000","last_peered":"2023-10-09T06:39:41.002087+0000","last_clean":"2023-10-09T06:39:41.002087+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:39:41.002087+0000","last_undegraded":"2023-10-09T06:39:41.002087+0000","last_fullsized":"2023-10-09T06:39:41.002087+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930963+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:39:40.930963+0000","last_peered":"2023-10-09T06:39:40.930963+0000","last_clean":"2023-10-09T06:39:40.930963+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:39:40.930963+0000","last_undegraded":"2023-10-09T06:39:40.930963+0000","last_fullsized":"2023-10-09T06:39:40.930963+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000452+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:39:41.000452+0000","last_peered":"2023-10-09T06:39:41.000452+0000","last_clean":"2023-10-09T06:39:41.000452+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:39:41.000452+0000","last_undegraded":"2023-10-09T06:39:41.000452+0000","last_fullsized":"2023-10-09T06:39:41.000452+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":638,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002124+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:39:41.002124+0000","last_peered":"2023-10-09T06:39:41.002124+0000","last_clean":"2023-10-09T06:39:41.002124+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:39:41.002124+0000","last_undegraded":"2023-10-09T06:39:41.002124+0000","last_fullsized":"2023-10-09T06:39:41.002124+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":180,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000035+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:39:41.000035+0000","last_peered":"2023-10-09T06:39:41.000035+0000","last_clean":"2023-10-09T06:39:41.000035+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:39:41.000035+0000","last_undegraded":"2023-10-09T06:39:41.000035+0000","last_fullsized":"2023-10-09T06:39:41.000035+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":195,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000090+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:39:41.000090+0000","last_peered":"2023-10-09T06:39:41.000090+0000","last_clean":"2023-10-09T06:39:41.000090+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:39:41.000090+0000","last_undegraded":"2023-10-09T06:39:41.000090+0000","last_fullsized":"2023-10-09T06:39:41.000090+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":621,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930684+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:39:40.930684+0000","last_peered":"2023-10-09T06:39:40.930684+0000","last_clean":"2023-10-09T06:39:40.930684+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:39:40.930684+0000","last_undegraded":"2023-10-09T06:39:40.930684+0000","last_fullsized":"2023-10-09T06:39:40.930684+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":374,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002377+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:39:41.002377+0000","last_peered":"2023-10-09T06:39:41.002377+0000","last_clean":"2023-10-09T06:39:41.002377+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:39:41.002377+0000","last_undegraded":"2023-10-09T06:39:41.002377+0000","last_fullsized":"2023-10-09T06:39:41.002377+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":430,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000157+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:39:41.000157+0000","last_peered":"2023-10-09T06:39:41.000157+0000","last_clean":"2023-10-09T06:39:41.000157+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:39:41.000157+0000","last_undegraded":"2023-10-09T06:39:41.000157+0000","last_fullsized":"2023-10-09T06:39:41.000157+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":602,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999161+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:39:40.999161+0000","last_peered":"2023-10-09T06:39:40.999161+0000","last_clean":"2023-10-09T06:39:40.999161+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:39:40.999161+0000","last_undegraded":"2023-10-09T06:39:40.999161+0000","last_fullsized":"2023-10-09T06:39:40.999161+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":499,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933258+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:39:40.933258+0000","last_peered":"2023-10-09T06:39:40.933258+0000","last_clean":"2023-10-09T06:39:40.933258+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:39:40.933258+0000","last_undegraded":"2023-10-09T06:39:40.933258+0000","last_fullsized":"2023-10-09T06:39:40.933258+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":463,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933292+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:39:40.933292+0000","last_peered":"2023-10-09T06:39:40.933292+0000","last_clean":"2023-10-09T06:39:40.933292+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:39:40.933292+0000","last_undegraded":"2023-10-09T06:39:40.933292+0000","last_fullsized":"2023-10-09T06:39:40.933292+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":766,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.929332+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:39:40.929332+0000","last_peered":"2023-10-09T06:39:40.929332+0000","last_clean":"2023-10-09T06:39:40.929332+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:39:40.929332+0000","last_undegraded":"2023-10-09T06:39:40.929332+0000","last_fullsized":"2023-10-09T06:39:40.929332+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":342,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001911+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:39:41.001911+0000","last_peered":"2023-10-09T06:39:41.001911+0000","last_clean":"2023-10-09T06:39:41.001911+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:39:41.001911+0000","last_undegraded":"2023-10-09T06:39:41.001911+0000","last_fullsized":"2023-10-09T06:39:41.001911+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998119+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:39:40.998119+0000","last_peered":"2023-10-09T06:39:40.998119+0000","last_clean":"2023-10-09T06:39:40.998119+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:39:40.998119+0000","last_undegraded":"2023-10-09T06:39:40.998119+0000","last_fullsized":"2023-10-09T06:39:40.998119+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":765,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001391+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:39:41.001391+0000","last_peered":"2023-10-09T06:39:41.001391+0000","last_clean":"2023-10-09T06:39:41.001391+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:39:41.001391+0000","last_undegraded":"2023-10-09T06:39:41.001391+0000","last_fullsized":"2023-10-09T06:39:41.001391+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":491,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.929789+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:39:40.929789+0000","last_peered":"2023-10-09T06:39:40.929789+0000","last_clean":"2023-10-09T06:39:40.929789+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:39:40.929789+0000","last_undegraded":"2023-10-09T06:39:40.929789+0000","last_fullsized":"2023-10-09T06:39:40.929789+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":313,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002283+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:39:41.002283+0000","last_peered":"2023-10-09T06:39:41.002283+0000","last_clean":"2023-10-09T06:39:41.002283+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:39:41.002283+0000","last_undegraded":"2023-10-09T06:39:41.002283+0000","last_fullsized":"2023-10-09T06:39:41.002283+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":820,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998131+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:39:40.998131+0000","last_peered":"2023-10-09T06:39:40.998131+0000","last_clean":"2023-10-09T06:39:40.998131+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:39:40.998131+0000","last_undegraded":"2023-10-09T06:39:40.998131+0000","last_fullsized":"2023-10-09T06:39:40.998131+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":336,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001724+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:39:41.001724+0000","last_peered":"2023-10-09T06:39:41.001724+0000","last_clean":"2023-10-09T06:39:41.001724+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:39:41.001724+0000","last_undegraded":"2023-10-09T06:39:41.001724+0000","last_fullsized":"2023-10-09T06:39:41.001724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930210+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:39:40.930210+0000","last_peered":"2023-10-09T06:39:40.930210+0000","last_clean":"2023-10-09T06:39:40.930210+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:39:40.930210+0000","last_undegraded":"2023-10-09T06:39:40.930210+0000","last_fullsized":"2023-10-09T06:39:40.930210+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":607,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999934+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:39:40.999934+0000","last_peered":"2023-10-09T06:39:40.999934+0000","last_clean":"2023-10-09T06:39:40.999934+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:39:40.999934+0000","last_undegraded":"2023-10-09T06:39:40.999934+0000","last_fullsized":"2023-10-09T06:39:40.999934+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":571,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002056+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:39:41.002056+0000","last_peered":"2023-10-09T06:39:41.002056+0000","last_clean":"2023-10-09T06:39:41.002056+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:39:41.002056+0000","last_undegraded":"2023-10-09T06:39:41.002056+0000","last_fullsized":"2023-10-09T06:39:41.002056+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.599936+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:39:42.599936+0000","last_peered":"2023-10-09T06:39:42.599936+0000","last_clean":"2023-10-09T06:39:42.599936+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:39:42.599936+0000","last_undegraded":"2023-10-09T06:39:42.599936+0000","last_fullsized":"2023-10-09T06:39:42.599936+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"493'1035","reported_seq":1047,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.544885+0000","last_change":"2023-10-09T06:39:33.925861+0000","last_active":"2023-10-09T06:39:52.544885+0000","last_peered":"2023-10-09T06:39:52.544885+0000","last_clean":"2023-10-09T06:39:52.544885+0000","last_became_active":"2023-10-09T06:39:33.925623+0000","last_became_peered":"2023-10-09T06:39:33.925623+0000","last_unstale":"2023-10-09T06:39:52.544885+0000","last_undegraded":"2023-10-09T06:39:52.544885+0000","last_fullsized":"2023-10-09T06:39:52.544885+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1035,"log_dups_size":0,"ondisk_log_size":1035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:10.504728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1035,"num_write_kb":66240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":620,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002166+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:39:41.002166+0000","last_peered":"2023-10-09T06:39:41.002166+0000","last_clean":"2023-10-09T06:39:41.002166+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:39:41.002166+0000","last_undegraded":"2023-10-09T06:39:41.002166+0000","last_fullsized":"2023-10-09T06:39:41.002166+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":378,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002588+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:39:41.002588+0000","last_peered":"2023-10-09T06:39:41.002588+0000","last_clean":"2023-10-09T06:39:41.002588+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:39:41.002588+0000","last_undegraded":"2023-10-09T06:39:41.002588+0000","last_fullsized":"2023-10-09T06:39:41.002588+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001631+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:39:41.001631+0000","last_peered":"2023-10-09T06:39:41.001631+0000","last_clean":"2023-10-09T06:39:41.001631+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:39:41.001631+0000","last_undegraded":"2023-10-09T06:39:41.001631+0000","last_fullsized":"2023-10-09T06:39:41.001631+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"493'1170","reported_seq":1183,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.117945+0000","last_change":"2023-10-09T06:39:33.923263+0000","last_active":"2023-10-09T06:39:54.117945+0000","last_peered":"2023-10-09T06:39:54.117945+0000","last_clean":"2023-10-09T06:39:54.117945+0000","last_became_active":"2023-10-09T06:39:33.922687+0000","last_became_peered":"2023-10-09T06:39:33.922687+0000","last_unstale":"2023-10-09T06:39:54.117945+0000","last_undegraded":"2023-10-09T06:39:54.117945+0000","last_fullsized":"2023-10-09T06:39:54.117945+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1170,"log_dups_size":0,"ondisk_log_size":1170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:59.579546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76677120,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1170,"num_write_kb":74880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":879,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.997857+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:39:40.997857+0000","last_peered":"2023-10-09T06:39:40.997857+0000","last_clean":"2023-10-09T06:39:40.997857+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:39:40.997857+0000","last_undegraded":"2023-10-09T06:39:40.997857+0000","last_fullsized":"2023-10-09T06:39:40.997857+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":340,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002558+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:39:41.002558+0000","last_peered":"2023-10-09T06:39:41.002558+0000","last_clean":"2023-10-09T06:39:41.002558+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:39:41.002558+0000","last_undegraded":"2023-10-09T06:39:41.002558+0000","last_fullsized":"2023-10-09T06:39:41.002558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":490,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933055+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:39:40.933055+0000","last_peered":"2023-10-09T06:39:40.933055+0000","last_clean":"2023-10-09T06:39:40.933055+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:39:40.933055+0000","last_undegraded":"2023-10-09T06:39:40.933055+0000","last_fullsized":"2023-10-09T06:39:40.933055+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"493'1227","reported_seq":1240,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.101273+0000","last_change":"2023-10-09T06:39:33.927681+0000","last_active":"2023-10-09T06:39:54.101273+0000","last_peered":"2023-10-09T06:39:54.101273+0000","last_clean":"2023-10-09T06:39:54.101273+0000","last_became_active":"2023-10-09T06:39:33.927408+0000","last_became_peered":"2023-10-09T06:39:33.927408+0000","last_unstale":"2023-10-09T06:39:54.101273+0000","last_undegraded":"2023-10-09T06:39:54.101273+0000","last_fullsized":"2023-10-09T06:39:54.101273+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1227,"log_dups_size":0,"ondisk_log_size":1227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:59.401749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80412672,"num_objects":1227,"num_object_clones":0,"num_object_copies":3681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1227,"num_write_kb":78528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":794,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003289+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:39:41.003289+0000","last_peered":"2023-10-09T06:39:41.003289+0000","last_clean":"2023-10-09T06:39:41.003289+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:39:41.003289+0000","last_undegraded":"2023-10-09T06:39:41.003289+0000","last_fullsized":"2023-10-09T06:39:41.003289+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":396,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002242+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:39:41.002242+0000","last_peered":"2023-10-09T06:39:41.002242+0000","last_clean":"2023-10-09T06:39:41.002242+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:39:41.002242+0000","last_undegraded":"2023-10-09T06:39:41.002242+0000","last_fullsized":"2023-10-09T06:39:41.002242+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":469,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002825+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:39:41.002825+0000","last_peered":"2023-10-09T06:39:41.002825+0000","last_clean":"2023-10-09T06:39:41.002825+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:39:41.002825+0000","last_undegraded":"2023-10-09T06:39:41.002825+0000","last_fullsized":"2023-10-09T06:39:41.002825+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"493'1178","reported_seq":1191,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:53.873923+0000","last_change":"2023-10-09T06:39:33.924837+0000","last_active":"2023-10-09T06:39:53.873923+0000","last_peered":"2023-10-09T06:39:53.873923+0000","last_clean":"2023-10-09T06:39:53.873923+0000","last_became_active":"2023-10-09T06:39:33.924498+0000","last_became_peered":"2023-10-09T06:39:33.924498+0000","last_unstale":"2023-10-09T06:39:53.873923+0000","last_undegraded":"2023-10-09T06:39:53.873923+0000","last_fullsized":"2023-10-09T06:39:53.873923+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1178,"log_dups_size":0,"ondisk_log_size":1178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:35.001011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1178,"num_write_kb":75392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":617,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933466+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:39:40.933466+0000","last_peered":"2023-10-09T06:39:40.933466+0000","last_clean":"2023-10-09T06:39:40.933466+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:39:40.933466+0000","last_undegraded":"2023-10-09T06:39:40.933466+0000","last_fullsized":"2023-10-09T06:39:40.933466+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":63,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998207+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:39:40.998207+0000","last_peered":"2023-10-09T06:39:40.998207+0000","last_clean":"2023-10-09T06:39:40.998207+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:39:40.998207+0000","last_undegraded":"2023-10-09T06:39:40.998207+0000","last_fullsized":"2023-10-09T06:39:40.998207+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":456,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.941377+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:39:41.941377+0000","last_peered":"2023-10-09T06:39:41.941377+0000","last_clean":"2023-10-09T06:39:41.941377+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:39:41.941377+0000","last_undegraded":"2023-10-09T06:39:41.941377+0000","last_fullsized":"2023-10-09T06:39:41.941377+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"493'1216","reported_seq":1229,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.117762+0000","last_change":"2023-10-09T06:39:33.922928+0000","last_active":"2023-10-09T06:39:54.117762+0000","last_peered":"2023-10-09T06:39:54.117762+0000","last_clean":"2023-10-09T06:39:54.117762+0000","last_became_active":"2023-10-09T06:39:33.922653+0000","last_became_peered":"2023-10-09T06:39:33.922653+0000","last_unstale":"2023-10-09T06:39:54.117762+0000","last_undegraded":"2023-10-09T06:39:54.117762+0000","last_fullsized":"2023-10-09T06:39:54.117762+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1216,"log_dups_size":0,"ondisk_log_size":1216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:31.453742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1216,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":845,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001529+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:39:41.001529+0000","last_peered":"2023-10-09T06:39:41.001529+0000","last_clean":"2023-10-09T06:39:41.001529+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:39:41.001529+0000","last_undegraded":"2023-10-09T06:39:41.001529+0000","last_fullsized":"2023-10-09T06:39:41.001529+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":221,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998509+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:39:40.998509+0000","last_peered":"2023-10-09T06:39:40.998509+0000","last_clean":"2023-10-09T06:39:40.998509+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:39:40.998509+0000","last_undegraded":"2023-10-09T06:39:40.998509+0000","last_fullsized":"2023-10-09T06:39:40.998509+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":473,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933108+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:39:40.933108+0000","last_peered":"2023-10-09T06:39:40.933108+0000","last_clean":"2023-10-09T06:39:40.933108+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:39:40.933108+0000","last_undegraded":"2023-10-09T06:39:40.933108+0000","last_fullsized":"2023-10-09T06:39:40.933108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"493'1016","reported_seq":1028,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.540307+0000","last_change":"2023-10-09T06:39:33.929339+0000","last_active":"2023-10-09T06:39:52.540307+0000","last_peered":"2023-10-09T06:39:52.540307+0000","last_clean":"2023-10-09T06:39:52.540307+0000","last_became_active":"2023-10-09T06:39:33.929105+0000","last_became_peered":"2023-10-09T06:39:33.929105+0000","last_unstale":"2023-10-09T06:39:52.540307+0000","last_undegraded":"2023-10-09T06:39:52.540307+0000","last_fullsized":"2023-10-09T06:39:52.540307+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1016,"log_dups_size":0,"ondisk_log_size":1016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:24.427366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66584576,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1016,"num_write_kb":65024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":527,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000545+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:39:41.000545+0000","last_peered":"2023-10-09T06:39:41.000545+0000","last_clean":"2023-10-09T06:39:41.000545+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:39:41.000545+0000","last_undegraded":"2023-10-09T06:39:41.000545+0000","last_fullsized":"2023-10-09T06:39:41.000545+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":450,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002687+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:39:41.002687+0000","last_peered":"2023-10-09T06:39:41.002687+0000","last_clean":"2023-10-09T06:39:41.002687+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:39:41.002687+0000","last_undegraded":"2023-10-09T06:39:41.002687+0000","last_fullsized":"2023-10-09T06:39:41.002687+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":470,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933543+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:39:40.933543+0000","last_peered":"2023-10-09T06:39:40.933543+0000","last_clean":"2023-10-09T06:39:40.933543+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:39:40.933543+0000","last_undegraded":"2023-10-09T06:39:40.933543+0000","last_fullsized":"2023-10-09T06:39:40.933543+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"493'999","reported_seq":1011,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.539348+0000","last_change":"2023-10-09T06:39:33.924988+0000","last_active":"2023-10-09T06:39:52.539348+0000","last_peered":"2023-10-09T06:39:52.539348+0000","last_clean":"2023-10-09T06:39:52.539348+0000","last_became_active":"2023-10-09T06:39:33.924698+0000","last_became_peered":"2023-10-09T06:39:33.924698+0000","last_unstale":"2023-10-09T06:39:52.539348+0000","last_undegraded":"2023-10-09T06:39:52.539348+0000","last_fullsized":"2023-10-09T06:39:52.539348+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":999,"log_dups_size":0,"ondisk_log_size":999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:03.806961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":999,"num_write_kb":63936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":829,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998214+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:39:40.998214+0000","last_peered":"2023-10-09T06:39:40.998214+0000","last_clean":"2023-10-09T06:39:40.998214+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:39:40.998214+0000","last_undegraded":"2023-10-09T06:39:40.998214+0000","last_fullsized":"2023-10-09T06:39:40.998214+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":474,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932891+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:39:40.932891+0000","last_peered":"2023-10-09T06:39:40.932891+0000","last_clean":"2023-10-09T06:39:40.932891+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:39:40.932891+0000","last_undegraded":"2023-10-09T06:39:40.932891+0000","last_fullsized":"2023-10-09T06:39:40.932891+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":453,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933619+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:39:40.933619+0000","last_peered":"2023-10-09T06:39:40.933619+0000","last_clean":"2023-10-09T06:39:40.933619+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:39:40.933619+0000","last_undegraded":"2023-10-09T06:39:40.933619+0000","last_fullsized":"2023-10-09T06:39:40.933619+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"493'1163","reported_seq":1175,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.117574+0000","last_change":"2023-10-09T06:39:33.922114+0000","last_active":"2023-10-09T06:39:54.117574+0000","last_peered":"2023-10-09T06:39:54.117574+0000","last_clean":"2023-10-09T06:39:54.117574+0000","last_became_active":"2023-10-09T06:39:33.921838+0000","last_became_peered":"2023-10-09T06:39:33.921838+0000","last_unstale":"2023-10-09T06:39:54.117574+0000","last_undegraded":"2023-10-09T06:39:54.117574+0000","last_fullsized":"2023-10-09T06:39:54.117574+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1163,"log_dups_size":0,"ondisk_log_size":1163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:43.308955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1163,"num_write_kb":74432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":455,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002611+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:39:41.002611+0000","last_peered":"2023-10-09T06:39:41.002611+0000","last_clean":"2023-10-09T06:39:41.002611+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:39:41.002611+0000","last_undegraded":"2023-10-09T06:39:41.002611+0000","last_fullsized":"2023-10-09T06:39:41.002611+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":195,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.000355+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:39:41.000355+0000","last_peered":"2023-10-09T06:39:41.000355+0000","last_clean":"2023-10-09T06:39:41.000355+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:39:41.000355+0000","last_undegraded":"2023-10-09T06:39:41.000355+0000","last_fullsized":"2023-10-09T06:39:41.000355+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":448,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998544+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:39:40.998544+0000","last_peered":"2023-10-09T06:39:40.998544+0000","last_clean":"2023-10-09T06:39:40.998544+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:39:40.998544+0000","last_undegraded":"2023-10-09T06:39:40.998544+0000","last_fullsized":"2023-10-09T06:39:40.998544+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"493'1165","reported_seq":1176,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.101454+0000","last_change":"2023-10-09T06:39:33.922898+0000","last_active":"2023-10-09T06:39:54.101454+0000","last_peered":"2023-10-09T06:39:54.101454+0000","last_clean":"2023-10-09T06:39:54.101454+0000","last_became_active":"2023-10-09T06:39:33.922619+0000","last_became_peered":"2023-10-09T06:39:33.922619+0000","last_unstale":"2023-10-09T06:39:54.101454+0000","last_undegraded":"2023-10-09T06:39:54.101454+0000","last_fullsized":"2023-10-09T06:39:54.101454+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1165,"log_dups_size":0,"ondisk_log_size":1165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.252997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76349440,"num_objects":1165,"num_object_clones":0,"num_object_copies":3495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1165,"num_write_kb":74560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":785,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003502+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:39:41.003502+0000","last_peered":"2023-10-09T06:39:41.003502+0000","last_clean":"2023-10-09T06:39:41.003502+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:39:41.003502+0000","last_undegraded":"2023-10-09T06:39:41.003502+0000","last_fullsized":"2023-10-09T06:39:41.003502+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":500,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999680+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:39:40.999680+0000","last_peered":"2023-10-09T06:39:40.999680+0000","last_clean":"2023-10-09T06:39:40.999680+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:39:40.999680+0000","last_undegraded":"2023-10-09T06:39:40.999680+0000","last_fullsized":"2023-10-09T06:39:40.999680+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":63,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.997893+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:39:40.997893+0000","last_peered":"2023-10-09T06:39:40.997893+0000","last_clean":"2023-10-09T06:39:40.997893+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:39:40.997893+0000","last_undegraded":"2023-10-09T06:39:40.997893+0000","last_fullsized":"2023-10-09T06:39:40.997893+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":434,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999851+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:39:40.999851+0000","last_peered":"2023-10-09T06:39:40.999851+0000","last_clean":"2023-10-09T06:39:40.999851+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:39:40.999851+0000","last_undegraded":"2023-10-09T06:39:40.999851+0000","last_fullsized":"2023-10-09T06:39:40.999851+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"493'991","reported_seq":1003,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.544274+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:39:52.544274+0000","last_peered":"2023-10-09T06:39:52.544274+0000","last_clean":"2023-10-09T06:39:52.544274+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:39:52.544274+0000","last_undegraded":"2023-10-09T06:39:52.544274+0000","last_fullsized":"2023-10-09T06:39:52.544274+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":991,"log_dups_size":0,"ondisk_log_size":991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":991,"num_write_kb":63424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":672,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.929597+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:39:40.929597+0000","last_peered":"2023-10-09T06:39:40.929597+0000","last_clean":"2023-10-09T06:39:40.929597+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:39:40.929597+0000","last_undegraded":"2023-10-09T06:39:40.929597+0000","last_fullsized":"2023-10-09T06:39:40.929597+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":224,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999974+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:39:40.999974+0000","last_peered":"2023-10-09T06:39:40.999974+0000","last_clean":"2023-10-09T06:39:40.999974+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:39:40.999974+0000","last_undegraded":"2023-10-09T06:39:40.999974+0000","last_fullsized":"2023-10-09T06:39:40.999974+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":474,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933216+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:39:40.933216+0000","last_peered":"2023-10-09T06:39:40.933216+0000","last_clean":"2023-10-09T06:39:40.933216+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:39:40.933216+0000","last_undegraded":"2023-10-09T06:39:40.933216+0000","last_fullsized":"2023-10-09T06:39:40.933216+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":464,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999613+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:39:40.999613+0000","last_peered":"2023-10-09T06:39:40.999613+0000","last_clean":"2023-10-09T06:39:40.999613+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:39:40.999613+0000","last_undegraded":"2023-10-09T06:39:40.999613+0000","last_fullsized":"2023-10-09T06:39:40.999613+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"493'1011","reported_seq":1023,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.640837+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:39:52.640837+0000","last_peered":"2023-10-09T06:39:52.640837+0000","last_clean":"2023-10-09T06:39:52.640837+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:39:52.640837+0000","last_undegraded":"2023-10-09T06:39:52.640837+0000","last_fullsized":"2023-10-09T06:39:52.640837+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1011,"log_dups_size":0,"ondisk_log_size":1011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66256896,"num_objects":1011,"num_object_clones":0,"num_object_copies":3033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1011,"num_write_kb":64704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":835,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001721+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:39:41.001721+0000","last_peered":"2023-10-09T06:39:41.001721+0000","last_clean":"2023-10-09T06:39:41.001721+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:39:41.001721+0000","last_undegraded":"2023-10-09T06:39:41.001721+0000","last_fullsized":"2023-10-09T06:39:41.001721+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":486,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999950+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:39:40.999950+0000","last_peered":"2023-10-09T06:39:40.999950+0000","last_clean":"2023-10-09T06:39:40.999950+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:39:40.999950+0000","last_undegraded":"2023-10-09T06:39:40.999950+0000","last_fullsized":"2023-10-09T06:39:40.999950+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":473,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.997693+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:39:40.997693+0000","last_peered":"2023-10-09T06:39:40.997693+0000","last_clean":"2023-10-09T06:39:40.997693+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:39:40.997693+0000","last_undegraded":"2023-10-09T06:39:40.997693+0000","last_fullsized":"2023-10-09T06:39:40.997693+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":495,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998627+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:39:40.998627+0000","last_peered":"2023-10-09T06:39:40.998627+0000","last_clean":"2023-10-09T06:39:40.998627+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:39:40.998627+0000","last_undegraded":"2023-10-09T06:39:40.998627+0000","last_fullsized":"2023-10-09T06:39:40.998627+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"493'1019","reported_seq":1031,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:52.528826+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:39:52.528826+0000","last_peered":"2023-10-09T06:39:52.528826+0000","last_clean":"2023-10-09T06:39:52.528826+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:39:52.528826+0000","last_undegraded":"2023-10-09T06:39:52.528826+0000","last_fullsized":"2023-10-09T06:39:52.528826+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1019,"log_dups_size":0,"ondisk_log_size":1019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1019,"num_write_kb":65216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":302,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001859+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:39:41.001859+0000","last_peered":"2023-10-09T06:39:41.001859+0000","last_clean":"2023-10-09T06:39:41.001859+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:39:41.001859+0000","last_undegraded":"2023-10-09T06:39:41.001859+0000","last_fullsized":"2023-10-09T06:39:41.001859+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":816,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001587+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:39:41.001587+0000","last_peered":"2023-10-09T06:39:41.001587+0000","last_clean":"2023-10-09T06:39:41.001587+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:39:41.001587+0000","last_undegraded":"2023-10-09T06:39:41.001587+0000","last_fullsized":"2023-10-09T06:39:41.001587+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":204,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002127+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:39:41.002127+0000","last_peered":"2023-10-09T06:39:41.002127+0000","last_clean":"2023-10-09T06:39:41.002127+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:39:41.002127+0000","last_undegraded":"2023-10-09T06:39:41.002127+0000","last_fullsized":"2023-10-09T06:39:41.002127+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":455,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003257+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:39:41.003257+0000","last_peered":"2023-10-09T06:39:41.003257+0000","last_clean":"2023-10-09T06:39:41.003257+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:39:41.003257+0000","last_undegraded":"2023-10-09T06:39:41.003257+0000","last_fullsized":"2023-10-09T06:39:41.003257+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"493'1195","reported_seq":1208,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.118452+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:39:54.118452+0000","last_peered":"2023-10-09T06:39:54.118452+0000","last_clean":"2023-10-09T06:39:54.118452+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:39:54.118452+0000","last_undegraded":"2023-10-09T06:39:54.118452+0000","last_fullsized":"2023-10-09T06:39:54.118452+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1195,"log_dups_size":0,"ondisk_log_size":1195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78315520,"num_objects":1195,"num_object_clones":0,"num_object_copies":3585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1195,"num_write_kb":76480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":803,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998322+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:39:40.998322+0000","last_peered":"2023-10-09T06:39:40.998322+0000","last_clean":"2023-10-09T06:39:40.998322+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:39:40.998322+0000","last_undegraded":"2023-10-09T06:39:40.998322+0000","last_fullsized":"2023-10-09T06:39:40.998322+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":356,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.001484+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:39:41.001484+0000","last_peered":"2023-10-09T06:39:41.001484+0000","last_clean":"2023-10-09T06:39:41.001484+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:39:41.001484+0000","last_undegraded":"2023-10-09T06:39:41.001484+0000","last_fullsized":"2023-10-09T06:39:41.001484+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":169,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.929516+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:39:40.929516+0000","last_peered":"2023-10-09T06:39:40.929516+0000","last_clean":"2023-10-09T06:39:40.929516+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:39:40.929516+0000","last_undegraded":"2023-10-09T06:39:40.929516+0000","last_fullsized":"2023-10-09T06:39:40.929516+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":479,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930514+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:39:40.930514+0000","last_peered":"2023-10-09T06:39:40.930514+0000","last_clean":"2023-10-09T06:39:40.930514+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:39:40.930514+0000","last_undegraded":"2023-10-09T06:39:40.930514+0000","last_fullsized":"2023-10-09T06:39:40.930514+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"493'1166","reported_seq":1179,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:53.882226+0000","last_change":"2023-10-09T06:39:33.927704+0000","last_active":"2023-10-09T06:39:53.882226+0000","last_peered":"2023-10-09T06:39:53.882226+0000","last_clean":"2023-10-09T06:39:53.882226+0000","last_became_active":"2023-10-09T06:39:33.927409+0000","last_became_peered":"2023-10-09T06:39:33.927409+0000","last_unstale":"2023-10-09T06:39:53.882226+0000","last_undegraded":"2023-10-09T06:39:53.882226+0000","last_fullsized":"2023-10-09T06:39:53.882226+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1166,"log_dups_size":0,"ondisk_log_size":1166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:18.487828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76414976,"num_objects":1166,"num_object_clones":0,"num_object_copies":3498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1166,"num_write_kb":74624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":815,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003337+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:39:41.003337+0000","last_peered":"2023-10-09T06:39:41.003337+0000","last_clean":"2023-10-09T06:39:41.003337+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:39:41.003337+0000","last_undegraded":"2023-10-09T06:39:41.003337+0000","last_fullsized":"2023-10-09T06:39:41.003337+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":356,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002321+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:39:41.002321+0000","last_peered":"2023-10-09T06:39:41.002321+0000","last_clean":"2023-10-09T06:39:41.002321+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:39:41.002321+0000","last_undegraded":"2023-10-09T06:39:41.002321+0000","last_fullsized":"2023-10-09T06:39:41.002321+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":490,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999540+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:39:40.999540+0000","last_peered":"2023-10-09T06:39:40.999540+0000","last_clean":"2023-10-09T06:39:40.999540+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:39:40.999540+0000","last_undegraded":"2023-10-09T06:39:40.999540+0000","last_fullsized":"2023-10-09T06:39:40.999540+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":478,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930549+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:39:40.930549+0000","last_peered":"2023-10-09T06:39:40.930549+0000","last_clean":"2023-10-09T06:39:40.930549+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:39:40.930549+0000","last_undegraded":"2023-10-09T06:39:40.930549+0000","last_fullsized":"2023-10-09T06:39:40.930549+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"493'1073","reported_seq":1086,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:53.087700+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:39:53.087700+0000","last_peered":"2023-10-09T06:39:53.087700+0000","last_clean":"2023-10-09T06:39:53.087700+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:39:53.087700+0000","last_undegraded":"2023-10-09T06:39:53.087700+0000","last_fullsized":"2023-10-09T06:39:53.087700+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1073,"log_dups_size":0,"ondisk_log_size":1073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1073,"num_write_kb":68672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":816,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933474+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:39:40.933474+0000","last_peered":"2023-10-09T06:39:40.933474+0000","last_clean":"2023-10-09T06:39:40.933474+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:39:40.933474+0000","last_undegraded":"2023-10-09T06:39:40.933474+0000","last_fullsized":"2023-10-09T06:39:40.933474+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":225,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998187+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:39:40.998187+0000","last_peered":"2023-10-09T06:39:40.998187+0000","last_clean":"2023-10-09T06:39:40.998187+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:39:40.998187+0000","last_undegraded":"2023-10-09T06:39:40.998187+0000","last_fullsized":"2023-10-09T06:39:40.998187+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":458,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999354+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:39:40.999354+0000","last_peered":"2023-10-09T06:39:40.999354+0000","last_clean":"2023-10-09T06:39:40.999354+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:39:40.999354+0000","last_undegraded":"2023-10-09T06:39:40.999354+0000","last_fullsized":"2023-10-09T06:39:40.999354+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":214,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.598269+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:39:42.598269+0000","last_peered":"2023-10-09T06:39:42.598269+0000","last_clean":"2023-10-09T06:39:42.598269+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:39:42.598269+0000","last_undegraded":"2023-10-09T06:39:42.598269+0000","last_fullsized":"2023-10-09T06:39:42.598269+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"493'1191","reported_seq":1204,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:54.092759+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:39:54.092759+0000","last_peered":"2023-10-09T06:39:54.092759+0000","last_clean":"2023-10-09T06:39:54.092759+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:39:54.092759+0000","last_undegraded":"2023-10-09T06:39:54.092759+0000","last_fullsized":"2023-10-09T06:39:54.092759+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":1191,"log_dups_size":0,"ondisk_log_size":1191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1191,"num_write_kb":76224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":847,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998324+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:39:40.998324+0000","last_peered":"2023-10-09T06:39:40.998324+0000","last_clean":"2023-10-09T06:39:40.998324+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:39:40.998324+0000","last_undegraded":"2023-10-09T06:39:40.998324+0000","last_fullsized":"2023-10-09T06:39:40.998324+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":477,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.003547+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:39:41.003547+0000","last_peered":"2023-10-09T06:39:41.003547+0000","last_clean":"2023-10-09T06:39:41.003547+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:39:41.003547+0000","last_undegraded":"2023-10-09T06:39:41.003547+0000","last_fullsized":"2023-10-09T06:39:41.003547+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":460,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930365+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:39:40.930365+0000","last_peered":"2023-10-09T06:39:40.930365+0000","last_clean":"2023-10-09T06:39:40.930365+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:39:40.930365+0000","last_undegraded":"2023-10-09T06:39:40.930365+0000","last_fullsized":"2023-10-09T06:39:40.930365+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":456,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932976+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:39:40.932976+0000","last_peered":"2023-10-09T06:39:40.932976+0000","last_clean":"2023-10-09T06:39:40.932976+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:39:40.932976+0000","last_undegraded":"2023-10-09T06:39:40.932976+0000","last_fullsized":"2023-10-09T06:39:40.932976+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":774,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999844+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:39:40.999844+0000","last_peered":"2023-10-09T06:39:40.999844+0000","last_clean":"2023-10-09T06:39:40.999844+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:39:40.999844+0000","last_undegraded":"2023-10-09T06:39:40.999844+0000","last_fullsized":"2023-10-09T06:39:40.999844+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":461,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930479+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:39:40.930479+0000","last_peered":"2023-10-09T06:39:40.930479+0000","last_clean":"2023-10-09T06:39:40.930479+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:39:40.930479+0000","last_undegraded":"2023-10-09T06:39:40.930479+0000","last_fullsized":"2023-10-09T06:39:40.930479+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":483,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.999471+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:39:40.999471+0000","last_peered":"2023-10-09T06:39:40.999471+0000","last_clean":"2023-10-09T06:39:40.999471+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:39:40.999471+0000","last_undegraded":"2023-10-09T06:39:40.999471+0000","last_fullsized":"2023-10-09T06:39:40.999471+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":704,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.998262+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:39:40.998262+0000","last_peered":"2023-10-09T06:39:40.998262+0000","last_clean":"2023-10-09T06:39:40.998262+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:39:40.998262+0000","last_undegraded":"2023-10-09T06:39:40.998262+0000","last_fullsized":"2023-10-09T06:39:40.998262+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":171,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933216+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:39:40.933216+0000","last_peered":"2023-10-09T06:39:40.933216+0000","last_clean":"2023-10-09T06:39:40.933216+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:39:40.933216+0000","last_undegraded":"2023-10-09T06:39:40.933216+0000","last_fullsized":"2023-10-09T06:39:40.933216+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":500,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933214+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:39:40.933214+0000","last_peered":"2023-10-09T06:39:40.933214+0000","last_clean":"2023-10-09T06:39:40.933214+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:39:40.933214+0000","last_undegraded":"2023-10-09T06:39:40.933214+0000","last_fullsized":"2023-10-09T06:39:40.933214+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":171,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.933585+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:39:40.933585+0000","last_peered":"2023-10-09T06:39:40.933585+0000","last_clean":"2023-10-09T06:39:40.933585+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:39:40.933585+0000","last_undegraded":"2023-10-09T06:39:40.933585+0000","last_fullsized":"2023-10-09T06:39:40.933585+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":510,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932959+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:39:40.932959+0000","last_peered":"2023-10-09T06:39:40.932959+0000","last_clean":"2023-10-09T06:39:40.932959+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:39:40.932959+0000","last_undegraded":"2023-10-09T06:39:40.932959+0000","last_fullsized":"2023-10-09T06:39:40.932959+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":849,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:41.002140+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:39:41.002140+0000","last_peered":"2023-10-09T06:39:41.002140+0000","last_clean":"2023-10-09T06:39:41.002140+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:39:41.002140+0000","last_undegraded":"2023-10-09T06:39:41.002140+0000","last_fullsized":"2023-10-09T06:39:41.002140+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":196,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.599672+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:39:42.599672+0000","last_peered":"2023-10-09T06:39:42.599672+0000","last_clean":"2023-10-09T06:39:42.599672+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:39:42.599672+0000","last_undegraded":"2023-10-09T06:39:42.599672+0000","last_fullsized":"2023-10-09T06:39:42.599672+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.598886+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:39:42.598886+0000","last_peered":"2023-10-09T06:39:42.598886+0000","last_clean":"2023-10-09T06:39:42.598886+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:39:42.598886+0000","last_undegraded":"2023-10-09T06:39:42.598886+0000","last_fullsized":"2023-10-09T06:39:42.598886+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":616,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-10-09T06:39:42.599495+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:39:42.599495+0000","last_peered":"2023-10-09T06:39:42.599495+0000","last_clean":"2023-10-09T06:39:42.599495+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:39:42.599495+0000","last_undegraded":"2023-10-09T06:39:42.599495+0000","last_fullsized":"2023-10-09T06:39:42.599495+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":491,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.930884+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:39:40.930884+0000","last_peered":"2023-10-09T06:39:40.930884+0000","last_clean":"2023-10-09T06:39:40.930884+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:39:40.930884+0000","last_undegraded":"2023-10-09T06:39:40.930884+0000","last_fullsized":"2023-10-09T06:39:40.930884+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":453,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-10-09T06:39:40.932787+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:39:40.932787+0000","last_peered":"2023-10-09T06:39:40.932787+0000","last_clean":"2023-10-09T06:39:40.932787+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:39:40.932787+0000","last_undegraded":"2023-10-09T06:39:40.932787+0000","last_fullsized":"2023-10-09T06:39:40.932787+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1167523840,"num_objects":17815,"num_object_clones":0,"num_object_copies":53445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17815,"num_write_kb":1140160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3349872640,"data_stored":3349872640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17815,"ondisk_log_size":17815,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234511,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1692604,"kb_used_data":195700,"kb_used_omap":0,"kb_used_meta":1496896,"kb_avail":92052548,"statfs":{"total":95995035648,"available":94261809152,"internally_reserved":0,"allocated":200396800,"data_stored":199331473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532821504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":1.4510000000000001,"15min":4.3040000000000003},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.218},"max":{"1min":8.0050000000000008,"5min":8.8819999999999997,"15min":215.97800000000001},"last":1.629},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.4870000000000001,"15min":4.3499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":10.449999999999999,"5min":10.449999999999999,"15min":215.577},"last":1.7190000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":3.0350000000000001,"5min":6.9569999999999999,"15min":5.3869999999999996},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23200000000000001},"max":{"1min":9.7919999999999998,"5min":208.626,"15min":215.61699999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":3.1589999999999998,"5min":5.0759999999999996,"15min":4.5380000000000003},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.23300000000000001},"max":{"1min":9.6679999999999993,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:38:52 2023","interfaces":[{"interface":"back","average":{"1min":20.891999999999999,"5min":9.2080000000000002,"15min":7.4980000000000002},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":228.24000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":21,"5min":9.1869999999999994,"15min":7.5990000000000002},"min":{"1min":0.33100000000000002,"5min":0.308,"15min":0.28000000000000003},"max":{"1min":228.291,"5min":228.291,"15min":228.291},"last":2.73}]},{"osd":3,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":5.835,"15min":5.5499999999999998},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":25.295000000000002,"5min":175.40600000000001,"15min":215.922},"last":0.311},{"interface":"front","average":{"1min":2.5750000000000002,"5min":5.7489999999999997,"15min":5.5540000000000003},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":25.324999999999999,"5min":175.30099999999999,"15min":215.94900000000001},"last":0.70299999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.606,"5min":4.6929999999999996,"15min":3.5009999999999999},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":225.041,"5min":225.041,"15min":225.041},"last":0.441},{"interface":"front","average":{"1min":11.720000000000001,"5min":4.3959999999999999,"15min":3.343},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":225.244,"5min":225.244,"15min":225.244},"last":0.34899999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.055999999999999,"5min":5.2699999999999996,"15min":3.4220000000000002},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":219.923,"5min":219.923,"15min":219.923},"last":0.41999999999999998},{"interface":"front","average":{"1min":10.946999999999999,"5min":5.0259999999999998,"15min":3.3889999999999998},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.28999999999999998},"max":{"1min":219.86000000000001,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":10.978999999999999,"5min":5.3319999999999999,"15min":3.552},"min":{"1min":0.39700000000000002,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":222.179,"5min":222.179,"15min":222.179},"last":0.65300000000000002},{"interface":"front","average":{"1min":11.074999999999999,"5min":5.2140000000000004,"15min":3.3300000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":223.261,"5min":223.261,"15min":223.261},"last":0.66900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496147,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158564,"kb_used_data":251356,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92586588,"statfs":{"total":95995035648,"available":94808666112,"internally_reserved":0,"allocated":257388544,"data_stored":256346515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.2170000000000001,"15min":2.5009999999999999},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":5.9569999999999999,"5min":66.867000000000004,"15min":75.061999999999998},"last":1.514},{"interface":"front","average":{"1min":13.164,"5min":4.8049999999999997,"15min":3.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":212.096,"5min":212.096,"15min":212.096},"last":0.89900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.738,"5min":4.6589999999999998,"15min":5.0199999999999996},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":20.050000000000001,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.92900000000000005},{"interface":"front","average":{"1min":2.5369999999999999,"5min":4.6719999999999997,"15min":5.024},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.676,"5min":219.94800000000001,"15min":292.904},"last":0.63600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":4.6600000000000001,"15min":3.5720000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":10.032999999999999,"5min":185.654,"15min":185.654},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.024,"5min":4.5049999999999999,"15min":3.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.29399999999999998,"15min":0.27100000000000002},"max":{"1min":6.0960000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":1.1000000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":3.5939999999999999,"15min":3.7570000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":8.5380000000000003,"5min":121.749,"15min":121.749},"last":1.546},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.5049999999999999,"15min":3.6829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.4770000000000003,"5min":120.70399999999999,"15min":120.70399999999999},"last":1.569}]},{"osd":4,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.3359999999999999,"15min":1.5629999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":10.009,"5min":86.171999999999997,"15min":86.171999999999997},"last":1.198},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.609,"15min":1.581},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.223},"max":{"1min":8.2759999999999998,"5min":86.727000000000004,"15min":86.727000000000004},"last":0.60999999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.696,"15min":1.3500000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":9.984,"5min":9.984,"15min":12.747999999999999},"last":1.274},{"interface":"front","average":{"1min":2.6970000000000001,"5min":2.1560000000000001,"15min":1.599},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":9.0060000000000002,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":3.0640000000000001,"15min":2.7280000000000002},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":8.5050000000000008,"5min":165.37,"15min":165.37},"last":1.133},{"interface":"front","average":{"1min":0.57899999999999996,"5min":3.5030000000000001,"15min":3.0150000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.589,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.65700000000000003}]}]},{"osd":1,"up_from":110,"seq":472446402884,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1492432,"kb_used_data":522632,"kb_used_omap":4,"kb_used_meta":969787,"kb_avail":92252720,"statfs":{"total":95995035648,"available":94466785280,"internally_reserved":0,"allocated":535175168,"data_stored":533041095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":993062798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":6.851,"5min":2.9369999999999998,"15min":5.992},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":110.479,"5min":110.479,"15min":269.79199999999997},"last":5.5190000000000001},{"interface":"front","average":{"1min":8.1400000000000006,"5min":3.1259999999999999,"15min":6.4020000000000001},"min":{"1min":0.23599999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":110.59999999999999,"5min":110.59999999999999,"15min":271.20999999999998},"last":5.4160000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.625,"15min":3.1850000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":6.6020000000000003,"5min":13.217000000000001,"15min":139.75399999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":3.1509999999999998,"5min":2.0870000000000002,"15min":3.8940000000000001},"min":{"1min":0.307,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":8.968,"5min":12.244,"15min":140.29900000000001},"last":5.4610000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.6839999999999999,"15min":2.6779999999999999},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":3.1139999999999999,"5min":12.329000000000001,"15min":31.062000000000001},"last":1.522},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.266,"15min":1.76},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":1.972,"5min":10.228999999999999,"15min":27.657},"last":0.61599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":3.4569999999999999,"15min":3.597},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27500000000000002},"max":{"1min":5.4820000000000002,"5min":120.309,"15min":120.309},"last":1.3500000000000001},{"interface":"front","average":{"1min":2.6840000000000002,"5min":3.3530000000000002,"15min":3.3519999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":7.9900000000000002,"5min":120.297,"15min":120.297},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":3.9369999999999998,"15min":2.9359999999999999},"min":{"1min":0.39600000000000002,"5min":0.28599999999999998,"15min":0.28299999999999997},"max":{"1min":15.83,"5min":119.408,"15min":119.408},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.3250000000000002,"5min":4.1779999999999999,"15min":3.2959999999999998},"min":{"1min":0.40600000000000003,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":7.9470000000000001,"5min":119.483,"15min":119.483},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.669,"5min":2.9820000000000002,"15min":3.403},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.317},"max":{"1min":8.0129999999999999,"5min":121.60899999999999,"15min":121.60899999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.0089999999999999,"5min":3.1070000000000002,"15min":3.6419999999999999},"min":{"1min":0.42099999999999999,"5min":0.379,"15min":0.316},"max":{"1min":5.5750000000000002,"5min":121.559,"15min":121.559},"last":1.3120000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:38:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6469999999999998,"5min":4.3049999999999997,"15min":4.2549999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32500000000000001},"max":{"1min":6.7000000000000002,"5min":123.20699999999999,"15min":123.20699999999999},"last":1.3779999999999999},{"interface":"front","average":{"1min":2.0379999999999998,"5min":3.9540000000000002,"15min":3.649},"min":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.33100000000000002},"max":{"1min":6.718,"5min":123.48999999999999,"15min":123.48999999999999},"last":0.59199999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024318,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575576,"kb_used_data":561744,"kb_used_omap":5,"kb_used_meta":1013818,"kb_avail":92169576,"statfs":{"total":95995035648,"available":94381645824,"internally_reserved":0,"allocated":575225856,"data_stored":572738020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1038150266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,13],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.6980000000000004,"5min":4.5549999999999997,"15min":3.387},"min":{"1min":0.20699999999999999,"5min":0.184,"15min":0.183},"max":{"1min":102.182,"5min":102.182,"15min":191.446},"last":2.2959999999999998},{"interface":"front","average":{"1min":6.5830000000000002,"5min":4.9539999999999997,"15min":3.7930000000000001},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.187},"max":{"1min":101.282,"5min":101.282,"15min":191.393},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:38:56 2023","interfaces":[{"interface":"back","average":{"1min":7.7439999999999998,"5min":6.1040000000000001,"15min":3.8999999999999999},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":97.650000000000006,"5min":100.425,"15min":100.425},"last":0.38400000000000001},{"interface":"front","average":{"1min":8.4260000000000002,"5min":6.1600000000000001,"15min":4.226},"min":{"1min":0.26200000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":96.082999999999998,"5min":99.769999999999996,"15min":99.769999999999996},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":7.5549999999999997,"5min":3.9510000000000001,"15min":5.0010000000000003},"min":{"1min":0.313,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":95.555000000000007,"5min":95.555000000000007,"15min":208.11199999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":7.7720000000000002,"5min":4.133,"15min":5.0060000000000002},"min":{"1min":0.32800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":97.605000000000004,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.624}]},{"osd":4,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":5.9390000000000001,"15min":3.02},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":11.519,"5min":200.654,"15min":200.654},"last":0.80100000000000005},{"interface":"front","average":{"1min":3.2050000000000001,"5min":6.6100000000000003,"15min":3.5059999999999998},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.0790000000000006,"5min":200.86600000000001,"15min":200.86600000000001},"last":0.59299999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":2.6989999999999998,"15min":2.0129999999999999},"min":{"1min":0.32900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.036,"5min":80.885999999999996,"15min":80.885999999999996},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.875,"5min":2.605,"15min":2.2210000000000001},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.2089999999999996,"5min":80.438999999999993,"15min":80.438999999999993},"last":0.53200000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":3.3069999999999999,"15min":3.2559999999999998},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":7.1660000000000004,"5min":118.056,"15min":120.572},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.448,"5min":3.1509999999999998,"15min":2.8929999999999998},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.563000000000001,"5min":118.274,"15min":120.64400000000001},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":2.2200000000000002,"15min":2.5670000000000002},"min":{"1min":0.497,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":7.0270000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.65400000000000003},{"interface":"front","average":{"1min":3.601,"5min":2.6579999999999999,"15min":2.9009999999999998},"min":{"1min":0.54100000000000004,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9190000000000005,"5min":22.683,"15min":122.32299999999999},"last":0.58299999999999996}]}]},{"osd":2,"up_from":128,"seq":549755814186,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364812,"kb_used_data":423812,"kb_used_omap":7,"kb_used_meta":940984,"kb_avail":92380340,"statfs":{"total":95995035648,"available":94597468160,"internally_reserved":0,"allocated":433983488,"data_stored":431910466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":963567697},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":7.5289999999999999,"15min":5.8550000000000004},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.19400000000000001},"max":{"1min":15.731999999999999,"5min":156.94499999999999,"15min":162.21100000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":2.552,"5min":7.056,"15min":5.1829999999999998},"min":{"1min":0.25700000000000001,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":13.795,"5min":156.922,"15min":168.49700000000001},"last":0.50700000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":5.952,"15min":5.101},"min":{"1min":0.21299999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.2370000000000001,"5min":140.452,"15min":205.148},"last":1.093},{"interface":"front","average":{"1min":1.958,"5min":6.1539999999999999,"15min":5.2329999999999997},"min":{"1min":0.34699999999999998,"5min":0.224,"15min":0.183},"max":{"1min":7.6470000000000002,"5min":139.75299999999999,"15min":205.267},"last":1.5109999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:39:29 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.3029999999999999,"15min":3.548},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":7.6870000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.004,"5min":3.3860000000000001,"15min":3.899},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":8.8510000000000009,"5min":79.614000000000004,"15min":91.685000000000002},"last":0.41799999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":3.4820000000000002,"15min":2.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.407,"5min":79.643000000000001,"15min":90.938999999999993},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.25,"5min":3.7240000000000002,"15min":3.1520000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.26900000000000002},"max":{"1min":2.9039999999999999,"5min":82.872,"15min":90.950000000000003},"last":0.34399999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":3.75,"15min":2.4729999999999999},"min":{"1min":0.51500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.4320000000000004,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.339,"5min":3.8809999999999998,"15min":2.5630000000000002},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.8620000000000001,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":3.8149999999999999,"15min":3.9529999999999998},"min":{"1min":0.38200000000000001,"5min":0.313,"15min":0.30199999999999999},"max":{"1min":3.7919999999999998,"5min":82.899000000000001,"15min":138.291},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.0640000000000001,"5min":3.8900000000000001,"15min":3.891},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":8.1349999999999998,"5min":80.364999999999995,"15min":138.01599999999999},"last":0.97099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:39:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":4.0190000000000001,"15min":3.1230000000000002},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.1809999999999992,"5min":80.390000000000001,"15min":80.390000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.403,"5min":3.387,"15min":2.7759999999999998},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":4.4530000000000003,"5min":82.798000000000002,"15min":82.798000000000002},"last":0.49299999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127384,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257508,"kb_used_data":487068,"kb_used_omap":2,"kb_used_meta":770429,"kb_avail":92487644,"statfs":{"total":95995035648,"available":94707347456,"internally_reserved":0,"allocated":498757632,"data_stored":496780225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":788919349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.9410000000000001,"15min":4.2610000000000001},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.19},"max":{"1min":16.562000000000001,"5min":16.562000000000001,"15min":161.821},"last":2.3969999999999998},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.121,"15min":3.9940000000000002},"min":{"1min":0.192,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":6.6050000000000004,"5min":12.298,"15min":163.52000000000001},"last":2.036}]},{"osd":1,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":4.5129999999999999,"15min":3.863},"min":{"1min":0.27100000000000002,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":5.1630000000000003,"5min":184.012,"15min":184.012},"last":1.9450000000000001},{"interface":"front","average":{"1min":2.246,"5min":4.7699999999999996,"15min":4.6399999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":22.184000000000001,"5min":184.703,"15min":184.703},"last":1.353}]},{"osd":2,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":3.7320000000000002,"15min":3.5990000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":12.390000000000001,"5min":182.44900000000001,"15min":182.44900000000001},"last":2.4700000000000002},{"interface":"front","average":{"1min":2.1440000000000001,"5min":4.0220000000000002,"15min":4.0590000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":18.109000000000002,"5min":182.56399999999999,"15min":182.56399999999999},"last":1.29}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.625,"5min":5.5949999999999998,"15min":3.3519999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26900000000000002},"max":{"1min":133.80099999999999,"5min":178.22200000000001,"15min":178.22200000000001},"last":1.3200000000000001},{"interface":"front","average":{"1min":8.8919999999999995,"5min":5.6079999999999997,"15min":3.359},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":133.648,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.72399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4130000000000003,"5min":6.4509999999999996,"15min":3.5859999999999999},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":128.083,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":8.2870000000000008,"5min":5.3540000000000001,"15min":2.9510000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":128.11799999999999,"5min":177.59399999999999,"15min":177.59399999999999},"last":1.9790000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.016,"15min":2.7530000000000001},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.5779999999999994,"5min":11.994999999999999,"15min":130.39599999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.151,"5min":1.581,"15min":2.3199999999999998},"min":{"1min":0.38300000000000001,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":5.359,"5min":9.8179999999999996,"15min":130.67699999999999},"last":2.0049999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.0510000000000002,"15min":2.7930000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.7590000000000003,"5min":10.195,"15min":121.613},"last":1.377},{"interface":"front","average":{"1min":1.3140000000000001,"5min":2.8149999999999999,"15min":3.1779999999999999},"min":{"1min":0.42699999999999999,"5min":0.372,"15min":0.33400000000000002},"max":{"1min":5.1849999999999996,"5min":87.001999999999995,"15min":121.559},"last":0.755}]}]},{"osd":4,"up_from":30,"seq":128849019420,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1655660,"kb_used_data":480484,"kb_used_omap":0,"kb_used_meta":1175168,"kb_avail":92089492,"statfs":{"total":95995035648,"available":94299639808,"internally_reserved":0,"allocated":492015616,"data_stored":490965772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1203372032},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":4.7210000000000001,"15min":6.1859999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":3.0750000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.298,"5min":5.3399999999999999,"15min":6.4199999999999999},"min":{"1min":0.317,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":4.202,"5min":286.36000000000001,"15min":287.57100000000003},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":3.5230000000000001,"15min":3.2389999999999999},"min":{"1min":0.247,"5min":0.246,"15min":0.22900000000000001},"max":{"1min":2.786,"5min":134.37100000000001,"15min":134.37100000000001},"last":2.2629999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":3.5350000000000001,"15min":3.3570000000000002},"min":{"1min":0.309,"5min":0.27900000000000003,"15min":0.26000000000000001},"max":{"1min":3.4180000000000001,"5min":134.21700000000001,"15min":134.21700000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":6.5839999999999996,"5min":6.0540000000000003,"15min":4.298},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":54.767000000000003,"5min":148.36099999999999,"15min":151.66399999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":6.2439999999999998,"5min":6.1929999999999996,"15min":4.2759999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":54.636000000000003,"5min":145.755,"15min":151.58699999999999},"last":1.476}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.5409999999999999,"15min":5.6559999999999997},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":6.8920000000000003,"5min":94.984999999999999,"15min":161.19300000000001},"last":1.6850000000000001},{"interface":"front","average":{"1min":1.411,"5min":3.1480000000000001,"15min":5.3979999999999997},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.524,"5min":94.966999999999999,"15min":161.934},"last":0.48399999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:38:47 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.3620000000000001,"15min":1.0589999999999999},"min":{"1min":0.27500000000000002,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":4.2679999999999998,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.595,"5min":1.526,"15min":1.131},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.26600000000000001},"max":{"1min":4.1340000000000003,"5min":9.5009999999999994,"15min":9.5009999999999994},"last":0.33700000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.7400000000000002,"15min":1.958},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.2879999999999998,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.073,"5min":3.0670000000000002,"15min":2.258},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":4.3200000000000003,"5min":115.68600000000001,"15min":163.381},"last":0.42899999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4820000000000002,"15min":1.6950000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":3.8140000000000001,"5min":106.801,"15min":106.801},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.6030000000000002,"15min":1.722},"min":{"1min":0.40000000000000002,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":3.3300000000000001,"5min":106.827,"15min":106.827},"last":0.51500000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757785,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957412,"kb_used_data":477916,"kb_used_omap":0,"kb_used_meta":1479488,"kb_avail":91787740,"statfs":{"total":95995035648,"available":93990645760,"internally_reserved":0,"allocated":489385984,"data_stored":488340584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514995712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":6.8890000000000002,"15min":6.0650000000000004},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.5899999999999999,"5min":149.149,"15min":294.89699999999999},"last":1.526},{"interface":"front","average":{"1min":2.214,"5min":7.9660000000000002,"15min":6.202},"min":{"1min":0.24099999999999999,"5min":0.215,"15min":0.215},"max":{"1min":6.9029999999999996,"5min":149.21100000000001,"15min":294.30399999999997},"last":0.28299999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":26.379000000000001,"5min":14.6,"15min":7.0659999999999998},"min":{"1min":0.28499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":153.91300000000001,"5min":188.34100000000001,"15min":188.34100000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":25.495999999999999,"5min":14.301,"15min":7.0369999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":154.023,"5min":188.839,"15min":188.839},"last":2.073}]},{"osd":2,"last update":"Mon Oct 9 06:38:59 2023","interfaces":[{"interface":"back","average":{"1min":14.561999999999999,"5min":7.6950000000000003,"15min":4.3739999999999997},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":100.46599999999999,"5min":120.613,"15min":120.613},"last":1.137},{"interface":"front","average":{"1min":15.493,"5min":7.4169999999999998,"15min":4.2469999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":100.56999999999999,"5min":120.962,"15min":120.962},"last":1.5649999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:39:18 2023","interfaces":[{"interface":"back","average":{"1min":15.433,"5min":8.0549999999999997,"15min":6.6719999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":220.41800000000001,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":15.595000000000001,"5min":7.508,"15min":6.3470000000000004},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":220.45500000000001,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.61699999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":2.472,"15min":1.536},"min":{"1min":0.255,"5min":0.247,"15min":0.247},"max":{"1min":4.5330000000000004,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.907,"5min":2.6030000000000002,"15min":1.5109999999999999},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.25800000000000001},"max":{"1min":5.5170000000000003,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.34300000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8480000000000001,"15min":1.655},"min":{"1min":0.35299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.5810000000000004,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1.9419999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0819999999999999,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.45000000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.4710000000000001,"15min":1.6779999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":2.9430000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":2.1589999999999998,"15min":1.5349999999999999},"min":{"1min":0.38300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.859,"5min":79.138999999999996,"15min":79.138999999999996},"last":0.437}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":623312896,"data_stored":623312896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":563806208,"data_stored":563806208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":436535296,"data_stored":436535296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":532480000,"data_stored":532480000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":397737984,"data_stored":397737984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":334168064,"data_stored":334168064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":264503296,"data_stored":264503296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":197328896,"data_stored":197328896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:39:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:59.167 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:39:58 smithi150 ceph-mon[194178]: pgmap v1520: 119 pgs: 119 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-10-09T06:39:59.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:39:59.269 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:39:59.340 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:59.625 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:39:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:59.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:39:59.734 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:39:59.767 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:39:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:39:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:39:59.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:39:59.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:39:59.965 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:00.030 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:00.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:00.095 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 15 16 29238 29222 121.73 98.75 0.00935742 0.008208 2023-10-09T06:40:00.136 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:00.172 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:00.227 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:00.230 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:00.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:00.267 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:00.314 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:00.333 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:00.354 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:00.371 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:00.393 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:00.427 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:00.428 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:00.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:00.465 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:00.506 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:00.508 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.508 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.508 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.509 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:40:00.533 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:00.560 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:00.602 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:00.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:00.707 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:00.737 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:01.096 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 16 16 30233 30217 118.008 62.1875 0.00518963 0.00846647 2023-10-09T06:40:01.118 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:40:01.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: pgmap v1521: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3680235166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: pgmap v1521: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:01.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3680235166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: pgmap v1521: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:01.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3680235166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:01.242 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:01.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:01.350 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:01.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:01.437 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:01.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:01.542 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:01.584 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:02.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:02.028 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:02.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:02.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 17 16 31494 31478 115.649 78.8125 0.0079065 0.00864013 2023-10-09T06:40:02.135 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:40:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:40:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:02.168 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:40:02.219 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:02.311 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:02] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T06:40:02.353 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:02.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:02.461 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:02.527 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:02.534 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:02.667 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:02.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:02.724 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:40:02.724 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:40:02.781 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:02.817 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:02.877 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:02.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:02.936 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:02 smithi107 ceph-mon[212750]: pgmap v1522: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:02.937 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:02 smithi107 ceph-mon[220921]: pgmap v1522: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:02.986 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:03.021 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:02 smithi150 ceph-mon[194178]: pgmap v1522: 119 pgs: 119 active+clean; 1.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T06:40:03.025 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:03.084 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:03.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:03.103 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 18 16 32919 32903 114.173 89.0625 0.00989744 0.00875383 2023-10-09T06:40:03.193 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:03.232 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:03.299 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:03.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:03.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:03.406 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:03.449 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:03.512 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:03.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:03.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:03.620 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:03.703 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:04.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 19 16 34671 34655 113.926 109.5 0.00531518 0.00877169 2023-10-09T06:40:04.217 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:40:04.681 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:04.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:04.788 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:04.814 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:04.856 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:04.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:04.966 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:05.025 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:04 smithi107 ceph-mon[220921]: pgmap v1523: 119 pgs: 119 active+clean; 1.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:40:05.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:04 smithi107 ceph-mon[212750]: pgmap v1523: 119 pgs: 119 active+clean; 1.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:40:05.036 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:05.084 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:40:05.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:40:05.104572+0000 min lat: 0.0010258 max lat: 0.405258 avg lat: 0.00891453 2023-10-09T06:40:05.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:05.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 20 15 35901 35886 112.077 76.9375 0.0122189 0.00891453 2023-10-09T06:40:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:04 smithi150 ceph-mon[194178]: pgmap v1523: 119 pgs: 119 active+clean; 1.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T06:40:05.442 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:05.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:05.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:05.493 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:05.547 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:05.551 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:05.638 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:05.908 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:05.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:06.018 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:06.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 21 16 37259 37243 110.779 84.8125 0.00530875 0.00902198 2023-10-09T06:40:06.104 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:06.440 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:06.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:06.546 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:06.557 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:06.636 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:06.742 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:06 smithi107 ceph-mon[212750]: pgmap v1524: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:40:06.743 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:06.760 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:40:07.104 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 22 16 38561 38545 109.443 81.375 0.00684129 0.00913192 2023-10-09T06:40:07.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:06 smithi107 ceph-mon[220921]: pgmap v1524: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:40:07.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:06 smithi150 ceph-mon[194178]: pgmap v1524: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T06:40:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:07.908 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:07.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:07.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:08.014 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:08.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:08.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:40:08.046 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:08.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:40:08.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.047 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[212750]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:40:08.048 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.049 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:07 smithi107 ceph-mon[220921]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.073 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:08.106 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 23 16 39686 39670 107.732 70.3125 0.0156852 0.00927677 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2953246528' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: osdmap e495: 8 total, 8 up, 8 in 2023-10-09T06:40:08.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:07 smithi150 ceph-mon[194178]: failed to encode map e495 with expected crc 2023-10-09T06:40:08.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:08.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:08.811 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:08.844 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:08.899 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:08.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:09.003 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:09.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:08 smithi107 ceph-mon[212750]: pgmap v1526: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:40:09.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:08 smithi107 ceph-mon[220921]: pgmap v1526: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:40:09.032 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:09.106 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 24 16 40644 40628 105.739 59.875 0.011515 0.00940769 2023-10-09T06:40:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:08 smithi150 ceph-mon[194178]: pgmap v1526: 119 pgs: 119 active+clean; 2.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-09T06:40:09.659 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:09.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:09.769 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:09.797 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:10.106 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 25 16 41593 41577 103.882 59.3125 0.00230505 0.00955087 2023-10-09T06:40:10.461 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:10.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:10.566 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:10.655 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:10.756 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:10 smithi107 ceph-mon[212750]: pgmap v1527: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:10.890 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:10.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:10.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:10.908 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:40:10.975 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:10.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:11.029 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:11.056 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:10 smithi107 ceph-mon[220921]: pgmap v1527: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:11.064 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:11.108 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 26 16 42899 42883 103.021 81.625 0.0190333 0.00970027 2023-10-09T06:40:11.112 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:11.118 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:10 smithi150 ceph-mon[194178]: pgmap v1527: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:11.171 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:11.225 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:11.274 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:11.314 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.332 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.333 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.333 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.333 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.333 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:40:11.368 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:11.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:11.479 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:11.522 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:40:11.523 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:40:11.523 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:40:11.523 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:40:11.567 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:11.758 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:40:11.773 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:40:12.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:11 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4190874984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:12.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:11 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4190874984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:12.108 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 27 16 44085 44069 101.951 74.125 0.00989416 0.00980324 2023-10-09T06:40:12.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:11 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4190874984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:12] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T06:40:12.718 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:12.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:12.828 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:12.866 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:12.935 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:12.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:12.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:13.047 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:13.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:12 smithi107 ceph-mon[212750]: pgmap v1528: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:13.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:12 smithi107 ceph-mon[212750]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:13.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:12 smithi107 ceph-mon[220921]: pgmap v1528: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:13.110 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:12 smithi107 ceph-mon[220921]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:13.110 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 28 16 45280 45264 100.973 74.6875 0.00980952 0.0098973 2023-10-09T06:40:13.113 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:12 smithi150 ceph-mon[194178]: pgmap v1528: 119 pgs: 119 active+clean; 2.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-09T06:40:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:12 smithi150 ceph-mon[194178]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:14.090 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1528,"stamp":"2023-10-09T06:40:11.554041+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2619021216,"num_objects":39978,"num_object_clones":0,"num_object_copies":119934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40015,"num_write_kb":2557085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45000,"ondisk_log_size":45000,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17272476,"kb_used_data":7843740,"kb_used_omap":20,"kb_used_meta":9428587,"kb_avail":732688740,"statfs":{"total":767960285184,"available":750273269760,"internally_reserved":0,"allocated":8031989760,"data_stored":8019095166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9654873742},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":869728256,"num_objects":13271,"num_object_clones":0,"num_object_copies":39813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13271,"num_write_kb":849344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005099"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":789,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697889+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:39:55.697889+0000","last_peered":"2023-10-09T06:39:55.697889+0000","last_clean":"2023-10-09T06:39:55.697889+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:39:55.697889+0000","last_undegraded":"2023-10-09T06:39:55.697889+0000","last_fullsized":"2023-10-09T06:39:55.697889+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":474,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686195+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:39:55.686195+0000","last_peered":"2023-10-09T06:39:55.686195+0000","last_clean":"2023-10-09T06:39:55.686195+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:39:55.686195+0000","last_undegraded":"2023-10-09T06:39:55.686195+0000","last_fullsized":"2023-10-09T06:39:55.686195+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":196,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697394+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:39:55.697394+0000","last_peered":"2023-10-09T06:39:55.697394+0000","last_clean":"2023-10-09T06:39:55.697394+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:39:55.697394+0000","last_undegraded":"2023-10-09T06:39:55.697394+0000","last_fullsized":"2023-10-09T06:39:55.697394+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":750,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686181+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:39:55.686181+0000","last_peered":"2023-10-09T06:39:55.686181+0000","last_clean":"2023-10-09T06:39:55.686181+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:39:55.686181+0000","last_undegraded":"2023-10-09T06:39:55.686181+0000","last_fullsized":"2023-10-09T06:39:55.686181+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":491,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.690120+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:39:55.690120+0000","last_peered":"2023-10-09T06:39:55.690120+0000","last_clean":"2023-10-09T06:39:55.690120+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:39:55.690120+0000","last_undegraded":"2023-10-09T06:39:55.690120+0000","last_fullsized":"2023-10-09T06:39:55.690120+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":495,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686609+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:39:55.686609+0000","last_peered":"2023-10-09T06:39:55.686609+0000","last_clean":"2023-10-09T06:39:55.686609+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:39:55.686609+0000","last_undegraded":"2023-10-09T06:39:55.686609+0000","last_fullsized":"2023-10-09T06:39:55.686609+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":742,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686138+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:39:55.686138+0000","last_peered":"2023-10-09T06:39:55.686138+0000","last_clean":"2023-10-09T06:39:55.686138+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:39:55.686138+0000","last_undegraded":"2023-10-09T06:39:55.686138+0000","last_fullsized":"2023-10-09T06:39:55.686138+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":215,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692133+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:39:55.692133+0000","last_peered":"2023-10-09T06:39:55.692133+0000","last_clean":"2023-10-09T06:39:55.692133+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:39:55.692133+0000","last_undegraded":"2023-10-09T06:39:55.692133+0000","last_fullsized":"2023-10-09T06:39:55.692133+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":484,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694242+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:39:55.694242+0000","last_peered":"2023-10-09T06:39:55.694242+0000","last_clean":"2023-10-09T06:39:55.694242+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:39:55.694242+0000","last_undegraded":"2023-10-09T06:39:55.694242+0000","last_fullsized":"2023-10-09T06:39:55.694242+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":603,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698798+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:39:55.698798+0000","last_peered":"2023-10-09T06:39:55.698798+0000","last_clean":"2023-10-09T06:39:55.698798+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:39:55.698798+0000","last_undegraded":"2023-10-09T06:39:55.698798+0000","last_fullsized":"2023-10-09T06:39:55.698798+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":170,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685558+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:39:55.685558+0000","last_peered":"2023-10-09T06:39:55.685558+0000","last_clean":"2023-10-09T06:39:55.685558+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:39:55.685558+0000","last_undegraded":"2023-10-09T06:39:55.685558+0000","last_fullsized":"2023-10-09T06:39:55.685558+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":197,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692374+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:39:55.692374+0000","last_peered":"2023-10-09T06:39:55.692374+0000","last_clean":"2023-10-09T06:39:55.692374+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:39:55.692374+0000","last_undegraded":"2023-10-09T06:39:55.692374+0000","last_fullsized":"2023-10-09T06:39:55.692374+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":640,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698858+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:39:55.698858+0000","last_peered":"2023-10-09T06:39:55.698858+0000","last_clean":"2023-10-09T06:39:55.698858+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:39:55.698858+0000","last_undegraded":"2023-10-09T06:39:55.698858+0000","last_fullsized":"2023-10-09T06:39:55.698858+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687108+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:39:55.687108+0000","last_peered":"2023-10-09T06:39:55.687108+0000","last_clean":"2023-10-09T06:39:55.687108+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:39:55.687108+0000","last_undegraded":"2023-10-09T06:39:55.687108+0000","last_fullsized":"2023-10-09T06:39:55.687108+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":197,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693131+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:39:55.693131+0000","last_peered":"2023-10-09T06:39:55.693131+0000","last_clean":"2023-10-09T06:39:55.693131+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:39:55.693131+0000","last_undegraded":"2023-10-09T06:39:55.693131+0000","last_fullsized":"2023-10-09T06:39:55.693131+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":623,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685224+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:39:55.685224+0000","last_peered":"2023-10-09T06:39:55.685224+0000","last_clean":"2023-10-09T06:39:55.685224+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:39:55.685224+0000","last_undegraded":"2023-10-09T06:39:55.685224+0000","last_fullsized":"2023-10-09T06:39:55.685224+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":376,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.699053+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:39:55.699053+0000","last_peered":"2023-10-09T06:39:55.699053+0000","last_clean":"2023-10-09T06:39:55.699053+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:39:55.699053+0000","last_undegraded":"2023-10-09T06:39:55.699053+0000","last_fullsized":"2023-10-09T06:39:55.699053+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":432,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.691099+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:39:55.691099+0000","last_peered":"2023-10-09T06:39:55.691099+0000","last_clean":"2023-10-09T06:39:55.691099+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:39:55.691099+0000","last_undegraded":"2023-10-09T06:39:55.691099+0000","last_fullsized":"2023-10-09T06:39:55.691099+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":604,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697051+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:39:55.697051+0000","last_peered":"2023-10-09T06:39:55.697051+0000","last_clean":"2023-10-09T06:39:55.697051+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:39:55.697051+0000","last_undegraded":"2023-10-09T06:39:55.697051+0000","last_fullsized":"2023-10-09T06:39:55.697051+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":501,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687150+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:39:55.687150+0000","last_peered":"2023-10-09T06:39:55.687150+0000","last_clean":"2023-10-09T06:39:55.687150+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:39:55.687150+0000","last_undegraded":"2023-10-09T06:39:55.687150+0000","last_fullsized":"2023-10-09T06:39:55.687150+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":465,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687054+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:39:55.687054+0000","last_peered":"2023-10-09T06:39:55.687054+0000","last_clean":"2023-10-09T06:39:55.687054+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:39:55.687054+0000","last_undegraded":"2023-10-09T06:39:55.687054+0000","last_fullsized":"2023-10-09T06:39:55.687054+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":768,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685205+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:39:55.685205+0000","last_peered":"2023-10-09T06:39:55.685205+0000","last_clean":"2023-10-09T06:39:55.685205+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:39:55.685205+0000","last_undegraded":"2023-10-09T06:39:55.685205+0000","last_fullsized":"2023-10-09T06:39:55.685205+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":344,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698377+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:39:55.698377+0000","last_peered":"2023-10-09T06:39:55.698377+0000","last_clean":"2023-10-09T06:39:55.698377+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:39:55.698377+0000","last_undegraded":"2023-10-09T06:39:55.698377+0000","last_fullsized":"2023-10-09T06:39:55.698377+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698103+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:39:55.698103+0000","last_peered":"2023-10-09T06:39:55.698103+0000","last_clean":"2023-10-09T06:39:55.698103+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:39:55.698103+0000","last_undegraded":"2023-10-09T06:39:55.698103+0000","last_fullsized":"2023-10-09T06:39:55.698103+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":767,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697982+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:39:55.697982+0000","last_peered":"2023-10-09T06:39:55.697982+0000","last_clean":"2023-10-09T06:39:55.697982+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:39:55.697982+0000","last_undegraded":"2023-10-09T06:39:55.697982+0000","last_fullsized":"2023-10-09T06:39:55.697982+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":493,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.684714+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:39:55.684714+0000","last_peered":"2023-10-09T06:39:55.684714+0000","last_clean":"2023-10-09T06:39:55.684714+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:39:55.684714+0000","last_undegraded":"2023-10-09T06:39:55.684714+0000","last_fullsized":"2023-10-09T06:39:55.684714+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":315,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698951+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:39:55.698951+0000","last_peered":"2023-10-09T06:39:55.698951+0000","last_clean":"2023-10-09T06:39:55.698951+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:39:55.698951+0000","last_undegraded":"2023-10-09T06:39:55.698951+0000","last_fullsized":"2023-10-09T06:39:55.698951+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":822,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.690287+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:39:55.690287+0000","last_peered":"2023-10-09T06:39:55.690287+0000","last_clean":"2023-10-09T06:39:55.690287+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:39:55.690287+0000","last_undegraded":"2023-10-09T06:39:55.690287+0000","last_fullsized":"2023-10-09T06:39:55.690287+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":338,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698160+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:39:55.698160+0000","last_peered":"2023-10-09T06:39:55.698160+0000","last_clean":"2023-10-09T06:39:55.698160+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:39:55.698160+0000","last_undegraded":"2023-10-09T06:39:55.698160+0000","last_fullsized":"2023-10-09T06:39:55.698160+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686216+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:39:55.686216+0000","last_peered":"2023-10-09T06:39:55.686216+0000","last_clean":"2023-10-09T06:39:55.686216+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:39:55.686216+0000","last_undegraded":"2023-10-09T06:39:55.686216+0000","last_fullsized":"2023-10-09T06:39:55.686216+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":609,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686542+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:39:55.686542+0000","last_peered":"2023-10-09T06:39:55.686542+0000","last_clean":"2023-10-09T06:39:55.686542+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:39:55.686542+0000","last_undegraded":"2023-10-09T06:39:55.686542+0000","last_fullsized":"2023-10-09T06:39:55.686542+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":573,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693578+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:39:55.693578+0000","last_peered":"2023-10-09T06:39:55.693578+0000","last_clean":"2023-10-09T06:39:55.693578+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:39:55.693578+0000","last_undegraded":"2023-10-09T06:39:55.693578+0000","last_fullsized":"2023-10-09T06:39:55.693578+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":197,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698781+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:39:55.698781+0000","last_peered":"2023-10-09T06:39:55.698781+0000","last_clean":"2023-10-09T06:39:55.698781+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:39:55.698781+0000","last_undegraded":"2023-10-09T06:39:55.698781+0000","last_fullsized":"2023-10-09T06:39:55.698781+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"495'2476","reported_seq":2489,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.654508+0000","last_change":"2023-10-09T06:39:33.925861+0000","last_active":"2023-10-09T06:40:07.654508+0000","last_peered":"2023-10-09T06:40:07.654508+0000","last_clean":"2023-10-09T06:40:07.654508+0000","last_became_active":"2023-10-09T06:39:33.925623+0000","last_became_peered":"2023-10-09T06:39:33.925623+0000","last_unstale":"2023-10-09T06:40:07.654508+0000","last_undegraded":"2023-10-09T06:40:07.654508+0000","last_fullsized":"2023-10-09T06:40:07.654508+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2476,"log_dups_size":0,"ondisk_log_size":2476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:10.504728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162267136,"num_objects":2476,"num_object_clones":0,"num_object_copies":7428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2476,"num_write_kb":158464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":622,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698870+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:39:55.698870+0000","last_peered":"2023-10-09T06:39:55.698870+0000","last_clean":"2023-10-09T06:39:55.698870+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:39:55.698870+0000","last_undegraded":"2023-10-09T06:39:55.698870+0000","last_fullsized":"2023-10-09T06:39:55.698870+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":380,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698973+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:39:55.698973+0000","last_peered":"2023-10-09T06:39:55.698973+0000","last_clean":"2023-10-09T06:39:55.698973+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:39:55.698973+0000","last_undegraded":"2023-10-09T06:39:55.698973+0000","last_fullsized":"2023-10-09T06:39:55.698973+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698110+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:39:55.698110+0000","last_peered":"2023-10-09T06:39:55.698110+0000","last_clean":"2023-10-09T06:39:55.698110+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:39:55.698110+0000","last_undegraded":"2023-10-09T06:39:55.698110+0000","last_fullsized":"2023-10-09T06:39:55.698110+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"495'2526","reported_seq":2541,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.984339+0000","last_change":"2023-10-09T06:39:33.923263+0000","last_active":"2023-10-09T06:40:08.984339+0000","last_peered":"2023-10-09T06:40:08.984339+0000","last_clean":"2023-10-09T06:40:08.984339+0000","last_became_active":"2023-10-09T06:39:33.922687+0000","last_became_peered":"2023-10-09T06:39:33.922687+0000","last_unstale":"2023-10-09T06:40:08.984339+0000","last_undegraded":"2023-10-09T06:40:08.984339+0000","last_fullsized":"2023-10-09T06:40:08.984339+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2526,"log_dups_size":0,"ondisk_log_size":2526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:59.579546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2526,"num_write_kb":161664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":881,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697864+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:39:55.697864+0000","last_peered":"2023-10-09T06:39:55.697864+0000","last_clean":"2023-10-09T06:39:55.697864+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:39:55.697864+0000","last_undegraded":"2023-10-09T06:39:55.697864+0000","last_fullsized":"2023-10-09T06:39:55.697864+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":342,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.700555+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:39:55.700555+0000","last_peered":"2023-10-09T06:39:55.700555+0000","last_clean":"2023-10-09T06:39:55.700555+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:39:55.700555+0000","last_undegraded":"2023-10-09T06:39:55.700555+0000","last_fullsized":"2023-10-09T06:39:55.700555+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":492,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686238+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:39:55.686238+0000","last_peered":"2023-10-09T06:39:55.686238+0000","last_clean":"2023-10-09T06:39:55.686238+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:39:55.686238+0000","last_undegraded":"2023-10-09T06:39:55.686238+0000","last_fullsized":"2023-10-09T06:39:55.686238+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"495'2532","reported_seq":2547,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.966195+0000","last_change":"2023-10-09T06:39:33.927681+0000","last_active":"2023-10-09T06:40:08.966195+0000","last_peered":"2023-10-09T06:40:08.966195+0000","last_clean":"2023-10-09T06:40:08.966195+0000","last_became_active":"2023-10-09T06:39:33.927408+0000","last_became_peered":"2023-10-09T06:39:33.927408+0000","last_unstale":"2023-10-09T06:40:08.966195+0000","last_undegraded":"2023-10-09T06:40:08.966195+0000","last_fullsized":"2023-10-09T06:40:08.966195+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2532,"log_dups_size":0,"ondisk_log_size":2532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:59.401749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2532,"num_write_kb":162048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":796,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694171+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:39:55.694171+0000","last_peered":"2023-10-09T06:39:55.694171+0000","last_clean":"2023-10-09T06:39:55.694171+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:39:55.694171+0000","last_undegraded":"2023-10-09T06:39:55.694171+0000","last_fullsized":"2023-10-09T06:39:55.694171+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":398,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698958+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:39:55.698958+0000","last_peered":"2023-10-09T06:39:55.698958+0000","last_clean":"2023-10-09T06:39:55.698958+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:39:55.698958+0000","last_undegraded":"2023-10-09T06:39:55.698958+0000","last_fullsized":"2023-10-09T06:39:55.698958+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":471,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693869+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:39:55.693869+0000","last_peered":"2023-10-09T06:39:55.693869+0000","last_clean":"2023-10-09T06:39:55.693869+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:39:55.693869+0000","last_undegraded":"2023-10-09T06:39:55.693869+0000","last_fullsized":"2023-10-09T06:39:55.693869+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"495'2528","reported_seq":2543,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.900739+0000","last_change":"2023-10-09T06:39:33.924837+0000","last_active":"2023-10-09T06:40:08.900739+0000","last_peered":"2023-10-09T06:40:08.900739+0000","last_clean":"2023-10-09T06:40:08.900739+0000","last_became_active":"2023-10-09T06:39:33.924498+0000","last_became_peered":"2023-10-09T06:39:33.924498+0000","last_unstale":"2023-10-09T06:40:08.900739+0000","last_undegraded":"2023-10-09T06:40:08.900739+0000","last_fullsized":"2023-10-09T06:40:08.900739+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2528,"log_dups_size":0,"ondisk_log_size":2528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:35.001011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165675008,"num_objects":2528,"num_object_clones":0,"num_object_copies":7584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2528,"num_write_kb":161792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":619,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687326+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:39:55.687326+0000","last_peered":"2023-10-09T06:39:55.687326+0000","last_clean":"2023-10-09T06:39:55.687326+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:39:55.687326+0000","last_undegraded":"2023-10-09T06:39:55.687326+0000","last_fullsized":"2023-10-09T06:39:55.687326+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":65,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698188+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:39:55.698188+0000","last_peered":"2023-10-09T06:39:55.698188+0000","last_clean":"2023-10-09T06:39:55.698188+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:39:55.698188+0000","last_undegraded":"2023-10-09T06:39:55.698188+0000","last_fullsized":"2023-10-09T06:39:55.698188+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":458,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685722+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:39:55.685722+0000","last_peered":"2023-10-09T06:39:55.685722+0000","last_clean":"2023-10-09T06:39:55.685722+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:39:55.685722+0000","last_undegraded":"2023-10-09T06:39:55.685722+0000","last_fullsized":"2023-10-09T06:39:55.685722+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"495'2520","reported_seq":2535,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.901555+0000","last_change":"2023-10-09T06:39:33.922928+0000","last_active":"2023-10-09T06:40:08.901555+0000","last_peered":"2023-10-09T06:40:08.901555+0000","last_clean":"2023-10-09T06:40:08.901555+0000","last_became_active":"2023-10-09T06:39:33.922653+0000","last_became_peered":"2023-10-09T06:39:33.922653+0000","last_unstale":"2023-10-09T06:40:08.901555+0000","last_undegraded":"2023-10-09T06:40:08.901555+0000","last_fullsized":"2023-10-09T06:40:08.901555+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2520,"log_dups_size":0,"ondisk_log_size":2520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:31.453742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165150720,"num_objects":2520,"num_object_clones":0,"num_object_copies":7560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2520,"num_write_kb":161280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":847,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698020+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:39:55.698020+0000","last_peered":"2023-10-09T06:39:55.698020+0000","last_clean":"2023-10-09T06:39:55.698020+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:39:55.698020+0000","last_undegraded":"2023-10-09T06:39:55.698020+0000","last_fullsized":"2023-10-09T06:39:55.698020+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":223,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697826+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:39:55.697826+0000","last_peered":"2023-10-09T06:39:55.697826+0000","last_clean":"2023-10-09T06:39:55.697826+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:39:55.697826+0000","last_undegraded":"2023-10-09T06:39:55.697826+0000","last_fullsized":"2023-10-09T06:39:55.697826+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":475,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686990+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:39:55.686990+0000","last_peered":"2023-10-09T06:39:55.686990+0000","last_clean":"2023-10-09T06:39:55.686990+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:39:55.686990+0000","last_undegraded":"2023-10-09T06:39:55.686990+0000","last_fullsized":"2023-10-09T06:39:55.686990+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"495'2447","reported_seq":2461,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.466032+0000","last_change":"2023-10-09T06:39:33.929339+0000","last_active":"2023-10-09T06:40:07.466032+0000","last_peered":"2023-10-09T06:40:07.466032+0000","last_clean":"2023-10-09T06:40:07.466032+0000","last_became_active":"2023-10-09T06:39:33.929105+0000","last_became_peered":"2023-10-09T06:39:33.929105+0000","last_unstale":"2023-10-09T06:40:07.466032+0000","last_undegraded":"2023-10-09T06:40:07.466032+0000","last_fullsized":"2023-10-09T06:40:07.466032+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2447,"log_dups_size":0,"ondisk_log_size":2447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:24.427366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2447,"num_write_kb":156608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":529,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692378+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:39:55.692378+0000","last_peered":"2023-10-09T06:39:55.692378+0000","last_clean":"2023-10-09T06:39:55.692378+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:39:55.692378+0000","last_undegraded":"2023-10-09T06:39:55.692378+0000","last_fullsized":"2023-10-09T06:39:55.692378+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":452,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693763+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:39:55.693763+0000","last_peered":"2023-10-09T06:39:55.693763+0000","last_clean":"2023-10-09T06:39:55.693763+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:39:55.693763+0000","last_undegraded":"2023-10-09T06:39:55.693763+0000","last_fullsized":"2023-10-09T06:39:55.693763+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":472,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687462+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:39:55.687462+0000","last_peered":"2023-10-09T06:39:55.687462+0000","last_clean":"2023-10-09T06:39:55.687462+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:39:55.687462+0000","last_undegraded":"2023-10-09T06:39:55.687462+0000","last_fullsized":"2023-10-09T06:39:55.687462+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"495'2444","reported_seq":2458,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.665758+0000","last_change":"2023-10-09T06:39:33.924988+0000","last_active":"2023-10-09T06:40:07.665758+0000","last_peered":"2023-10-09T06:40:07.665758+0000","last_clean":"2023-10-09T06:40:07.665758+0000","last_became_active":"2023-10-09T06:39:33.924698+0000","last_became_peered":"2023-10-09T06:39:33.924698+0000","last_unstale":"2023-10-09T06:40:07.665758+0000","last_undegraded":"2023-10-09T06:40:07.665758+0000","last_fullsized":"2023-10-09T06:40:07.665758+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2444,"log_dups_size":0,"ondisk_log_size":2444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:03.806961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160169984,"num_objects":2444,"num_object_clones":0,"num_object_copies":7332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2444,"num_write_kb":156416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":831,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.690382+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:39:55.690382+0000","last_peered":"2023-10-09T06:39:55.690382+0000","last_clean":"2023-10-09T06:39:55.690382+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:39:55.690382+0000","last_undegraded":"2023-10-09T06:39:55.690382+0000","last_fullsized":"2023-10-09T06:39:55.690382+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":476,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686140+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:39:55.686140+0000","last_peered":"2023-10-09T06:39:55.686140+0000","last_clean":"2023-10-09T06:39:55.686140+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:39:55.686140+0000","last_undegraded":"2023-10-09T06:39:55.686140+0000","last_fullsized":"2023-10-09T06:39:55.686140+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":455,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687530+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:39:55.687530+0000","last_peered":"2023-10-09T06:39:55.687530+0000","last_clean":"2023-10-09T06:39:55.687530+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:39:55.687530+0000","last_undegraded":"2023-10-09T06:39:55.687530+0000","last_fullsized":"2023-10-09T06:39:55.687530+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"495'2495","reported_seq":2510,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:09.207367+0000","last_change":"2023-10-09T06:39:33.922114+0000","last_active":"2023-10-09T06:40:09.207367+0000","last_peered":"2023-10-09T06:40:09.207367+0000","last_clean":"2023-10-09T06:40:09.207367+0000","last_became_active":"2023-10-09T06:39:33.921838+0000","last_became_peered":"2023-10-09T06:39:33.921838+0000","last_unstale":"2023-10-09T06:40:09.207367+0000","last_undegraded":"2023-10-09T06:40:09.207367+0000","last_fullsized":"2023-10-09T06:40:09.207367+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2495,"log_dups_size":0,"ondisk_log_size":2495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:43.308955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163512320,"num_objects":2495,"num_object_clones":0,"num_object_copies":7485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2495,"num_write_kb":159680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":457,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693768+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:39:55.693768+0000","last_peered":"2023-10-09T06:39:55.693768+0000","last_clean":"2023-10-09T06:39:55.693768+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:39:55.693768+0000","last_undegraded":"2023-10-09T06:39:55.693768+0000","last_fullsized":"2023-10-09T06:39:55.693768+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":197,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692295+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:39:55.692295+0000","last_peered":"2023-10-09T06:39:55.692295+0000","last_clean":"2023-10-09T06:39:55.692295+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:39:55.692295+0000","last_undegraded":"2023-10-09T06:39:55.692295+0000","last_fullsized":"2023-10-09T06:39:55.692295+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":451,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:40:03.094525+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:40:03.094525+0000","last_peered":"2023-10-09T06:40:03.094525+0000","last_clean":"2023-10-09T06:40:03.094525+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:40:03.094525+0000","last_undegraded":"2023-10-09T06:40:03.094525+0000","last_fullsized":"2023-10-09T06:40:03.094525+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"495'2581","reported_seq":2596,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.974955+0000","last_change":"2023-10-09T06:39:33.922898+0000","last_active":"2023-10-09T06:40:08.974955+0000","last_peered":"2023-10-09T06:40:08.974955+0000","last_clean":"2023-10-09T06:40:08.974955+0000","last_became_active":"2023-10-09T06:39:33.922619+0000","last_became_peered":"2023-10-09T06:39:33.922619+0000","last_unstale":"2023-10-09T06:40:08.974955+0000","last_undegraded":"2023-10-09T06:40:08.974955+0000","last_fullsized":"2023-10-09T06:40:08.974955+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2581,"log_dups_size":0,"ondisk_log_size":2581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.252997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2581,"num_write_kb":165184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":787,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694297+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:39:55.694297+0000","last_peered":"2023-10-09T06:39:55.694297+0000","last_clean":"2023-10-09T06:39:55.694297+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:39:55.694297+0000","last_undegraded":"2023-10-09T06:39:55.694297+0000","last_fullsized":"2023-10-09T06:39:55.694297+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":503,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.732609+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:40:06.732609+0000","last_peered":"2023-10-09T06:40:06.732609+0000","last_clean":"2023-10-09T06:40:06.732609+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:40:06.732609+0000","last_undegraded":"2023-10-09T06:40:06.732609+0000","last_fullsized":"2023-10-09T06:40:06.732609+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":65,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697321+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:39:55.697321+0000","last_peered":"2023-10-09T06:39:55.697321+0000","last_clean":"2023-10-09T06:39:55.697321+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:39:55.697321+0000","last_undegraded":"2023-10-09T06:39:55.697321+0000","last_fullsized":"2023-10-09T06:39:55.697321+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":437,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.731889+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:40:06.731889+0000","last_peered":"2023-10-09T06:40:06.731889+0000","last_clean":"2023-10-09T06:40:06.731889+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:40:06.731889+0000","last_undegraded":"2023-10-09T06:40:06.731889+0000","last_fullsized":"2023-10-09T06:40:06.731889+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"495'2365","reported_seq":2379,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.673885+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:40:07.673885+0000","last_peered":"2023-10-09T06:40:07.673885+0000","last_clean":"2023-10-09T06:40:07.673885+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:40:07.673885+0000","last_undegraded":"2023-10-09T06:40:07.673885+0000","last_fullsized":"2023-10-09T06:40:07.673885+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2365,"log_dups_size":0,"ondisk_log_size":2365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154992640,"num_objects":2365,"num_object_clones":0,"num_object_copies":7095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2365,"num_write_kb":151360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":674,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685621+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:39:55.685621+0000","last_peered":"2023-10-09T06:39:55.685621+0000","last_clean":"2023-10-09T06:39:55.685621+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:39:55.685621+0000","last_undegraded":"2023-10-09T06:39:55.685621+0000","last_fullsized":"2023-10-09T06:39:55.685621+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":226,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.690915+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:39:55.690915+0000","last_peered":"2023-10-09T06:39:55.690915+0000","last_clean":"2023-10-09T06:39:55.690915+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:39:55.690915+0000","last_undegraded":"2023-10-09T06:39:55.690915+0000","last_fullsized":"2023-10-09T06:39:55.690915+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":476,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686982+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:39:55.686982+0000","last_peered":"2023-10-09T06:39:55.686982+0000","last_clean":"2023-10-09T06:39:55.686982+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:39:55.686982+0000","last_undegraded":"2023-10-09T06:39:55.686982+0000","last_fullsized":"2023-10-09T06:39:55.686982+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":467,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.732547+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:40:06.732547+0000","last_peered":"2023-10-09T06:40:06.732547+0000","last_clean":"2023-10-09T06:40:06.732547+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:40:06.732547+0000","last_undegraded":"2023-10-09T06:40:06.732547+0000","last_fullsized":"2023-10-09T06:40:06.732547+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"495'2440","reported_seq":2454,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.676960+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:40:07.676960+0000","last_peered":"2023-10-09T06:40:07.676960+0000","last_clean":"2023-10-09T06:40:07.676960+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:40:07.676960+0000","last_undegraded":"2023-10-09T06:40:07.676960+0000","last_fullsized":"2023-10-09T06:40:07.676960+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2440,"log_dups_size":0,"ondisk_log_size":2440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2440,"num_write_kb":156160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":837,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698284+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:39:55.698284+0000","last_peered":"2023-10-09T06:39:55.698284+0000","last_clean":"2023-10-09T06:39:55.698284+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:39:55.698284+0000","last_undegraded":"2023-10-09T06:39:55.698284+0000","last_fullsized":"2023-10-09T06:39:55.698284+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":488,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692904+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:39:55.692904+0000","last_peered":"2023-10-09T06:39:55.692904+0000","last_clean":"2023-10-09T06:39:55.692904+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:39:55.692904+0000","last_undegraded":"2023-10-09T06:39:55.692904+0000","last_fullsized":"2023-10-09T06:39:55.692904+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":475,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.689823+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:39:55.689823+0000","last_peered":"2023-10-09T06:39:55.689823+0000","last_clean":"2023-10-09T06:39:55.689823+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:39:55.689823+0000","last_undegraded":"2023-10-09T06:39:55.689823+0000","last_fullsized":"2023-10-09T06:39:55.689823+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":497,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697063+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:39:55.697063+0000","last_peered":"2023-10-09T06:39:55.697063+0000","last_clean":"2023-10-09T06:39:55.697063+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:39:55.697063+0000","last_undegraded":"2023-10-09T06:39:55.697063+0000","last_fullsized":"2023-10-09T06:39:55.697063+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"495'2442","reported_seq":2456,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:07.659660+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:40:07.659660+0000","last_peered":"2023-10-09T06:40:07.659660+0000","last_clean":"2023-10-09T06:40:07.659660+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:40:07.659660+0000","last_undegraded":"2023-10-09T06:40:07.659660+0000","last_fullsized":"2023-10-09T06:40:07.659660+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2442,"log_dups_size":0,"ondisk_log_size":2442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2442,"num_write_kb":156288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":304,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698331+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:39:55.698331+0000","last_peered":"2023-10-09T06:39:55.698331+0000","last_clean":"2023-10-09T06:39:55.698331+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:39:55.698331+0000","last_undegraded":"2023-10-09T06:39:55.698331+0000","last_fullsized":"2023-10-09T06:39:55.698331+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":818,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698138+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:39:55.698138+0000","last_peered":"2023-10-09T06:39:55.698138+0000","last_clean":"2023-10-09T06:39:55.698138+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:39:55.698138+0000","last_undegraded":"2023-10-09T06:39:55.698138+0000","last_fullsized":"2023-10-09T06:39:55.698138+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":207,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.738322+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:40:06.738322+0000","last_peered":"2023-10-09T06:40:06.738322+0000","last_clean":"2023-10-09T06:40:06.738322+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:40:06.738322+0000","last_undegraded":"2023-10-09T06:40:06.738322+0000","last_fullsized":"2023-10-09T06:40:06.738322+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":457,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694054+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:39:55.694054+0000","last_peered":"2023-10-09T06:39:55.694054+0000","last_clean":"2023-10-09T06:39:55.694054+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:39:55.694054+0000","last_undegraded":"2023-10-09T06:39:55.694054+0000","last_fullsized":"2023-10-09T06:39:55.694054+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"495'2551","reported_seq":2566,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.982858+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:40:08.982858+0000","last_peered":"2023-10-09T06:40:08.982858+0000","last_clean":"2023-10-09T06:40:08.982858+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:40:08.982858+0000","last_undegraded":"2023-10-09T06:40:08.982858+0000","last_fullsized":"2023-10-09T06:40:08.982858+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2551,"log_dups_size":0,"ondisk_log_size":2551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167182336,"num_objects":2551,"num_object_clones":0,"num_object_copies":7653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2551,"num_write_kb":163264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":806,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.732927+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:40:06.732927+0000","last_peered":"2023-10-09T06:40:06.732927+0000","last_clean":"2023-10-09T06:40:06.732927+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:40:06.732927+0000","last_undegraded":"2023-10-09T06:40:06.732927+0000","last_fullsized":"2023-10-09T06:40:06.732927+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":358,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698032+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:39:55.698032+0000","last_peered":"2023-10-09T06:39:55.698032+0000","last_clean":"2023-10-09T06:39:55.698032+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:39:55.698032+0000","last_undegraded":"2023-10-09T06:39:55.698032+0000","last_fullsized":"2023-10-09T06:39:55.698032+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":171,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685713+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:39:55.685713+0000","last_peered":"2023-10-09T06:39:55.685713+0000","last_clean":"2023-10-09T06:39:55.685713+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:39:55.685713+0000","last_undegraded":"2023-10-09T06:39:55.685713+0000","last_fullsized":"2023-10-09T06:39:55.685713+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":481,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686681+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:39:55.686681+0000","last_peered":"2023-10-09T06:39:55.686681+0000","last_clean":"2023-10-09T06:39:55.686681+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:39:55.686681+0000","last_undegraded":"2023-10-09T06:39:55.686681+0000","last_fullsized":"2023-10-09T06:39:55.686681+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"495'2548","reported_seq":2563,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.892117+0000","last_change":"2023-10-09T06:39:33.927704+0000","last_active":"2023-10-09T06:40:08.892117+0000","last_peered":"2023-10-09T06:40:08.892117+0000","last_clean":"2023-10-09T06:40:08.892117+0000","last_became_active":"2023-10-09T06:39:33.927409+0000","last_became_peered":"2023-10-09T06:39:33.927409+0000","last_unstale":"2023-10-09T06:40:08.892117+0000","last_undegraded":"2023-10-09T06:40:08.892117+0000","last_fullsized":"2023-10-09T06:40:08.892117+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2548,"log_dups_size":0,"ondisk_log_size":2548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:18.487828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2548,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":817,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694080+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:39:55.694080+0000","last_peered":"2023-10-09T06:39:55.694080+0000","last_clean":"2023-10-09T06:39:55.694080+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:39:55.694080+0000","last_undegraded":"2023-10-09T06:39:55.694080+0000","last_fullsized":"2023-10-09T06:39:55.694080+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":358,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.699050+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:39:55.699050+0000","last_peered":"2023-10-09T06:39:55.699050+0000","last_clean":"2023-10-09T06:39:55.699050+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:39:55.699050+0000","last_undegraded":"2023-10-09T06:39:55.699050+0000","last_fullsized":"2023-10-09T06:39:55.699050+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":493,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:06.732482+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:40:06.732482+0000","last_peered":"2023-10-09T06:40:06.732482+0000","last_clean":"2023-10-09T06:40:06.732482+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:40:06.732482+0000","last_undegraded":"2023-10-09T06:40:06.732482+0000","last_fullsized":"2023-10-09T06:40:06.732482+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":480,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685936+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:39:55.685936+0000","last_peered":"2023-10-09T06:39:55.685936+0000","last_clean":"2023-10-09T06:39:55.685936+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:39:55.685936+0000","last_undegraded":"2023-10-09T06:39:55.685936+0000","last_fullsized":"2023-10-09T06:39:55.685936+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"495'2514","reported_seq":2529,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.066275+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:40:08.066275+0000","last_peered":"2023-10-09T06:40:08.066275+0000","last_clean":"2023-10-09T06:40:08.066275+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:40:08.066275+0000","last_undegraded":"2023-10-09T06:40:08.066275+0000","last_fullsized":"2023-10-09T06:40:08.066275+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2514,"log_dups_size":0,"ondisk_log_size":2514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164757504,"num_objects":2514,"num_object_clones":0,"num_object_copies":7542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2514,"num_write_kb":160896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":818,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687391+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:39:55.687391+0000","last_peered":"2023-10-09T06:39:55.687391+0000","last_clean":"2023-10-09T06:39:55.687391+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:39:55.687391+0000","last_undegraded":"2023-10-09T06:39:55.687391+0000","last_fullsized":"2023-10-09T06:39:55.687391+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":227,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698704+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:39:55.698704+0000","last_peered":"2023-10-09T06:39:55.698704+0000","last_clean":"2023-10-09T06:39:55.698704+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:39:55.698704+0000","last_undegraded":"2023-10-09T06:39:55.698704+0000","last_fullsized":"2023-10-09T06:39:55.698704+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":460,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697478+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:39:55.697478+0000","last_peered":"2023-10-09T06:39:55.697478+0000","last_clean":"2023-10-09T06:39:55.697478+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:39:55.697478+0000","last_undegraded":"2023-10-09T06:39:55.697478+0000","last_fullsized":"2023-10-09T06:39:55.697478+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692212+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:39:55.692212+0000","last_peered":"2023-10-09T06:39:55.692212+0000","last_clean":"2023-10-09T06:39:55.692212+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:39:55.692212+0000","last_undegraded":"2023-10-09T06:39:55.692212+0000","last_fullsized":"2023-10-09T06:39:55.692212+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"495'2545","reported_seq":2560,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-10-09T06:40:08.977429+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:40:08.977429+0000","last_peered":"2023-10-09T06:40:08.977429+0000","last_clean":"2023-10-09T06:40:08.977429+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:40:08.977429+0000","last_undegraded":"2023-10-09T06:40:08.977429+0000","last_fullsized":"2023-10-09T06:40:08.977429+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":2545,"log_dups_size":0,"ondisk_log_size":2545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166789120,"num_objects":2545,"num_object_clones":0,"num_object_copies":7635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":162880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":849,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698234+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:39:55.698234+0000","last_peered":"2023-10-09T06:39:55.698234+0000","last_clean":"2023-10-09T06:39:55.698234+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:39:55.698234+0000","last_undegraded":"2023-10-09T06:39:55.698234+0000","last_fullsized":"2023-10-09T06:39:55.698234+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":479,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.694353+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:39:55.694353+0000","last_peered":"2023-10-09T06:39:55.694353+0000","last_clean":"2023-10-09T06:39:55.694353+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:39:55.694353+0000","last_undegraded":"2023-10-09T06:39:55.694353+0000","last_fullsized":"2023-10-09T06:39:55.694353+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":462,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686534+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:39:55.686534+0000","last_peered":"2023-10-09T06:39:55.686534+0000","last_clean":"2023-10-09T06:39:55.686534+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:39:55.686534+0000","last_undegraded":"2023-10-09T06:39:55.686534+0000","last_fullsized":"2023-10-09T06:39:55.686534+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":458,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686223+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:39:55.686223+0000","last_peered":"2023-10-09T06:39:55.686223+0000","last_clean":"2023-10-09T06:39:55.686223+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:39:55.686223+0000","last_undegraded":"2023-10-09T06:39:55.686223+0000","last_fullsized":"2023-10-09T06:39:55.686223+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":776,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.692738+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:39:55.692738+0000","last_peered":"2023-10-09T06:39:55.692738+0000","last_clean":"2023-10-09T06:39:55.692738+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:39:55.692738+0000","last_undegraded":"2023-10-09T06:39:55.692738+0000","last_fullsized":"2023-10-09T06:39:55.692738+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":463,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685866+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:39:55.685866+0000","last_peered":"2023-10-09T06:39:55.685866+0000","last_clean":"2023-10-09T06:39:55.685866+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:39:55.685866+0000","last_undegraded":"2023-10-09T06:39:55.685866+0000","last_fullsized":"2023-10-09T06:39:55.685866+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":485,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.689875+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:39:55.689875+0000","last_peered":"2023-10-09T06:39:55.689875+0000","last_clean":"2023-10-09T06:39:55.689875+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:39:55.689875+0000","last_undegraded":"2023-10-09T06:39:55.689875+0000","last_fullsized":"2023-10-09T06:39:55.689875+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":706,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697493+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:39:55.697493+0000","last_peered":"2023-10-09T06:39:55.697493+0000","last_clean":"2023-10-09T06:39:55.697493+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:39:55.697493+0000","last_undegraded":"2023-10-09T06:39:55.697493+0000","last_fullsized":"2023-10-09T06:39:55.697493+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":173,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687069+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:39:55.687069+0000","last_peered":"2023-10-09T06:39:55.687069+0000","last_clean":"2023-10-09T06:39:55.687069+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:39:55.687069+0000","last_undegraded":"2023-10-09T06:39:55.687069+0000","last_fullsized":"2023-10-09T06:39:55.687069+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":502,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686959+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:39:55.686959+0000","last_peered":"2023-10-09T06:39:55.686959+0000","last_clean":"2023-10-09T06:39:55.686959+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:39:55.686959+0000","last_undegraded":"2023-10-09T06:39:55.686959+0000","last_fullsized":"2023-10-09T06:39:55.686959+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":173,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.687515+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:39:55.687515+0000","last_peered":"2023-10-09T06:39:55.687515+0000","last_clean":"2023-10-09T06:39:55.687515+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:39:55.687515+0000","last_undegraded":"2023-10-09T06:39:55.687515+0000","last_fullsized":"2023-10-09T06:39:55.687515+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":512,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686228+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:39:55.686228+0000","last_peered":"2023-10-09T06:39:55.686228+0000","last_clean":"2023-10-09T06:39:55.686228+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:39:55.686228+0000","last_undegraded":"2023-10-09T06:39:55.686228+0000","last_fullsized":"2023-10-09T06:39:55.686228+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":851,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.698879+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:39:55.698879+0000","last_peered":"2023-10-09T06:39:55.698879+0000","last_clean":"2023-10-09T06:39:55.698879+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:39:55.698879+0000","last_undegraded":"2023-10-09T06:39:55.698879+0000","last_fullsized":"2023-10-09T06:39:55.698879+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":198,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.697683+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:39:55.697683+0000","last_peered":"2023-10-09T06:39:55.697683+0000","last_clean":"2023-10-09T06:39:55.697683+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:39:55.697683+0000","last_undegraded":"2023-10-09T06:39:55.697683+0000","last_fullsized":"2023-10-09T06:39:55.697683+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":215,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.693067+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:39:55.693067+0000","last_peered":"2023-10-09T06:39:55.693067+0000","last_clean":"2023-10-09T06:39:55.693067+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:39:55.693067+0000","last_undegraded":"2023-10-09T06:39:55.693067+0000","last_fullsized":"2023-10-09T06:39:55.693067+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":618,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.690500+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:39:55.690500+0000","last_peered":"2023-10-09T06:39:55.690500+0000","last_clean":"2023-10-09T06:39:55.690500+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:39:55.690500+0000","last_undegraded":"2023-10-09T06:39:55.690500+0000","last_fullsized":"2023-10-09T06:39:55.690500+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":493,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.685470+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:39:55.685470+0000","last_peered":"2023-10-09T06:39:55.685470+0000","last_clean":"2023-10-09T06:39:55.685470+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:39:55.685470+0000","last_undegraded":"2023-10-09T06:39:55.685470+0000","last_fullsized":"2023-10-09T06:39:55.685470+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":455,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-10-09T06:39:55.686060+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:39:55.686060+0000","last_peered":"2023-10-09T06:39:55.686060+0000","last_clean":"2023-10-09T06:39:55.686060+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:39:55.686060+0000","last_undegraded":"2023-10-09T06:39:55.686060+0000","last_fullsized":"2023-10-09T06:39:55.686060+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2618425344,"num_objects":39954,"num_object_clones":0,"num_object_copies":119862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39954,"num_write_kb":2557056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7911374848,"data_stored":7911374848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39954,"ondisk_log_size":39954,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234514,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965260,"kb_used_data":468356,"kb_used_omap":0,"kb_used_meta":1496896,"kb_avail":91779892,"statfs":{"total":95995035648,"available":93982609408,"internally_reserved":0,"allocated":479596544,"data_stored":478531095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532821504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":1.4510000000000001,"15min":4.3040000000000003},"min":{"1min":0.25700000000000001,"5min":0.223,"15min":0.218},"max":{"1min":8.0050000000000008,"5min":8.8819999999999997,"15min":215.97800000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.4870000000000001,"15min":4.3499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":10.449999999999999,"5min":10.449999999999999,"15min":215.577},"last":1.5609999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":6.7649999999999997,"15min":5.3440000000000003},"min":{"1min":0.26400000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.6459999999999999,"5min":208.626,"15min":215.61699999999999},"last":2.2570000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":4.9900000000000002,"15min":4.4630000000000001},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.23300000000000001},"max":{"1min":4.923,"5min":181.24199999999999,"15min":216.22200000000001},"last":0.69599999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":8.5220000000000002,"15min":6.4269999999999996},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.276,"5min":228.24000000000001,"15min":228.24000000000001},"last":2.2320000000000002},{"interface":"front","average":{"1min":1.008,"5min":8.4990000000000006,"15min":6.4619999999999997},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":3.2749999999999999,"5min":228.291,"15min":228.291},"last":0.71799999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":5.835,"15min":5.5499999999999998},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":25.295000000000002,"5min":175.40600000000001,"15min":215.922},"last":1.423},{"interface":"front","average":{"1min":2.5750000000000002,"5min":5.7489999999999997,"15min":5.5540000000000003},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":25.324999999999999,"5min":175.30099999999999,"15min":215.94900000000001},"last":1.4510000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.606,"5min":4.6929999999999996,"15min":3.5009999999999999},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":225.041,"5min":225.041,"15min":225.041},"last":2.3849999999999998},{"interface":"front","average":{"1min":11.720000000000001,"5min":4.3959999999999999,"15min":3.343},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":225.244,"5min":225.244,"15min":225.244},"last":1.341}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.055999999999999,"5min":5.2699999999999996,"15min":3.4220000000000002},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":219.923,"5min":219.923,"15min":219.923},"last":2.1989999999999998},{"interface":"front","average":{"1min":10.946999999999999,"5min":5.0259999999999998,"15min":3.3889999999999998},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.28999999999999998},"max":{"1min":219.86000000000001,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":10.978999999999999,"5min":5.3319999999999999,"15min":3.552},"min":{"1min":0.39700000000000002,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":222.179,"5min":222.179,"15min":222.179},"last":3.1059999999999999},{"interface":"front","average":{"1min":11.074999999999999,"5min":5.2140000000000004,"15min":3.3300000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":223.261,"5min":223.261,"15min":223.261},"last":1.3879999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496150,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799988,"kb_used_data":616684,"kb_used_omap":0,"kb_used_meta":1183296,"kb_avail":91945164,"statfs":{"total":95995035648,"available":94151847936,"internally_reserved":0,"allocated":631484416,"data_stored":630442265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1211695104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.2170000000000001,"15min":2.5009999999999999},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":5.9569999999999999,"5min":66.867000000000004,"15min":75.061999999999998},"last":1.4950000000000001},{"interface":"front","average":{"1min":13.164,"5min":4.8049999999999997,"15min":3.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":212.096,"5min":212.096,"15min":212.096},"last":1.466}]},{"osd":1,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":4.4649999999999999,"15min":4.9279999999999999},"min":{"1min":0.31,"5min":0.26500000000000001,"15min":0.20899999999999999},"max":{"1min":3.8740000000000001,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.75700000000000001,"5min":4.452,"15min":4.9800000000000004},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":2.9630000000000001,"5min":219.94800000000001,"15min":292.904},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.4560000000000004,"15min":3.504},"min":{"1min":0.35299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.8570000000000002,"5min":185.654,"15min":185.654},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":4.3019999999999996,"15min":3.3279999999999998},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":2.2850000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":1.04}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":3.5939999999999999,"15min":3.7570000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":8.5380000000000003,"5min":121.749,"15min":121.749},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.5049999999999999,"15min":3.6829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.4770000000000003,"5min":120.70399999999999,"15min":120.70399999999999},"last":0.79700000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.5680000000000001,"15min":1.5149999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":3.8140000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.317},{"interface":"front","average":{"1min":10.757,"5min":3.8490000000000002,"15min":2.1560000000000001},"min":{"1min":0.32700000000000001,"5min":0.253,"15min":0.223},"max":{"1min":212.03399999999999,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.28999999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.6519999999999999,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":3.835,"5min":9.984,"15min":12.747999999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.0750000000000002,"15min":1.5349999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.242,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.33300000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":3.0640000000000001,"15min":2.7280000000000002},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":8.5050000000000008,"5min":165.37,"15min":165.37},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":3.5030000000000001,"15min":3.0150000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.589,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.36899999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402887,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262308,"kb_used_data":1256412,"kb_used_omap":4,"kb_used_meta":1005883,"kb_avail":91482844,"statfs":{"total":95995035648,"available":93678432256,"internally_reserved":0,"allocated":1286565888,"data_stored":1284427957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1030025102},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":2.117,"15min":4.6619999999999999},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.182},"max":{"1min":5.5190000000000001,"5min":110.479,"15min":269.79199999999997},"last":3.5230000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.5739999999999998,"15min":5.0199999999999996},"min":{"1min":0.22900000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.1989999999999998,"5min":110.59999999999999,"15min":271.20999999999998},"last":0.35299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.042,"15min":2.2109999999999999},"min":{"1min":0.25800000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":2.2170000000000001,"5min":10.284000000000001,"15min":138.452},"last":3.4729999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.6910000000000001,"15min":2.8599999999999999},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":6.234,"5min":10.379,"15min":138.47499999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.6839999999999999,"15min":2.6779999999999999},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":3.1139999999999999,"5min":12.329000000000001,"15min":31.062000000000001},"last":3.593},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.266,"15min":1.76},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":1.972,"5min":10.228999999999999,"15min":27.657},"last":3.5449999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":3.5470000000000002,"15min":2.6499999999999999},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.940000000000001,"5min":120.309,"15min":120.309},"last":3.379},{"interface":"front","average":{"1min":3.0099999999999998,"5min":3.6259999999999999,"15min":2.4420000000000002},"min":{"1min":0.41099999999999998,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":29.018000000000001,"5min":120.297,"15min":120.297},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.6019999999999999,"15min":2.4910000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":4.7050000000000001,"5min":119.408,"15min":119.408},"last":3.5009999999999999},{"interface":"front","average":{"1min":1.3640000000000001,"5min":4.1130000000000004,"15min":2.8220000000000001},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":6.1379999999999999,"5min":119.483,"15min":119.483},"last":0.45000000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8900000000000001,"5min":3.2200000000000002,"15min":2.96},"min":{"1min":0.36599999999999999,"5min":0.317,"15min":0.317},"max":{"1min":25.771000000000001,"5min":121.60899999999999,"15min":121.60899999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.0960000000000001,"5min":3.222,"15min":3.262},"min":{"1min":0.40300000000000002,"5min":0.39900000000000002,"15min":0.316},"max":{"1min":28.588999999999999,"5min":121.559,"15min":121.559},"last":3.456}]},{"osd":7,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":4.0300000000000002,"15min":3.7519999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":3.1389999999999998,"5min":123.20699999999999,"15min":123.20699999999999},"last":3.431},{"interface":"front","average":{"1min":0.68300000000000005,"5min":3.544,"15min":3.153},"min":{"1min":0.372,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":2.177,"5min":123.48999999999999,"15min":123.48999999999999},"last":3.5659999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024321,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334384,"kb_used_data":1288552,"kb_used_omap":5,"kb_used_meta":1045818,"kb_avail":91410768,"statfs":{"total":95995035648,"available":93604626432,"internally_reserved":0,"allocated":1319477248,"data_stored":1316981818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1070918266},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.6980000000000004,"5min":4.5549999999999997,"15min":3.387},"min":{"1min":0.20699999999999999,"5min":0.184,"15min":0.183},"max":{"1min":102.182,"5min":102.182,"15min":191.446},"last":2.613},{"interface":"front","average":{"1min":6.5830000000000002,"5min":4.9539999999999997,"15min":3.7930000000000001},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.187},"max":{"1min":101.282,"5min":101.282,"15min":191.393},"last":1.0309999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":5.524,"15min":3.7389999999999999},"min":{"1min":0.248,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":2.3769999999999998,"5min":100.425,"15min":100.425},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":5.4660000000000002,"15min":4.1189999999999998},"min":{"1min":0.27300000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.3599999999999999,"5min":99.769999999999996,"15min":99.769999999999996},"last":4.1989999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":7.5549999999999997,"5min":3.9510000000000001,"15min":5.0010000000000003},"min":{"1min":0.313,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":95.555000000000007,"5min":95.555000000000007,"15min":208.11199999999999},"last":10.454000000000001},{"interface":"front","average":{"1min":7.7720000000000002,"5min":4.133,"15min":5.0060000000000002},"min":{"1min":0.32800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":97.605000000000004,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0569999999999999,"5min":5.9390000000000001,"15min":3.02},"min":{"1min":0.35799999999999998,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":11.519,"5min":200.654,"15min":200.654},"last":1.958},{"interface":"front","average":{"1min":3.2050000000000001,"5min":6.6100000000000003,"15min":3.5059999999999998},"min":{"1min":0.38200000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.0790000000000006,"5min":200.86600000000001,"15min":200.86600000000001},"last":1.1499999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":2.6989999999999998,"15min":2.0129999999999999},"min":{"1min":0.32900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.036,"5min":80.885999999999996,"15min":80.885999999999996},"last":2.0169999999999999},{"interface":"front","average":{"1min":1.875,"5min":2.605,"15min":2.2210000000000001},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.2089999999999996,"5min":80.438999999999993,"15min":80.438999999999993},"last":4.2679999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":3.3069999999999999,"15min":3.2559999999999998},"min":{"1min":0.39100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":7.1660000000000004,"5min":118.056,"15min":120.572},"last":4.2380000000000004},{"interface":"front","average":{"1min":2.448,"5min":3.1509999999999998,"15min":2.8929999999999998},"min":{"1min":0.39300000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":11.563000000000001,"5min":118.274,"15min":120.64400000000001},"last":1.9930000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:39:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2090000000000001,"5min":2.2200000000000002,"15min":2.5670000000000002},"min":{"1min":0.497,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":7.0270000000000001,"5min":22.879000000000001,"15min":122.018},"last":0.58099999999999996},{"interface":"front","average":{"1min":3.601,"5min":2.6579999999999999,"15min":2.9009999999999998},"min":{"1min":0.54100000000000004,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.9190000000000005,"5min":22.683,"15min":122.32299999999999},"last":4.3129999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814190,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2131812,"kb_used_data":1186716,"kb_used_omap":7,"kb_used_meta":945080,"kb_avail":91613340,"statfs":{"total":95995035648,"available":93812060160,"internally_reserved":0,"allocated":1215197184,"data_stored":1213116568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":967762001},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":11.968999999999999,"5min":6.7930000000000001,"15min":6.5099999999999998},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.19400000000000001},"max":{"1min":214.595,"5min":214.595,"15min":214.595},"last":106.348},{"interface":"front","average":{"1min":11.67,"5min":6.335,"15min":5.7939999999999996},"min":{"1min":0.23899999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":214.94300000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":106.979}]},{"osd":1,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":3.2970000000000002,"15min":4.3250000000000002},"min":{"1min":0.20499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6750000000000007,"5min":80.141000000000005,"15min":140.452},"last":1.351},{"interface":"front","average":{"1min":1.1259999999999999,"5min":3.4409999999999998,"15min":4.5010000000000003},"min":{"1min":0.26000000000000001,"5min":0.224,"15min":0.183},"max":{"1min":4.7859999999999996,"5min":79.474000000000004,"15min":139.75299999999999},"last":0.51900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:39:29 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.3029999999999999,"15min":3.548},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":7.6870000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":1.302},{"interface":"front","average":{"1min":2.004,"5min":3.3860000000000001,"15min":3.899},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":8.8510000000000009,"5min":79.614000000000004,"15min":91.685000000000002},"last":0.56499999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":3.3109999999999999,"15min":2.4630000000000001},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3570000000000002,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.224,"5min":3.5569999999999999,"15min":2.7509999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":5.2889999999999997,"5min":82.872,"15min":82.872},"last":0.88300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.6110000000000002,"15min":2.3639999999999999},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.5199999999999996,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.212,"5min":3.7320000000000002,"15min":2.532},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":5.3300000000000001,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.90500000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":3.2559999999999998,"15min":3.4180000000000001},"min":{"1min":0.35199999999999998,"5min":0.313,"15min":0.30199999999999999},"max":{"1min":5.3630000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.4369999999999998,"15min":3.367},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":9.3740000000000006,"5min":80.364999999999995,"15min":122.517},"last":0.40799999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":3.786,"15min":3.0459999999999998},"min":{"1min":0.39200000000000002,"5min":0.35199999999999998,"15min":0.34300000000000003},"max":{"1min":9.4190000000000005,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":3.3079999999999998,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":5.5990000000000002,"5min":82.798000000000002,"15min":82.798000000000002},"last":0.61099999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127387,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1913012,"kb_used_data":1128556,"kb_used_omap":2,"kb_used_meta":784381,"kb_avail":91832140,"statfs":{"total":95995035648,"available":94036111360,"internally_reserved":0,"allocated":1155641344,"data_stored":1153663815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":803206197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.9410000000000001,"15min":4.2610000000000001},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.19},"max":{"1min":16.562000000000001,"5min":16.562000000000001,"15min":161.821},"last":1.593},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.121,"15min":3.9940000000000002},"min":{"1min":0.192,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":6.6050000000000004,"5min":12.298,"15min":163.52000000000001},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":4.5129999999999999,"15min":3.863},"min":{"1min":0.27100000000000002,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":5.1630000000000003,"5min":184.012,"15min":184.012},"last":0.442},{"interface":"front","average":{"1min":2.246,"5min":4.7699999999999996,"15min":4.6399999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":22.184000000000001,"5min":184.703,"15min":184.703},"last":0.999}]},{"osd":2,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":3.7320000000000002,"15min":3.5990000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":12.390000000000001,"5min":182.44900000000001,"15min":182.44900000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.1440000000000001,"5min":4.0220000000000002,"15min":4.0590000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":18.109000000000002,"5min":182.56399999999999,"15min":182.56399999999999},"last":0.629}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.625,"5min":5.5949999999999998,"15min":3.3519999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26900000000000002},"max":{"1min":133.80099999999999,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":8.8919999999999995,"5min":5.6079999999999997,"15min":3.359},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":133.648,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4130000000000003,"5min":6.4509999999999996,"15min":3.5859999999999999},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":128.083,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":8.2870000000000008,"5min":5.3540000000000001,"15min":2.9510000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":128.11799999999999,"5min":177.59399999999999,"15min":177.59399999999999},"last":0.50700000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.016,"15min":2.7530000000000001},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.5779999999999994,"5min":11.994999999999999,"15min":130.39599999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.581,"15min":2.3199999999999998},"min":{"1min":0.38300000000000001,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":5.359,"5min":9.8179999999999996,"15min":130.67699999999999},"last":0.47099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.0510000000000002,"15min":2.7930000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.7590000000000003,"5min":10.195,"15min":121.613},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.3140000000000001,"5min":2.8149999999999999,"15min":3.1779999999999999},"min":{"1min":0.42699999999999999,"5min":0.372,"15min":0.33400000000000002},"max":{"1min":5.1849999999999996,"5min":87.001999999999995,"15min":121.559},"last":0.48499999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019423,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2468860,"kb_used_data":993396,"kb_used_omap":0,"kb_used_meta":1475456,"kb_avail":91276292,"statfs":{"total":95995035648,"available":93466923008,"internally_reserved":0,"allocated":1017237504,"data_stored":1016187538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1510866944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":4.7210000000000001,"15min":6.1859999999999999},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":3.0750000000000002,"5min":286.30099999999999,"15min":287.58800000000002},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.298,"5min":5.3399999999999999,"15min":6.4199999999999999},"min":{"1min":0.317,"5min":0.23200000000000001,"15min":0.23100000000000001},"max":{"1min":4.202,"5min":286.36000000000001,"15min":287.57100000000003},"last":0.30599999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":8.7319999999999993,"5min":3.141,"15min":3.7349999999999999},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.22900000000000001},"max":{"1min":163.392,"5min":163.392,"15min":163.392},"last":0.51600000000000001},{"interface":"front","average":{"1min":8.9659999999999993,"5min":3.1749999999999998,"15min":3.8479999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":163.42400000000001,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.60399999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":9.641,"5min":5.5220000000000002,"15min":4.8550000000000004},"min":{"1min":0.32400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":175.97900000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":9.4800000000000004,"5min":5.657,"15min":4.7969999999999997},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":175.875,"5min":175.875,"15min":175.875},"last":0.68899999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.5409999999999999,"15min":5.6559999999999997},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":6.8920000000000003,"5min":94.984999999999999,"15min":161.19300000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.411,"5min":3.1480000000000001,"15min":5.3979999999999997},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.524,"5min":94.966999999999999,"15min":161.934},"last":1.1870000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.3819999999999999,"15min":1.028},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":1.1799999999999999,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":1.518,"15min":1.1080000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":0.879,"5min":9.5009999999999994,"15min":9.5009999999999994},"last":0.34999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.7400000000000002,"15min":1.958},"min":{"1min":0.29399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.2879999999999998,"5min":115.70999999999999,"15min":163.36699999999999},"last":0.371},{"interface":"front","average":{"1min":1.073,"5min":3.0670000000000002,"15min":2.258},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29199999999999998},"max":{"1min":4.3200000000000003,"5min":115.68600000000001,"15min":163.381},"last":0.32700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4820000000000002,"15min":1.6950000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":3.8140000000000001,"5min":106.801,"15min":106.801},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.6030000000000002,"15min":1.722},"min":{"1min":0.40000000000000002,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":3.3300000000000001,"5min":106.827,"15min":106.827},"last":0.39800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757788,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2396852,"kb_used_data":905068,"kb_used_omap":0,"kb_used_meta":1491776,"kb_avail":91348300,"statfs":{"total":95995035648,"available":93540659200,"internally_reserved":0,"allocated":926789632,"data_stored":925744110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1527578624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":6.8890000000000002,"15min":6.0650000000000004},"min":{"1min":0.30099999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.5899999999999999,"5min":149.149,"15min":294.89699999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":2.214,"5min":7.9660000000000002,"15min":6.202},"min":{"1min":0.24099999999999999,"5min":0.215,"15min":0.215},"max":{"1min":6.9029999999999996,"5min":149.21100000000001,"15min":294.30399999999997},"last":0.28699999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:40:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":13.241,"15min":6.9169999999999998},"min":{"1min":0.26400000000000001,"5min":0.22,"15min":0.22},"max":{"1min":2.9169999999999998,"5min":188.34100000000001,"15min":188.34100000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.81699999999999995,"5min":12.952999999999999,"15min":6.9160000000000004},"min":{"1min":0.27700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.2280000000000002,"5min":188.839,"15min":188.839},"last":2.1269999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:40:00 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":7.5,"15min":4.306},"min":{"1min":0.29799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0630000000000002,"5min":120.613,"15min":120.613},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":7.2720000000000002,"15min":4.1529999999999996},"min":{"1min":0.307,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":2.9889999999999999,"5min":120.962,"15min":120.962},"last":0.873}]},{"osd":3,"last update":"Mon Oct 9 06:39:18 2023","interfaces":[{"interface":"back","average":{"1min":15.433,"5min":8.0549999999999997,"15min":6.6719999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":220.41800000000001,"5min":220.41800000000001,"15min":220.41800000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":15.595000000000001,"5min":7.508,"15min":6.3470000000000004},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.28699999999999998},"max":{"1min":220.45500000000001,"5min":220.45500000000001,"15min":220.45500000000001},"last":1.7490000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":2.4340000000000002,"15min":1.478},"min":{"1min":0.29399999999999998,"5min":0.255,"15min":0.247},"max":{"1min":2.8740000000000001,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":2.6080000000000001,"15min":1.4790000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":2.8319999999999999,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 06:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.8480000000000001,"15min":1.655},"min":{"1min":0.35299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":6.5810000000000004,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1.9419999999999999,"15min":1.7350000000000001},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0819999999999999,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.40999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.4710000000000001,"15min":1.6779999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":2.9430000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":2.1589999999999998,"15min":1.5349999999999999},"min":{"1min":0.38300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.859,"5min":79.138999999999996,"15min":79.138999999999996},"last":0.52400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1330708480,"data_stored":1330708480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1306001408,"data_stored":1306001408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1238761472,"data_stored":1238761472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1167851520,"data_stored":1167851520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":953417728,"data_stored":953417728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":799997952,"data_stored":799997952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":637403136,"data_stored":637403136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":477233152,"data_stored":477233152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:40:14.100 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:40:14.100 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:40:14.100 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T06:40:14.100 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T06:40:14.100 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T06:40:14.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T06:40:14.109 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 29 16 46165 46149 99.3992 55.3125 0.0112052 0.010053 2023-10-09T06:40:14.141 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:40:14.168 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:14.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:14.278 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:14.288 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:14.324 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:14.398 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:14.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:14.505 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:14.559 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:14.568 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:40:14.682 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:14.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:14.789 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:14.830 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:14 smithi107 ceph-mon[220921]: pgmap v1529: 119 pgs: 119 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:40:14.830 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:14 smithi107 ceph-mon[212750]: pgmap v1529: 119 pgs: 119 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:40:14.835 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:14.907 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:14.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:15.012 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.040 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:14 smithi150 ceph-mon[194178]: pgmap v1529: 119 pgs: 119 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T06:40:15.043 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:15.108 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:15.112 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 30 16 47427 47411 98.7053 78.875 0.0130411 0.0101252 2023-10-09T06:40:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:15.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:15.215 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:15.262 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:15.331 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:15.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:15.435 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:15.463 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:15.476 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:40:15.504 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:15.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:15.608 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:15.634 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:15.697 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:15.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:15.799 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:15.828 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:15.885 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:15.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:15.989 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:16.003 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:16.070 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:16.112 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 31 16 48418 48402 97.5198 61.9375 0.0118024 0.0102482 2023-10-09T06:40:16.177 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:16.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:16.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:16.287 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:16.317 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:16.550 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:16.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:16.622 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:16.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:16.749 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:17.103 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:17.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[220921]: pgmap v1530: 119 pgs: 119 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:40:17.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:17.109 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:17.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:17.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:17.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[212750]: pgmap v1530: 119 pgs: 119 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:40:17.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:17.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:17.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 32 16 49584 49568 96.7499 72.875 0.00792825 0.0103293 2023-10-09T06:40:17.115 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:40:17.115 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "" 2023-10-09T06:40:17.116 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:40:17.137 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:16 smithi150 ceph-mon[194178]: pgmap v1530: 119 pgs: 119 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-09T06:40:17.138 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:17.138 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:17.211 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:17.260 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:17.349 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:17.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:17.453 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:17.501 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:17.640 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:17.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:17.746 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:17.778 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:17.847 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:40:18.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2959215809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:17 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:18.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:18.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 33 16 50652 50636 95.8412 66.75 0.0182479 0.0104293 2023-10-09T06:40:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2959215809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:17 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2959215809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:17 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:40:18.154 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:18.188 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:18.261 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:18.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:18.365 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:18.430 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: pgmap v1531: 119 pgs: 119 active+clean; 2.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.080 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.080 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.080 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[220921]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: pgmap v1531: 119 pgs: 119 active+clean; 2.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.081 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:18 smithi107 ceph-mon[212750]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 34 16 51813 51797 95.1567 72.5625 0.009062 0.0105045 2023-10-09T06:40:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: pgmap v1531: 119 pgs: 119 active+clean; 2.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T06:40:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:40:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: osdmap e496: 8 total, 8 up, 8 in 2023-10-09T06:40:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:18 smithi150 ceph-mon[194178]: failed to encode map e496 with expected crc 2023-10-09T06:40:19.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:19.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:19.766 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:19.843 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:20.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 35 16 52842 52826 94.2756 64.3125 0.00696214 0.0106021 2023-10-09T06:40:20.142 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:40:20.142 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:40:20.142 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:40:20.143 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:40:20.143 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:40:20.172 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:20.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:20.279 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:20.344 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:20.500 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:20.524 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:20.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:20.539 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:20.588 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:20.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:20.661 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:20.703 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:20.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:20.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:20.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:20.837 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:20.879 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:20.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:20.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:20.886 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:20.919 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:20.951 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:20.984 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:20.990 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:21.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:20 smithi150 ceph-mon[194178]: pgmap v1533: 119 pgs: 119 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:21.017 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:20 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1204093410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:21.020 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:21.022 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:21.027 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.027 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.027 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.027 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.028 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.028 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.028 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.028 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:40:21.081 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:21.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:21.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:21.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 36 16 53752 53736 93.2359 56.875 0.0070532 0.0107203 2023-10-09T06:40:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:20 smithi107 ceph-mon[220921]: pgmap v1533: 119 pgs: 119 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:21.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:20 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1204093410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:21.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:20 smithi107 ceph-mon[212750]: pgmap v1533: 119 pgs: 119 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:21.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:20 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1204093410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:21.187 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:21.223 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:21.278 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:21.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:21.383 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:21.417 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:21.702 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:40:21.768 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:21.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:40:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:21.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:21.876 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:21.909 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:21.965 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:21.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:22.071 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:22.099 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:22.113 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 37 16 54717 54701 92.3464 60.3125 0.0114565 0.0108228 2023-10-09T06:40:22.166 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:22.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:22.273 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:22.338 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:22] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T06:40:22.347 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:22.454 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:22.504 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:22.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:22.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:22.573 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:22.614 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:22.682 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:22.821 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:22.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:22.929 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:22.958 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:22 smithi107 ceph-mon[212750]: pgmap v1534: 119 pgs: 119 active+clean; 3.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:22.959 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:22 smithi107 ceph-mon[220921]: pgmap v1534: 119 pgs: 119 active+clean; 3.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:22.959 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:23.014 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:23.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:23.061 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:40:23.061 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:40:23.062 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-10-09T06:40:23.062 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:40:23.114 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 38 16 55782 55766 91.6678 66.5625 0.00773582 0.0109042 2023-10-09T06:40:23.123 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:23.148 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:22 smithi150 ceph-mon[194178]: pgmap v1534: 119 pgs: 119 active+clean; 3.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T06:40:23.151 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:23.201 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:23.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:23.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:23.309 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:23.369 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:24.059 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:40:24.117 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 39 16 56815 56799 90.9644 64.5625 0.01704 0.0109888 2023-10-09T06:40:24.207 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:24.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:24.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:24.342 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:24.384 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:24.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:24.491 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:24.561 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:24.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:24 smithi107 ceph-mon[220921]: pgmap v1535: 119 pgs: 119 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:24.855 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:24 smithi107 ceph-mon[212750]: pgmap v1535: 119 pgs: 119 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:24.934 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:24.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:25.065 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:25.096 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:25.521 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:40:25.118363+0000 min lat: 0.0010258 max lat: 0.405258 avg lat: 0.0111017 2023-10-09T06:40:25.521 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:25.521 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 40 16 57667 57651 90.022 53.25 0.0120791 0.0111017 2023-10-09T06:40:25.522 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:25.522 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:25.523 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:24 smithi150 ceph-mon[194178]: pgmap v1535: 119 pgs: 119 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T06:40:25.523 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:25.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:25.625 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:25.697 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:25.765 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:40:25.765 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:40:25.898 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:25.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:26.008 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:26.193 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 41 16 58728 58712 89.4438 66.3125 0.0053068 0.0111755 2023-10-09T06:40:26.235 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:26.288 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:26.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:26.396 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:26.428 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:26.490 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:26.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:26.597 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:26.630 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:26.686 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:26.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:26.791 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:26.821 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:26.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:26.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:26.898 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:40:26.988 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:27.026 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:27.118 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 42 16 59685 59669 88.7385 59.8125 0.00275911 0.0112331 2023-10-09T06:40:27.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:26 smithi150 ceph-mon[194178]: pgmap v1536: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:27.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1928812156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.387 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:26 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.387 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[212750]: pgmap v1536: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:27.670 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1928812156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.670 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[220921]: pgmap v1536: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T06:40:27.670 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1928812156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.671 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:26 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:27.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:27.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[212750]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.118 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 43 16 60695 60679 88.143 63.125 0.00258005 0.0113373 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:27 smithi107 ceph-mon[220921]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: osdmap e497: 8 total, 8 up, 8 in 2023-10-09T06:40:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:27 smithi150 ceph-mon[194178]: failed to encode map e497 with expected crc 2023-10-09T06:40:29.090 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:40:29.121 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 44 16 61623 61607 87.455 58 0.0639065 0.0114221 2023-10-09T06:40:29.190 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:28 smithi107 ceph-mon[212750]: pgmap v1538: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T06:40:29.194 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:28 smithi107 ceph-mon[220921]: pgmap v1538: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T06:40:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:28 smithi150 ceph-mon[194178]: pgmap v1538: 119 pgs: 119 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-09T06:40:29.656 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:29.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:29.769 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:29.853 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:30.029 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:30.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:30.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:30.120 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 45 16 62662 62646 86.9547 64.9375 0.0109117 0.0114949 2023-10-09T06:40:30.137 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:30.165 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:30.223 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:30.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:30.332 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:30.361 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:30.428 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:30.482 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:30.525 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:30.547 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:30.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:30.565 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:30.624 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:30.656 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:30.662 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:30.672 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:40:30.689 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:30.698 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:40:30.705 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:30.742 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:30.748 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:30.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:30.783 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.792 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:40:30.855 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:30.923 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:31.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:30 smithi107 ceph-mon[212750]: pgmap v1539: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:31.025 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/332309554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:31.027 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:30 smithi107 ceph-mon[220921]: pgmap v1539: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:31.027 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/332309554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:31.120 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 46 16 63597 63581 86.335 58.4375 0.0218427 0.0115778 2023-10-09T06:40:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:30 smithi150 ceph-mon[194178]: pgmap v1539: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/332309554' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:31.655 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:40:32.014 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[220921]: from='client.47108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:32.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:32.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:32.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[212750]: from='client.47108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:32.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:32.017 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:32.017 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:32] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-10-09T06:40:32.121 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 47 16 64600 64584 85.8286 62.6875 0.00895512 0.0116465 2023-10-09T06:40:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:31 smithi150 ceph-mon[194178]: from='client.47108 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:40:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:33.117 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:33.122 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 48 16 65553 65537 85.2816 59.5625 0.00451635 0.0117205 2023-10-09T06:40:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:33.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:33.228 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:33.285 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:32 smithi107 ceph-mon[220921]: pgmap v1540: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:33.289 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:32 smithi107 ceph-mon[212750]: pgmap v1540: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:33.290 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:33.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:32 smithi150 ceph-mon[194178]: pgmap v1540: 119 pgs: 119 active+clean; 3.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T06:40:33.442 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1539,"stamp":"2023-10-09T06:40:29.561618+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4012447648,"num_objects":61240,"num_object_clones":0,"num_object_copies":183720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61277,"num_write_kb":3917853,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66262,"ondisk_log_size":66262,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21160176,"kb_used_data":11681392,"kb_used_omap":20,"kb_used_meta":9478635,"kb_avail":728801040,"statfs":{"total":767960285184,"available":746292264960,"internally_reserved":0,"allocated":11961745408,"data_stored":11948823747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9706122894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":663814144,"num_objects":10129,"num_object_clones":0,"num_object_copies":30387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10129,"num_write_kb":648256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005715"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":791,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836602+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:40:17.836602+0000","last_peered":"2023-10-09T06:40:17.836602+0000","last_clean":"2023-10-09T06:40:17.836602+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:40:17.836602+0000","last_undegraded":"2023-10-09T06:40:17.836602+0000","last_fullsized":"2023-10-09T06:40:17.836602+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":476,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823604+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:40:17.823604+0000","last_peered":"2023-10-09T06:40:17.823604+0000","last_clean":"2023-10-09T06:40:17.823604+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:40:17.823604+0000","last_undegraded":"2023-10-09T06:40:17.823604+0000","last_fullsized":"2023-10-09T06:40:17.823604+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833974+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:40:17.833974+0000","last_peered":"2023-10-09T06:40:17.833974+0000","last_clean":"2023-10-09T06:40:17.833974+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:40:17.833974+0000","last_undegraded":"2023-10-09T06:40:17.833974+0000","last_fullsized":"2023-10-09T06:40:17.833974+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":752,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823616+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:40:17.823616+0000","last_peered":"2023-10-09T06:40:17.823616+0000","last_clean":"2023-10-09T06:40:17.823616+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:40:17.823616+0000","last_undegraded":"2023-10-09T06:40:17.823616+0000","last_fullsized":"2023-10-09T06:40:17.823616+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":493,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842676+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:40:17.842676+0000","last_peered":"2023-10-09T06:40:17.842676+0000","last_clean":"2023-10-09T06:40:17.842676+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:40:17.842676+0000","last_undegraded":"2023-10-09T06:40:17.842676+0000","last_fullsized":"2023-10-09T06:40:17.842676+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":498,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:19.309604+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:40:19.309604+0000","last_peered":"2023-10-09T06:40:19.309604+0000","last_clean":"2023-10-09T06:40:19.309604+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:40:19.309604+0000","last_undegraded":"2023-10-09T06:40:19.309604+0000","last_fullsized":"2023-10-09T06:40:19.309604+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":744,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823569+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:40:17.823569+0000","last_peered":"2023-10-09T06:40:17.823569+0000","last_clean":"2023-10-09T06:40:17.823569+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:40:17.823569+0000","last_undegraded":"2023-10-09T06:40:17.823569+0000","last_fullsized":"2023-10-09T06:40:17.823569+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":217,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.832932+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:40:17.832932+0000","last_peered":"2023-10-09T06:40:17.832932+0000","last_clean":"2023-10-09T06:40:17.832932+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:40:17.832932+0000","last_undegraded":"2023-10-09T06:40:17.832932+0000","last_fullsized":"2023-10-09T06:40:17.832932+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":486,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844308+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:40:17.844308+0000","last_peered":"2023-10-09T06:40:17.844308+0000","last_clean":"2023-10-09T06:40:17.844308+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:40:17.844308+0000","last_undegraded":"2023-10-09T06:40:17.844308+0000","last_fullsized":"2023-10-09T06:40:17.844308+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":605,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836867+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:40:17.836867+0000","last_peered":"2023-10-09T06:40:17.836867+0000","last_clean":"2023-10-09T06:40:17.836867+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:40:17.836867+0000","last_undegraded":"2023-10-09T06:40:17.836867+0000","last_fullsized":"2023-10-09T06:40:17.836867+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.832920+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:40:17.832920+0000","last_peered":"2023-10-09T06:40:17.832920+0000","last_clean":"2023-10-09T06:40:17.832920+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:40:17.832920+0000","last_undegraded":"2023-10-09T06:40:17.832920+0000","last_fullsized":"2023-10-09T06:40:17.832920+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833162+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:40:17.833162+0000","last_peered":"2023-10-09T06:40:17.833162+0000","last_clean":"2023-10-09T06:40:17.833162+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:40:17.833162+0000","last_undegraded":"2023-10-09T06:40:17.833162+0000","last_fullsized":"2023-10-09T06:40:17.833162+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":642,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837087+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:40:17.837087+0000","last_peered":"2023-10-09T06:40:17.837087+0000","last_clean":"2023-10-09T06:40:17.837087+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:40:17.837087+0000","last_undegraded":"2023-10-09T06:40:17.837087+0000","last_fullsized":"2023-10-09T06:40:17.837087+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":184,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.839204+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:40:17.839204+0000","last_peered":"2023-10-09T06:40:17.839204+0000","last_clean":"2023-10-09T06:40:17.839204+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:40:17.839204+0000","last_undegraded":"2023-10-09T06:40:17.839204+0000","last_fullsized":"2023-10-09T06:40:17.839204+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834231+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:40:17.834231+0000","last_peered":"2023-10-09T06:40:17.834231+0000","last_clean":"2023-10-09T06:40:17.834231+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:40:17.834231+0000","last_undegraded":"2023-10-09T06:40:17.834231+0000","last_fullsized":"2023-10-09T06:40:17.834231+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":625,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.832655+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:40:17.832655+0000","last_peered":"2023-10-09T06:40:17.832655+0000","last_clean":"2023-10-09T06:40:17.832655+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:40:17.832655+0000","last_undegraded":"2023-10-09T06:40:17.832655+0000","last_fullsized":"2023-10-09T06:40:17.832655+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":378,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837917+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:40:17.837917+0000","last_peered":"2023-10-09T06:40:17.837917+0000","last_clean":"2023-10-09T06:40:17.837917+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:40:17.837917+0000","last_undegraded":"2023-10-09T06:40:17.837917+0000","last_fullsized":"2023-10-09T06:40:17.837917+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":434,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.847587+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:40:17.847587+0000","last_peered":"2023-10-09T06:40:17.847587+0000","last_clean":"2023-10-09T06:40:17.847587+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:40:17.847587+0000","last_undegraded":"2023-10-09T06:40:17.847587+0000","last_fullsized":"2023-10-09T06:40:17.847587+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":606,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843839+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:40:17.843839+0000","last_peered":"2023-10-09T06:40:17.843839+0000","last_clean":"2023-10-09T06:40:17.843839+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:40:17.843839+0000","last_undegraded":"2023-10-09T06:40:17.843839+0000","last_fullsized":"2023-10-09T06:40:17.843839+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":503,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824002+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:40:17.824002+0000","last_peered":"2023-10-09T06:40:17.824002+0000","last_clean":"2023-10-09T06:40:17.824002+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:40:17.824002+0000","last_undegraded":"2023-10-09T06:40:17.824002+0000","last_fullsized":"2023-10-09T06:40:17.824002+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":467,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824140+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:40:17.824140+0000","last_peered":"2023-10-09T06:40:17.824140+0000","last_clean":"2023-10-09T06:40:17.824140+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:40:17.824140+0000","last_undegraded":"2023-10-09T06:40:17.824140+0000","last_fullsized":"2023-10-09T06:40:17.824140+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":770,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.829462+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:40:17.829462+0000","last_peered":"2023-10-09T06:40:17.829462+0000","last_clean":"2023-10-09T06:40:17.829462+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:40:17.829462+0000","last_undegraded":"2023-10-09T06:40:17.829462+0000","last_fullsized":"2023-10-09T06:40:17.829462+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":346,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837316+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:40:17.837316+0000","last_peered":"2023-10-09T06:40:17.837316+0000","last_clean":"2023-10-09T06:40:17.837316+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:40:17.837316+0000","last_undegraded":"2023-10-09T06:40:17.837316+0000","last_fullsized":"2023-10-09T06:40:17.837316+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834912+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:40:17.834912+0000","last_peered":"2023-10-09T06:40:17.834912+0000","last_clean":"2023-10-09T06:40:17.834912+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:40:17.834912+0000","last_undegraded":"2023-10-09T06:40:17.834912+0000","last_fullsized":"2023-10-09T06:40:17.834912+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":769,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836630+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:40:17.836630+0000","last_peered":"2023-10-09T06:40:17.836630+0000","last_clean":"2023-10-09T06:40:17.836630+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:40:17.836630+0000","last_undegraded":"2023-10-09T06:40:17.836630+0000","last_fullsized":"2023-10-09T06:40:17.836630+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":495,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.832474+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:40:17.832474+0000","last_peered":"2023-10-09T06:40:17.832474+0000","last_clean":"2023-10-09T06:40:17.832474+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:40:17.832474+0000","last_undegraded":"2023-10-09T06:40:17.832474+0000","last_fullsized":"2023-10-09T06:40:17.832474+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":317,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837829+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:40:17.837829+0000","last_peered":"2023-10-09T06:40:17.837829+0000","last_clean":"2023-10-09T06:40:17.837829+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:40:17.837829+0000","last_undegraded":"2023-10-09T06:40:17.837829+0000","last_fullsized":"2023-10-09T06:40:17.837829+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":824,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842955+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:40:17.842955+0000","last_peered":"2023-10-09T06:40:17.842955+0000","last_clean":"2023-10-09T06:40:17.842955+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:40:17.842955+0000","last_undegraded":"2023-10-09T06:40:17.842955+0000","last_fullsized":"2023-10-09T06:40:17.842955+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":340,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836770+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:40:17.836770+0000","last_peered":"2023-10-09T06:40:17.836770+0000","last_clean":"2023-10-09T06:40:17.836770+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:40:17.836770+0000","last_undegraded":"2023-10-09T06:40:17.836770+0000","last_fullsized":"2023-10-09T06:40:17.836770+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":172,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.829817+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:40:17.829817+0000","last_peered":"2023-10-09T06:40:17.829817+0000","last_clean":"2023-10-09T06:40:17.829817+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:40:17.829817+0000","last_undegraded":"2023-10-09T06:40:17.829817+0000","last_fullsized":"2023-10-09T06:40:17.829817+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":611,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.839100+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:40:17.839100+0000","last_peered":"2023-10-09T06:40:17.839100+0000","last_clean":"2023-10-09T06:40:17.839100+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:40:17.839100+0000","last_undegraded":"2023-10-09T06:40:17.839100+0000","last_fullsized":"2023-10-09T06:40:17.839100+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":575,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843696+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:40:17.843696+0000","last_peered":"2023-10-09T06:40:17.843696+0000","last_clean":"2023-10-09T06:40:17.843696+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:40:17.843696+0000","last_undegraded":"2023-10-09T06:40:17.843696+0000","last_fullsized":"2023-10-09T06:40:17.843696+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834706+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:40:17.834706+0000","last_peered":"2023-10-09T06:40:17.834706+0000","last_clean":"2023-10-09T06:40:17.834706+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:40:17.834706+0000","last_undegraded":"2023-10-09T06:40:17.834706+0000","last_fullsized":"2023-10-09T06:40:17.834706+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"497'3785","reported_seq":3801,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.674680+0000","last_change":"2023-10-09T06:39:33.925861+0000","last_active":"2023-10-09T06:40:27.674680+0000","last_peered":"2023-10-09T06:40:27.674680+0000","last_clean":"2023-10-09T06:40:27.674680+0000","last_became_active":"2023-10-09T06:39:33.925623+0000","last_became_peered":"2023-10-09T06:39:33.925623+0000","last_unstale":"2023-10-09T06:40:27.674680+0000","last_undegraded":"2023-10-09T06:40:27.674680+0000","last_fullsized":"2023-10-09T06:40:27.674680+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3785,"log_dups_size":0,"ondisk_log_size":3785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:10.504728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248053760,"num_objects":3785,"num_object_clones":0,"num_object_copies":11355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3785,"num_write_kb":242240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":624,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837674+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:40:17.837674+0000","last_peered":"2023-10-09T06:40:17.837674+0000","last_clean":"2023-10-09T06:40:17.837674+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:40:17.837674+0000","last_undegraded":"2023-10-09T06:40:17.837674+0000","last_fullsized":"2023-10-09T06:40:17.837674+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":382,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.838184+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:40:17.838184+0000","last_peered":"2023-10-09T06:40:17.838184+0000","last_clean":"2023-10-09T06:40:17.838184+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:40:17.838184+0000","last_undegraded":"2023-10-09T06:40:17.838184+0000","last_fullsized":"2023-10-09T06:40:17.838184+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":280,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836814+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:40:17.836814+0000","last_peered":"2023-10-09T06:40:17.836814+0000","last_clean":"2023-10-09T06:40:17.836814+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:40:17.836814+0000","last_undegraded":"2023-10-09T06:40:17.836814+0000","last_fullsized":"2023-10-09T06:40:17.836814+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"497'3898","reported_seq":3915,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.496995+0000","last_change":"2023-10-09T06:39:33.923263+0000","last_active":"2023-10-09T06:40:29.496995+0000","last_peered":"2023-10-09T06:40:29.496995+0000","last_clean":"2023-10-09T06:40:29.496995+0000","last_became_active":"2023-10-09T06:39:33.922687+0000","last_became_peered":"2023-10-09T06:39:33.922687+0000","last_unstale":"2023-10-09T06:40:29.496995+0000","last_undegraded":"2023-10-09T06:40:29.496995+0000","last_fullsized":"2023-10-09T06:40:29.496995+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3898,"log_dups_size":0,"ondisk_log_size":3898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:05:59.579546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255459328,"num_objects":3898,"num_object_clones":0,"num_object_copies":11694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3898,"num_write_kb":249472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":883,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834698+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:40:17.834698+0000","last_peered":"2023-10-09T06:40:17.834698+0000","last_clean":"2023-10-09T06:40:17.834698+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:40:17.834698+0000","last_undegraded":"2023-10-09T06:40:17.834698+0000","last_fullsized":"2023-10-09T06:40:17.834698+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":344,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.838081+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:40:17.838081+0000","last_peered":"2023-10-09T06:40:17.838081+0000","last_clean":"2023-10-09T06:40:17.838081+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:40:17.838081+0000","last_undegraded":"2023-10-09T06:40:17.838081+0000","last_fullsized":"2023-10-09T06:40:17.838081+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":494,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823669+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:40:17.823669+0000","last_peered":"2023-10-09T06:40:17.823669+0000","last_clean":"2023-10-09T06:40:17.823669+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:40:17.823669+0000","last_undegraded":"2023-10-09T06:40:17.823669+0000","last_fullsized":"2023-10-09T06:40:17.823669+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"497'3886","reported_seq":3903,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.498702+0000","last_change":"2023-10-09T06:39:33.927681+0000","last_active":"2023-10-09T06:40:29.498702+0000","last_peered":"2023-10-09T06:40:29.498702+0000","last_clean":"2023-10-09T06:40:29.498702+0000","last_became_active":"2023-10-09T06:39:33.927408+0000","last_became_peered":"2023-10-09T06:39:33.927408+0000","last_unstale":"2023-10-09T06:40:29.498702+0000","last_undegraded":"2023-10-09T06:40:29.498702+0000","last_fullsized":"2023-10-09T06:40:29.498702+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3886,"log_dups_size":0,"ondisk_log_size":3886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:54:59.401749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254672896,"num_objects":3886,"num_object_clones":0,"num_object_copies":11658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3886,"num_write_kb":248704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":798,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844226+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:40:17.844226+0000","last_peered":"2023-10-09T06:40:17.844226+0000","last_clean":"2023-10-09T06:40:17.844226+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:40:17.844226+0000","last_undegraded":"2023-10-09T06:40:17.844226+0000","last_fullsized":"2023-10-09T06:40:17.844226+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":400,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837785+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:40:17.837785+0000","last_peered":"2023-10-09T06:40:17.837785+0000","last_clean":"2023-10-09T06:40:17.837785+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:40:17.837785+0000","last_undegraded":"2023-10-09T06:40:17.837785+0000","last_fullsized":"2023-10-09T06:40:17.837785+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":473,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843990+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:40:17.843990+0000","last_peered":"2023-10-09T06:40:17.843990+0000","last_clean":"2023-10-09T06:40:17.843990+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:40:17.843990+0000","last_undegraded":"2023-10-09T06:40:17.843990+0000","last_fullsized":"2023-10-09T06:40:17.843990+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"497'3827","reported_seq":3844,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:28.992703+0000","last_change":"2023-10-09T06:39:33.924837+0000","last_active":"2023-10-09T06:40:28.992703+0000","last_peered":"2023-10-09T06:40:28.992703+0000","last_clean":"2023-10-09T06:40:28.992703+0000","last_became_active":"2023-10-09T06:39:33.924498+0000","last_became_peered":"2023-10-09T06:39:33.924498+0000","last_unstale":"2023-10-09T06:40:28.992703+0000","last_undegraded":"2023-10-09T06:40:28.992703+0000","last_fullsized":"2023-10-09T06:40:28.992703+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3827,"log_dups_size":0,"ondisk_log_size":3827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:35.001011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250806272,"num_objects":3827,"num_object_clones":0,"num_object_copies":11481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3827,"num_write_kb":244928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":621,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823981+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:40:17.823981+0000","last_peered":"2023-10-09T06:40:17.823981+0000","last_clean":"2023-10-09T06:40:17.823981+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:40:17.823981+0000","last_undegraded":"2023-10-09T06:40:17.823981+0000","last_fullsized":"2023-10-09T06:40:17.823981+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":67,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834993+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:40:17.834993+0000","last_peered":"2023-10-09T06:40:17.834993+0000","last_clean":"2023-10-09T06:40:17.834993+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:40:17.834993+0000","last_undegraded":"2023-10-09T06:40:17.834993+0000","last_fullsized":"2023-10-09T06:40:17.834993+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":460,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834822+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:40:17.834822+0000","last_peered":"2023-10-09T06:40:17.834822+0000","last_clean":"2023-10-09T06:40:17.834822+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:40:17.834822+0000","last_undegraded":"2023-10-09T06:40:17.834822+0000","last_fullsized":"2023-10-09T06:40:17.834822+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"497'3817","reported_seq":3833,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.503713+0000","last_change":"2023-10-09T06:39:33.922928+0000","last_active":"2023-10-09T06:40:29.503713+0000","last_peered":"2023-10-09T06:40:29.503713+0000","last_clean":"2023-10-09T06:40:29.503713+0000","last_became_active":"2023-10-09T06:39:33.922653+0000","last_became_peered":"2023-10-09T06:39:33.922653+0000","last_unstale":"2023-10-09T06:40:29.503713+0000","last_undegraded":"2023-10-09T06:40:29.503713+0000","last_fullsized":"2023-10-09T06:40:29.503713+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3817,"log_dups_size":0,"ondisk_log_size":3817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:09:31.453742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250150912,"num_objects":3817,"num_object_clones":0,"num_object_copies":11451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3817,"num_write_kb":244288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":849,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836681+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:40:17.836681+0000","last_peered":"2023-10-09T06:40:17.836681+0000","last_clean":"2023-10-09T06:40:17.836681+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:40:17.836681+0000","last_undegraded":"2023-10-09T06:40:17.836681+0000","last_fullsized":"2023-10-09T06:40:17.836681+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":225,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.835432+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:40:17.835432+0000","last_peered":"2023-10-09T06:40:17.835432+0000","last_clean":"2023-10-09T06:40:17.835432+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:40:17.835432+0000","last_undegraded":"2023-10-09T06:40:17.835432+0000","last_fullsized":"2023-10-09T06:40:17.835432+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":477,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823883+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:40:17.823883+0000","last_peered":"2023-10-09T06:40:17.823883+0000","last_clean":"2023-10-09T06:40:17.823883+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:40:17.823883+0000","last_undegraded":"2023-10-09T06:40:17.823883+0000","last_fullsized":"2023-10-09T06:40:17.823883+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"497'3764","reported_seq":3780,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.632763+0000","last_change":"2023-10-09T06:39:33.929339+0000","last_active":"2023-10-09T06:40:27.632763+0000","last_peered":"2023-10-09T06:40:27.632763+0000","last_clean":"2023-10-09T06:40:27.632763+0000","last_became_active":"2023-10-09T06:39:33.929105+0000","last_became_peered":"2023-10-09T06:39:33.929105+0000","last_unstale":"2023-10-09T06:40:27.632763+0000","last_undegraded":"2023-10-09T06:40:27.632763+0000","last_fullsized":"2023-10-09T06:40:27.632763+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3764,"log_dups_size":0,"ondisk_log_size":3764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:24.427366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677504,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3764,"num_write_kb":240896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":531,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833248+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:40:17.833248+0000","last_peered":"2023-10-09T06:40:17.833248+0000","last_clean":"2023-10-09T06:40:17.833248+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:40:17.833248+0000","last_undegraded":"2023-10-09T06:40:17.833248+0000","last_fullsized":"2023-10-09T06:40:17.833248+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":454,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843897+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:40:17.843897+0000","last_peered":"2023-10-09T06:40:17.843897+0000","last_clean":"2023-10-09T06:40:17.843897+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:40:17.843897+0000","last_undegraded":"2023-10-09T06:40:17.843897+0000","last_fullsized":"2023-10-09T06:40:17.843897+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":474,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824093+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:40:17.824093+0000","last_peered":"2023-10-09T06:40:17.824093+0000","last_clean":"2023-10-09T06:40:17.824093+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:40:17.824093+0000","last_undegraded":"2023-10-09T06:40:17.824093+0000","last_fullsized":"2023-10-09T06:40:17.824093+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"497'3710","reported_seq":3726,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.619939+0000","last_change":"2023-10-09T06:39:33.924988+0000","last_active":"2023-10-09T06:40:27.619939+0000","last_peered":"2023-10-09T06:40:27.619939+0000","last_clean":"2023-10-09T06:40:27.619939+0000","last_became_active":"2023-10-09T06:39:33.924698+0000","last_became_peered":"2023-10-09T06:39:33.924698+0000","last_unstale":"2023-10-09T06:40:27.619939+0000","last_undegraded":"2023-10-09T06:40:27.619939+0000","last_fullsized":"2023-10-09T06:40:27.619939+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3710,"log_dups_size":0,"ondisk_log_size":3710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:03.806961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243138560,"num_objects":3710,"num_object_clones":0,"num_object_copies":11130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3710,"num_write_kb":237440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":833,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843040+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:40:17.843040+0000","last_peered":"2023-10-09T06:40:17.843040+0000","last_clean":"2023-10-09T06:40:17.843040+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:40:17.843040+0000","last_undegraded":"2023-10-09T06:40:17.843040+0000","last_fullsized":"2023-10-09T06:40:17.843040+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":478,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823567+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:40:17.823567+0000","last_peered":"2023-10-09T06:40:17.823567+0000","last_clean":"2023-10-09T06:40:17.823567+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:40:17.823567+0000","last_undegraded":"2023-10-09T06:40:17.823567+0000","last_fullsized":"2023-10-09T06:40:17.823567+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":457,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824079+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:40:17.824079+0000","last_peered":"2023-10-09T06:40:17.824079+0000","last_clean":"2023-10-09T06:40:17.824079+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:40:17.824079+0000","last_undegraded":"2023-10-09T06:40:17.824079+0000","last_fullsized":"2023-10-09T06:40:17.824079+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"497'3793","reported_seq":3809,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.507006+0000","last_change":"2023-10-09T06:39:33.922114+0000","last_active":"2023-10-09T06:40:29.507006+0000","last_peered":"2023-10-09T06:40:29.507006+0000","last_clean":"2023-10-09T06:40:29.507006+0000","last_became_active":"2023-10-09T06:39:33.921838+0000","last_became_peered":"2023-10-09T06:39:33.921838+0000","last_unstale":"2023-10-09T06:40:29.507006+0000","last_undegraded":"2023-10-09T06:40:29.507006+0000","last_fullsized":"2023-10-09T06:40:29.507006+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3793,"log_dups_size":0,"ondisk_log_size":3793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:09:43.308955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248578048,"num_objects":3793,"num_object_clones":0,"num_object_copies":11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3793,"num_write_kb":242752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":459,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843868+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:40:17.843868+0000","last_peered":"2023-10-09T06:40:17.843868+0000","last_clean":"2023-10-09T06:40:17.843868+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:40:17.843868+0000","last_undegraded":"2023-10-09T06:40:17.843868+0000","last_fullsized":"2023-10-09T06:40:17.843868+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":199,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833088+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:40:17.833088+0000","last_peered":"2023-10-09T06:40:17.833088+0000","last_clean":"2023-10-09T06:40:17.833088+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:40:17.833088+0000","last_undegraded":"2023-10-09T06:40:17.833088+0000","last_fullsized":"2023-10-09T06:40:17.833088+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":453,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844175+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:40:17.844175+0000","last_peered":"2023-10-09T06:40:17.844175+0000","last_clean":"2023-10-09T06:40:17.844175+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:40:17.844175+0000","last_undegraded":"2023-10-09T06:40:17.844175+0000","last_fullsized":"2023-10-09T06:40:17.844175+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"497'4003","reported_seq":4020,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.456252+0000","last_change":"2023-10-09T06:39:33.922898+0000","last_active":"2023-10-09T06:40:29.456252+0000","last_peered":"2023-10-09T06:40:29.456252+0000","last_clean":"2023-10-09T06:40:29.456252+0000","last_became_active":"2023-10-09T06:39:33.922619+0000","last_became_peered":"2023-10-09T06:39:33.922619+0000","last_unstale":"2023-10-09T06:40:29.456252+0000","last_undegraded":"2023-10-09T06:40:29.456252+0000","last_fullsized":"2023-10-09T06:40:29.456252+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":4003,"log_dups_size":0,"ondisk_log_size":4003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:22.252997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262340608,"num_objects":4003,"num_object_clones":0,"num_object_copies":12009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4003,"num_write_kb":256192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":789,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844318+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:40:17.844318+0000","last_peered":"2023-10-09T06:40:17.844318+0000","last_clean":"2023-10-09T06:40:17.844318+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:40:17.844318+0000","last_undegraded":"2023-10-09T06:40:17.844318+0000","last_fullsized":"2023-10-09T06:40:17.844318+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":505,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842921+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:40:17.842921+0000","last_peered":"2023-10-09T06:40:17.842921+0000","last_clean":"2023-10-09T06:40:17.842921+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:40:17.842921+0000","last_undegraded":"2023-10-09T06:40:17.842921+0000","last_fullsized":"2023-10-09T06:40:17.842921+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":67,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833906+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:40:17.833906+0000","last_peered":"2023-10-09T06:40:17.833906+0000","last_clean":"2023-10-09T06:40:17.833906+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:40:17.833906+0000","last_undegraded":"2023-10-09T06:40:17.833906+0000","last_fullsized":"2023-10-09T06:40:17.833906+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":439,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.847291+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:40:17.847291+0000","last_peered":"2023-10-09T06:40:17.847291+0000","last_clean":"2023-10-09T06:40:17.847291+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:40:17.847291+0000","last_undegraded":"2023-10-09T06:40:17.847291+0000","last_fullsized":"2023-10-09T06:40:17.847291+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"497'3721","reported_seq":3737,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.653498+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:40:27.653498+0000","last_peered":"2023-10-09T06:40:27.653498+0000","last_clean":"2023-10-09T06:40:27.653498+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:40:27.653498+0000","last_undegraded":"2023-10-09T06:40:27.653498+0000","last_fullsized":"2023-10-09T06:40:27.653498+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3721,"log_dups_size":0,"ondisk_log_size":3721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243859456,"num_objects":3721,"num_object_clones":0,"num_object_copies":11163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3721,"num_write_kb":238144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":676,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.829635+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:40:17.829635+0000","last_peered":"2023-10-09T06:40:17.829635+0000","last_clean":"2023-10-09T06:40:17.829635+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:40:17.829635+0000","last_undegraded":"2023-10-09T06:40:17.829635+0000","last_fullsized":"2023-10-09T06:40:17.829635+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":228,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.847403+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:40:17.847403+0000","last_peered":"2023-10-09T06:40:17.847403+0000","last_clean":"2023-10-09T06:40:17.847403+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:40:17.847403+0000","last_undegraded":"2023-10-09T06:40:17.847403+0000","last_fullsized":"2023-10-09T06:40:17.847403+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":478,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824077+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:40:17.824077+0000","last_peered":"2023-10-09T06:40:17.824077+0000","last_clean":"2023-10-09T06:40:17.824077+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:40:17.824077+0000","last_undegraded":"2023-10-09T06:40:17.824077+0000","last_fullsized":"2023-10-09T06:40:17.824077+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":469,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842856+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:40:17.842856+0000","last_peered":"2023-10-09T06:40:17.842856+0000","last_clean":"2023-10-09T06:40:17.842856+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:40:17.842856+0000","last_undegraded":"2023-10-09T06:40:17.842856+0000","last_fullsized":"2023-10-09T06:40:17.842856+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"497'3729","reported_seq":3745,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.658017+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:40:27.658017+0000","last_peered":"2023-10-09T06:40:27.658017+0000","last_clean":"2023-10-09T06:40:27.658017+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:40:27.658017+0000","last_undegraded":"2023-10-09T06:40:27.658017+0000","last_fullsized":"2023-10-09T06:40:27.658017+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3729,"log_dups_size":0,"ondisk_log_size":3729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244383744,"num_objects":3729,"num_object_clones":0,"num_object_copies":11187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3729,"num_write_kb":238656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":839,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836955+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:40:17.836955+0000","last_peered":"2023-10-09T06:40:17.836955+0000","last_clean":"2023-10-09T06:40:17.836955+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:40:17.836955+0000","last_undegraded":"2023-10-09T06:40:17.836955+0000","last_fullsized":"2023-10-09T06:40:17.836955+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":490,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833997+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:40:17.833997+0000","last_peered":"2023-10-09T06:40:17.833997+0000","last_clean":"2023-10-09T06:40:17.833997+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:40:17.833997+0000","last_undegraded":"2023-10-09T06:40:17.833997+0000","last_fullsized":"2023-10-09T06:40:17.833997+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":477,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842547+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:40:17.842547+0000","last_peered":"2023-10-09T06:40:17.842547+0000","last_clean":"2023-10-09T06:40:17.842547+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:40:17.842547+0000","last_undegraded":"2023-10-09T06:40:17.842547+0000","last_fullsized":"2023-10-09T06:40:17.842547+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":499,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844252+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:40:17.844252+0000","last_peered":"2023-10-09T06:40:17.844252+0000","last_clean":"2023-10-09T06:40:17.844252+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:40:17.844252+0000","last_undegraded":"2023-10-09T06:40:17.844252+0000","last_fullsized":"2023-10-09T06:40:17.844252+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"497'3756","reported_seq":3772,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:27.652545+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:40:27.652545+0000","last_peered":"2023-10-09T06:40:27.652545+0000","last_clean":"2023-10-09T06:40:27.652545+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:40:27.652545+0000","last_undegraded":"2023-10-09T06:40:27.652545+0000","last_fullsized":"2023-10-09T06:40:27.652545+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3756,"log_dups_size":0,"ondisk_log_size":3756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246153216,"num_objects":3756,"num_object_clones":0,"num_object_copies":11268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3756,"num_write_kb":240384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":306,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837267+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:40:17.837267+0000","last_peered":"2023-10-09T06:40:17.837267+0000","last_clean":"2023-10-09T06:40:17.837267+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:40:17.837267+0000","last_undegraded":"2023-10-09T06:40:17.837267+0000","last_fullsized":"2023-10-09T06:40:17.837267+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":820,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836896+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:40:17.836896+0000","last_peered":"2023-10-09T06:40:17.836896+0000","last_clean":"2023-10-09T06:40:17.836896+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:40:17.836896+0000","last_undegraded":"2023-10-09T06:40:17.836896+0000","last_fullsized":"2023-10-09T06:40:17.836896+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":209,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833692+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:40:17.833692+0000","last_peered":"2023-10-09T06:40:17.833692+0000","last_clean":"2023-10-09T06:40:17.833692+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:40:17.833692+0000","last_undegraded":"2023-10-09T06:40:17.833692+0000","last_fullsized":"2023-10-09T06:40:17.833692+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":459,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844214+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:40:17.844214+0000","last_peered":"2023-10-09T06:40:17.844214+0000","last_clean":"2023-10-09T06:40:17.844214+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:40:17.844214+0000","last_undegraded":"2023-10-09T06:40:17.844214+0000","last_fullsized":"2023-10-09T06:40:17.844214+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"497'3881","reported_seq":3898,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.444330+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:40:29.444330+0000","last_peered":"2023-10-09T06:40:29.444330+0000","last_clean":"2023-10-09T06:40:29.444330+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:40:29.444330+0000","last_undegraded":"2023-10-09T06:40:29.444330+0000","last_fullsized":"2023-10-09T06:40:29.444330+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3881,"log_dups_size":0,"ondisk_log_size":3881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254345216,"num_objects":3881,"num_object_clones":0,"num_object_copies":11643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3881,"num_write_kb":248384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":808,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843126+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:40:17.843126+0000","last_peered":"2023-10-09T06:40:17.843126+0000","last_clean":"2023-10-09T06:40:17.843126+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:40:17.843126+0000","last_undegraded":"2023-10-09T06:40:17.843126+0000","last_fullsized":"2023-10-09T06:40:17.843126+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":360,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.836723+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:40:17.836723+0000","last_peered":"2023-10-09T06:40:17.836723+0000","last_clean":"2023-10-09T06:40:17.836723+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:40:17.836723+0000","last_undegraded":"2023-10-09T06:40:17.836723+0000","last_fullsized":"2023-10-09T06:40:17.836723+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":173,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.829551+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:40:17.829551+0000","last_peered":"2023-10-09T06:40:17.829551+0000","last_clean":"2023-10-09T06:40:17.829551+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:40:17.829551+0000","last_undegraded":"2023-10-09T06:40:17.829551+0000","last_fullsized":"2023-10-09T06:40:17.829551+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":483,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.830098+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:40:17.830098+0000","last_peered":"2023-10-09T06:40:17.830098+0000","last_clean":"2023-10-09T06:40:17.830098+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:40:17.830098+0000","last_undegraded":"2023-10-09T06:40:17.830098+0000","last_fullsized":"2023-10-09T06:40:17.830098+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"497'3888","reported_seq":3905,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:28.987477+0000","last_change":"2023-10-09T06:39:33.927704+0000","last_active":"2023-10-09T06:40:28.987477+0000","last_peered":"2023-10-09T06:40:28.987477+0000","last_clean":"2023-10-09T06:40:28.987477+0000","last_became_active":"2023-10-09T06:39:33.927409+0000","last_became_peered":"2023-10-09T06:39:33.927409+0000","last_unstale":"2023-10-09T06:40:28.987477+0000","last_undegraded":"2023-10-09T06:40:28.987477+0000","last_fullsized":"2023-10-09T06:40:28.987477+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3888,"log_dups_size":0,"ondisk_log_size":3888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:27:18.487828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254803968,"num_objects":3888,"num_object_clones":0,"num_object_copies":11664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3888,"num_write_kb":248832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":819,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844171+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:40:17.844171+0000","last_peered":"2023-10-09T06:40:17.844171+0000","last_clean":"2023-10-09T06:40:17.844171+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:40:17.844171+0000","last_undegraded":"2023-10-09T06:40:17.844171+0000","last_fullsized":"2023-10-09T06:40:17.844171+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":360,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837867+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:40:17.837867+0000","last_peered":"2023-10-09T06:40:17.837867+0000","last_clean":"2023-10-09T06:40:17.837867+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:40:17.837867+0000","last_undegraded":"2023-10-09T06:40:17.837867+0000","last_fullsized":"2023-10-09T06:40:17.837867+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":495,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842806+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:40:17.842806+0000","last_peered":"2023-10-09T06:40:17.842806+0000","last_clean":"2023-10-09T06:40:17.842806+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:40:17.842806+0000","last_undegraded":"2023-10-09T06:40:17.842806+0000","last_fullsized":"2023-10-09T06:40:17.842806+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834986+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:40:17.834986+0000","last_peered":"2023-10-09T06:40:17.834986+0000","last_clean":"2023-10-09T06:40:17.834986+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:40:17.834986+0000","last_undegraded":"2023-10-09T06:40:17.834986+0000","last_fullsized":"2023-10-09T06:40:17.834986+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"497'3842","reported_seq":3859,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:28.089293+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:40:28.089293+0000","last_peered":"2023-10-09T06:40:28.089293+0000","last_clean":"2023-10-09T06:40:28.089293+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:40:28.089293+0000","last_undegraded":"2023-10-09T06:40:28.089293+0000","last_fullsized":"2023-10-09T06:40:28.089293+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3842,"log_dups_size":0,"ondisk_log_size":3842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251789312,"num_objects":3842,"num_object_clones":0,"num_object_copies":11526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":245888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":820,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823980+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:40:17.823980+0000","last_peered":"2023-10-09T06:40:17.823980+0000","last_clean":"2023-10-09T06:40:17.823980+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:40:17.823980+0000","last_undegraded":"2023-10-09T06:40:17.823980+0000","last_fullsized":"2023-10-09T06:40:17.823980+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":229,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834839+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:40:17.834839+0000","last_peered":"2023-10-09T06:40:17.834839+0000","last_clean":"2023-10-09T06:40:17.834839+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:40:17.834839+0000","last_undegraded":"2023-10-09T06:40:17.834839+0000","last_fullsized":"2023-10-09T06:40:17.834839+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":462,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844019+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:40:17.844019+0000","last_peered":"2023-10-09T06:40:17.844019+0000","last_clean":"2023-10-09T06:40:17.844019+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:40:17.844019+0000","last_undegraded":"2023-10-09T06:40:17.844019+0000","last_fullsized":"2023-10-09T06:40:17.844019+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":218,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833012+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:40:17.833012+0000","last_peered":"2023-10-09T06:40:17.833012+0000","last_clean":"2023-10-09T06:40:17.833012+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:40:17.833012+0000","last_undegraded":"2023-10-09T06:40:17.833012+0000","last_fullsized":"2023-10-09T06:40:17.833012+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"497'3916","reported_seq":3933,"reported_epoch":497,"state":"active+clean","last_fresh":"2023-10-09T06:40:29.436179+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:40:29.436179+0000","last_peered":"2023-10-09T06:40:29.436179+0000","last_clean":"2023-10-09T06:40:29.436179+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:40:29.436179+0000","last_undegraded":"2023-10-09T06:40:29.436179+0000","last_fullsized":"2023-10-09T06:40:29.436179+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":3916,"log_dups_size":0,"ondisk_log_size":3916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256638976,"num_objects":3916,"num_object_clones":0,"num_object_copies":11748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3916,"num_write_kb":250624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":852,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.835072+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:40:17.835072+0000","last_peered":"2023-10-09T06:40:17.835072+0000","last_clean":"2023-10-09T06:40:17.835072+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:40:17.835072+0000","last_undegraded":"2023-10-09T06:40:17.835072+0000","last_fullsized":"2023-10-09T06:40:17.835072+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":481,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.844394+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:40:17.844394+0000","last_peered":"2023-10-09T06:40:17.844394+0000","last_clean":"2023-10-09T06:40:17.844394+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:40:17.844394+0000","last_undegraded":"2023-10-09T06:40:17.844394+0000","last_fullsized":"2023-10-09T06:40:17.844394+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":464,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.829948+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:40:17.829948+0000","last_peered":"2023-10-09T06:40:17.829948+0000","last_clean":"2023-10-09T06:40:17.829948+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:40:17.829948+0000","last_undegraded":"2023-10-09T06:40:17.829948+0000","last_fullsized":"2023-10-09T06:40:17.829948+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":460,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823656+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:40:17.823656+0000","last_peered":"2023-10-09T06:40:17.823656+0000","last_clean":"2023-10-09T06:40:17.823656+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:40:17.823656+0000","last_undegraded":"2023-10-09T06:40:17.823656+0000","last_fullsized":"2023-10-09T06:40:17.823656+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":778,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.833824+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:40:17.833824+0000","last_peered":"2023-10-09T06:40:17.833824+0000","last_clean":"2023-10-09T06:40:17.833824+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:40:17.833824+0000","last_undegraded":"2023-10-09T06:40:17.833824+0000","last_fullsized":"2023-10-09T06:40:17.833824+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":465,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834936+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:40:17.834936+0000","last_peered":"2023-10-09T06:40:17.834936+0000","last_clean":"2023-10-09T06:40:17.834936+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:40:17.834936+0000","last_undegraded":"2023-10-09T06:40:17.834936+0000","last_fullsized":"2023-10-09T06:40:17.834936+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":487,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.842762+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:40:17.842762+0000","last_peered":"2023-10-09T06:40:17.842762+0000","last_clean":"2023-10-09T06:40:17.842762+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:40:17.842762+0000","last_undegraded":"2023-10-09T06:40:17.842762+0000","last_fullsized":"2023-10-09T06:40:17.842762+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":709,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:18.910859+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:40:18.910859+0000","last_peered":"2023-10-09T06:40:18.910859+0000","last_clean":"2023-10-09T06:40:18.910859+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:40:18.910859+0000","last_undegraded":"2023-10-09T06:40:18.910859+0000","last_fullsized":"2023-10-09T06:40:18.910859+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":175,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823895+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:40:17.823895+0000","last_peered":"2023-10-09T06:40:17.823895+0000","last_clean":"2023-10-09T06:40:17.823895+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:40:17.823895+0000","last_undegraded":"2023-10-09T06:40:17.823895+0000","last_fullsized":"2023-10-09T06:40:17.823895+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":504,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823889+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:40:17.823889+0000","last_peered":"2023-10-09T06:40:17.823889+0000","last_clean":"2023-10-09T06:40:17.823889+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:40:17.823889+0000","last_undegraded":"2023-10-09T06:40:17.823889+0000","last_fullsized":"2023-10-09T06:40:17.823889+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":175,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.824157+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:40:17.824157+0000","last_peered":"2023-10-09T06:40:17.824157+0000","last_clean":"2023-10-09T06:40:17.824157+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:40:17.824157+0000","last_undegraded":"2023-10-09T06:40:17.824157+0000","last_fullsized":"2023-10-09T06:40:17.824157+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":514,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823666+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:40:17.823666+0000","last_peered":"2023-10-09T06:40:17.823666+0000","last_clean":"2023-10-09T06:40:17.823666+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:40:17.823666+0000","last_undegraded":"2023-10-09T06:40:17.823666+0000","last_fullsized":"2023-10-09T06:40:17.823666+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":853,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.837164+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:40:17.837164+0000","last_peered":"2023-10-09T06:40:17.837164+0000","last_clean":"2023-10-09T06:40:17.837164+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:40:17.837164+0000","last_undegraded":"2023-10-09T06:40:17.837164+0000","last_fullsized":"2023-10-09T06:40:17.837164+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":200,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834273+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:40:17.834273+0000","last_peered":"2023-10-09T06:40:17.834273+0000","last_clean":"2023-10-09T06:40:17.834273+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:40:17.834273+0000","last_undegraded":"2023-10-09T06:40:17.834273+0000","last_fullsized":"2023-10-09T06:40:17.834273+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":217,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.834166+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:40:17.834166+0000","last_peered":"2023-10-09T06:40:17.834166+0000","last_clean":"2023-10-09T06:40:17.834166+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:40:17.834166+0000","last_undegraded":"2023-10-09T06:40:17.834166+0000","last_fullsized":"2023-10-09T06:40:17.834166+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":620,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.843135+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:40:17.843135+0000","last_peered":"2023-10-09T06:40:17.843135+0000","last_clean":"2023-10-09T06:40:17.843135+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:40:17.843135+0000","last_undegraded":"2023-10-09T06:40:17.843135+0000","last_fullsized":"2023-10-09T06:40:17.843135+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":495,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.832848+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:40:17.832848+0000","last_peered":"2023-10-09T06:40:17.832848+0000","last_clean":"2023-10-09T06:40:17.832848+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:40:17.832848+0000","last_undegraded":"2023-10-09T06:40:17.832848+0000","last_fullsized":"2023-10-09T06:40:17.832848+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":457,"reported_epoch":496,"state":"active+clean","last_fresh":"2023-10-09T06:40:17.823485+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:40:17.823485+0000","last_peered":"2023-10-09T06:40:17.823485+0000","last_clean":"2023-10-09T06:40:17.823485+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:40:17.823485+0000","last_undegraded":"2023-10-09T06:40:17.823485+0000","last_fullsized":"2023-10-09T06:40:17.823485+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":4011851776,"num_objects":61216,"num_object_clones":0,"num_object_copies":183648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61216,"num_write_kb":3917824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11925979136,"data_stored":11925979136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61216,"ondisk_log_size":61216,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234518,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2205660,"kb_used_data":712852,"kb_used_omap":0,"kb_used_meta":1492800,"kb_avail":91539492,"statfs":{"total":95995035648,"available":93736439808,"internally_reserved":0,"allocated":729960448,"data_stored":728894877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1528627200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:40:14 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.4239999999999999,"15min":4.0739999999999998},"min":{"1min":0.246,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":4.6219999999999999,"5min":8.8819999999999997,"15min":215.97800000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.4430000000000001,"15min":4.1239999999999997},"min":{"1min":0.32700000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":4.6390000000000002,"5min":10.449999999999999,"15min":215.577},"last":0.37}]},{"osd":1,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":6.7649999999999997,"15min":5.3440000000000003},"min":{"1min":0.26400000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":3.6459999999999999,"5min":208.626,"15min":215.61699999999999},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":4.9900000000000002,"15min":4.4630000000000001},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.23300000000000001},"max":{"1min":4.923,"5min":181.24199999999999,"15min":216.22200000000001},"last":1.482}]},{"osd":2,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":8.5220000000000002,"15min":6.4269999999999996},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.276,"5min":228.24000000000001,"15min":228.24000000000001},"last":4.5449999999999999},{"interface":"front","average":{"1min":1.008,"5min":8.4990000000000006,"15min":6.4619999999999997},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.28000000000000003},"max":{"1min":3.2749999999999999,"5min":228.291,"15min":228.291},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":5.835,"15min":5.5499999999999998},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":25.295000000000002,"5min":175.40600000000001,"15min":215.922},"last":6.8959999999999999},{"interface":"front","average":{"1min":2.5750000000000002,"5min":5.7489999999999997,"15min":5.5540000000000003},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":25.324999999999999,"5min":175.30099999999999,"15min":215.94900000000001},"last":1.3}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.606,"5min":4.6929999999999996,"15min":3.5009999999999999},"min":{"1min":0.26900000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":225.041,"5min":225.041,"15min":225.041},"last":0.35599999999999998},{"interface":"front","average":{"1min":11.720000000000001,"5min":4.3959999999999999,"15min":3.343},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":225.244,"5min":225.244,"15min":225.244},"last":0.30299999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":11.055999999999999,"5min":5.2699999999999996,"15min":3.4220000000000002},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":219.923,"5min":219.923,"15min":219.923},"last":0.28199999999999997},{"interface":"front","average":{"1min":10.946999999999999,"5min":5.0259999999999998,"15min":3.3889999999999998},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.28999999999999998},"max":{"1min":219.86000000000001,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":10.978999999999999,"5min":5.3319999999999999,"15min":3.552},"min":{"1min":0.39700000000000002,"5min":0.32300000000000001,"15min":0.28000000000000003},"max":{"1min":222.179,"5min":222.179,"15min":222.179},"last":0.39800000000000002},{"interface":"front","average":{"1min":11.074999999999999,"5min":5.2140000000000004,"15min":3.3300000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":223.261,"5min":223.261,"15min":223.261},"last":0.33800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496154,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2139204,"kb_used_data":951804,"kb_used_omap":0,"kb_used_meta":1187392,"kb_avail":91605948,"statfs":{"total":95995035648,"available":93804490752,"internally_reserved":0,"allocated":974647296,"data_stored":973605023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1215889408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":2.2170000000000001,"15min":2.5009999999999999},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":5.9569999999999999,"5min":66.867000000000004,"15min":75.061999999999998},"last":1.621},{"interface":"front","average":{"1min":13.164,"5min":4.8049999999999997,"15min":3.4740000000000002},"min":{"1min":0.33800000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":212.096,"5min":212.096,"15min":212.096},"last":1.635}]},{"osd":1,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":4.4649999999999999,"15min":4.9279999999999999},"min":{"1min":0.31,"5min":0.26500000000000001,"15min":0.20899999999999999},"max":{"1min":3.8740000000000001,"5min":219.93199999999999,"15min":292.92200000000003},"last":1.571},{"interface":"front","average":{"1min":0.75700000000000001,"5min":4.452,"15min":4.9800000000000004},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":2.9630000000000001,"5min":219.94800000000001,"15min":292.904},"last":1.554}]},{"osd":2,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":4.4560000000000004,"15min":3.504},"min":{"1min":0.35299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.8570000000000002,"5min":185.654,"15min":185.654},"last":2.286},{"interface":"front","average":{"1min":0.79300000000000004,"5min":4.3019999999999996,"15min":3.3279999999999998},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":2.2850000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":2.3029999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:39:28 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":3.5939999999999999,"15min":3.7570000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":8.5380000000000003,"5min":121.749,"15min":121.749},"last":1.8169999999999999},{"interface":"front","average":{"1min":1.7609999999999999,"5min":3.5049999999999999,"15min":3.6829999999999998},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.311},"max":{"1min":8.4770000000000003,"5min":120.70399999999999,"15min":120.70399999999999},"last":1.9059999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.5680000000000001,"15min":1.5149999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":3.8140000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.318},{"interface":"front","average":{"1min":10.757,"5min":3.8490000000000002,"15min":2.1560000000000001},"min":{"1min":0.32700000000000001,"5min":0.253,"15min":0.223},"max":{"1min":212.03399999999999,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.36099999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:40:07 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.6519999999999999,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.29499999999999998},"max":{"1min":3.835,"5min":9.984,"15min":12.747999999999999},"last":0.378},{"interface":"front","average":{"1min":0.63500000000000001,"5min":2.0750000000000002,"15min":1.5349999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.242,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":3.0640000000000001,"15min":2.7280000000000002},"min":{"1min":0.34399999999999997,"5min":0.307,"15min":0.307},"max":{"1min":8.5050000000000008,"5min":165.37,"15min":165.37},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":3.5030000000000001,"15min":3.0150000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.589,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.52100000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402891,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2944892,"kb_used_data":1943092,"kb_used_omap":4,"kb_used_meta":1001787,"kb_avail":90800260,"statfs":{"total":95995035648,"available":92979466240,"internally_reserved":0,"allocated":1989726208,"data_stored":1987580683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1025830798},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":2.117,"15min":4.6619999999999999},"min":{"1min":0.20100000000000001,"5min":0.182,"15min":0.182},"max":{"1min":5.5190000000000001,"5min":110.479,"15min":269.79199999999997},"last":9.2400000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.5739999999999998,"15min":5.0199999999999996},"min":{"1min":0.22900000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.1989999999999998,"5min":110.59999999999999,"15min":271.20999999999998},"last":1.121}]},{"osd":2,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.042,"15min":2.2109999999999999},"min":{"1min":0.25800000000000001,"5min":0.22500000000000001,"15min":0.184},"max":{"1min":2.2170000000000001,"5min":10.284000000000001,"15min":138.452},"last":5.0220000000000002},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.6910000000000001,"15min":2.8599999999999999},"min":{"1min":0.26200000000000001,"5min":0.22,"15min":0.20599999999999999},"max":{"1min":6.234,"5min":10.379,"15min":138.47499999999999},"last":0.85599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:39:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.6839999999999999,"15min":2.6779999999999999},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":3.1139999999999999,"5min":12.329000000000001,"15min":31.062000000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.266,"15min":1.76},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":1.972,"5min":10.228999999999999,"15min":27.657},"last":7.202}]},{"osd":4,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":3.5470000000000002,"15min":2.6499999999999999},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.940000000000001,"5min":120.309,"15min":120.309},"last":0.94999999999999996},{"interface":"front","average":{"1min":3.0099999999999998,"5min":3.6259999999999999,"15min":2.4420000000000002},"min":{"1min":0.41099999999999998,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":29.018000000000001,"5min":120.297,"15min":120.297},"last":0.83399999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.6019999999999999,"15min":2.4910000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":4.7050000000000001,"5min":119.408,"15min":119.408},"last":5.0709999999999997},{"interface":"front","average":{"1min":1.3640000000000001,"5min":4.1130000000000004,"15min":2.8220000000000001},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":6.1379999999999999,"5min":119.483,"15min":119.483},"last":0.78600000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8900000000000001,"5min":3.2200000000000002,"15min":2.96},"min":{"1min":0.36599999999999999,"5min":0.317,"15min":0.317},"max":{"1min":25.771000000000001,"5min":121.60899999999999,"15min":121.60899999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":3.0960000000000001,"5min":3.222,"15min":3.262},"min":{"1min":0.40300000000000002,"5min":0.39900000000000002,"15min":0.316},"max":{"1min":28.588999999999999,"5min":121.559,"15min":121.559},"last":0.91900000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":4.0300000000000002,"15min":3.7519999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":3.1389999999999998,"5min":123.20699999999999,"15min":123.20699999999999},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":3.544,"15min":3.153},"min":{"1min":0.372,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":2.177,"5min":123.48999999999999,"15min":123.48999999999999},"last":5.1040000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024325,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3041672,"kb_used_data":1965504,"kb_used_omap":5,"kb_used_meta":1076154,"kb_avail":90703480,"statfs":{"total":95995035648,"available":92880363520,"internally_reserved":0,"allocated":2012676096,"data_stored":2010173072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1101982330},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:39:48 2023","interfaces":[{"interface":"back","average":{"1min":6.6980000000000004,"5min":4.5549999999999997,"15min":3.387},"min":{"1min":0.20699999999999999,"5min":0.184,"15min":0.183},"max":{"1min":102.182,"5min":102.182,"15min":191.446},"last":0.17899999999999999},{"interface":"front","average":{"1min":6.5830000000000002,"5min":4.9539999999999997,"15min":3.7930000000000001},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.187},"max":{"1min":101.282,"5min":101.282,"15min":191.393},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":5.524,"15min":3.7389999999999999},"min":{"1min":0.248,"5min":0.215,"15min":0.19900000000000001},"max":{"1min":2.3769999999999998,"5min":100.425,"15min":100.425},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":5.4660000000000002,"15min":4.1189999999999998},"min":{"1min":0.27300000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.3599999999999999,"5min":99.769999999999996,"15min":99.769999999999996},"last":3.9729999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":3.1680000000000001,"15min":4.968},"min":{"1min":0.317,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":10.454000000000001,"5min":95.555000000000007,"15min":208.11199999999999},"last":4.5259999999999998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.2330000000000001,"15min":4.8460000000000001},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.5339999999999998,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.46400000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":5.7910000000000004,"15min":2.9729999999999999},"min":{"1min":0.40899999999999997,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":3.5230000000000001,"5min":200.654,"15min":200.654},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.89100000000000001,"5min":6.2880000000000003,"15min":3.431},"min":{"1min":0.44700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.0190000000000001,"5min":200.86600000000001,"15min":200.86600000000001},"last":0.44}]},{"osd":5,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":2.5350000000000001,"15min":1.9950000000000001},"min":{"1min":0.41399999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.7810000000000001,"5min":80.885999999999996,"15min":80.885999999999996},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.036,"5min":2.2919999999999998,"15min":2.1880000000000002},"min":{"1min":0.41099999999999998,"5min":0.32900000000000001,"15min":0.31900000000000001},"max":{"1min":4.7089999999999996,"5min":80.438999999999993,"15min":80.438999999999993},"last":4.016}]},{"osd":6,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":2.9249999999999998,"15min":3.23},"min":{"1min":0.38200000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7400000000000002,"5min":118.056,"15min":120.572},"last":4.274},{"interface":"front","average":{"1min":1.429,"5min":2.8639999999999999,"15min":2.8769999999999998},"min":{"1min":0.496,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2610000000000001,"5min":118.274,"15min":120.64400000000001},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.671,"15min":2.452},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.0350000000000001,"5min":16.911000000000001,"15min":122.018},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.3109999999999999,"5min":2.097,"15min":2.8330000000000002},"min":{"1min":0.434,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.7649999999999997,"5min":9.9190000000000005,"15min":122.32299999999999},"last":8.0069999999999997}]}]},{"osd":2,"up_from":128,"seq":549755814193,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2591664,"kb_used_data":1640808,"kb_used_omap":7,"kb_used_meta":950840,"kb_avail":91153488,"statfs":{"total":95995035648,"available":93341171712,"internally_reserved":0,"allocated":1680187392,"data_stored":1678102979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":973660241},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":11.968999999999999,"5min":6.7930000000000001,"15min":6.5099999999999998},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.19400000000000001},"max":{"1min":214.595,"5min":214.595,"15min":214.595},"last":11.007},{"interface":"front","average":{"1min":11.67,"5min":6.335,"15min":5.7939999999999996},"min":{"1min":0.23899999999999999,"5min":0.21099999999999999,"15min":0.20200000000000001},"max":{"1min":214.94300000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":13.875}]},{"osd":1,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":3.2970000000000002,"15min":4.3250000000000002},"min":{"1min":0.20499999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6750000000000007,"5min":80.141000000000005,"15min":140.452},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.1259999999999999,"5min":3.4409999999999998,"15min":4.5010000000000003},"min":{"1min":0.26000000000000001,"5min":0.224,"15min":0.183},"max":{"1min":4.7859999999999996,"5min":79.474000000000004,"15min":139.75299999999999},"last":0.29499999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:39:29 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":3.3029999999999999,"15min":3.548},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.20599999999999999},"max":{"1min":7.6870000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.004,"5min":3.3860000000000001,"15min":3.899},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":8.8510000000000009,"5min":79.614000000000004,"15min":91.685000000000002},"last":3.2530000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":3.3109999999999999,"15min":2.4630000000000001},"min":{"1min":0.36599999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3570000000000002,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.224,"5min":3.5569999999999999,"15min":2.7509999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":5.2889999999999997,"5min":82.872,"15min":82.872},"last":0.40300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":3.6110000000000002,"15min":2.3639999999999999},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.5199999999999996,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.212,"5min":3.7320000000000002,"15min":2.532},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":5.3300000000000001,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":3.2559999999999998,"15min":3.4180000000000001},"min":{"1min":0.35199999999999998,"5min":0.313,"15min":0.30199999999999999},"max":{"1min":5.3630000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.4369999999999998,"15min":3.367},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":9.3740000000000006,"5min":80.364999999999995,"15min":122.517},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":3.786,"15min":3.0459999999999998},"min":{"1min":0.39200000000000002,"5min":0.35199999999999998,"15min":0.34300000000000003},"max":{"1min":9.4190000000000005,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.0469999999999999,"5min":3.3079999999999998,"15min":2.6629999999999998},"min":{"1min":0.40300000000000002,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":5.5990000000000002,"5min":82.798000000000002,"15min":82.798000000000002},"last":0.54100000000000004}]}]},{"osd":3,"up_from":149,"seq":639950127391,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2530956,"kb_used_data":1716228,"kb_used_omap":2,"kb_used_meta":814717,"kb_avail":91214196,"statfs":{"total":95995035648,"available":93403336704,"internally_reserved":0,"allocated":1757417472,"data_stored":1755432349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":834270261},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4319999999999999,"5min":1.9410000000000001,"15min":4.2610000000000001},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.19},"max":{"1min":16.562000000000001,"5min":16.562000000000001,"15min":161.821},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.121,"15min":3.9940000000000002},"min":{"1min":0.192,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":6.6050000000000004,"5min":12.298,"15min":163.52000000000001},"last":0.51800000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":4.5129999999999999,"15min":3.863},"min":{"1min":0.27100000000000002,"5min":0.22800000000000001,"15min":0.21199999999999999},"max":{"1min":5.1630000000000003,"5min":184.012,"15min":184.012},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.246,"5min":4.7699999999999996,"15min":4.6399999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":22.184000000000001,"5min":184.703,"15min":184.703},"last":0.50600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":3.7320000000000002,"15min":3.5990000000000002},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":12.390000000000001,"5min":182.44900000000001,"15min":182.44900000000001},"last":118.93899999999999},{"interface":"front","average":{"1min":2.1440000000000001,"5min":4.0220000000000002,"15min":4.0590000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":18.109000000000002,"5min":182.56399999999999,"15min":182.56399999999999},"last":118.97199999999999}]},{"osd":4,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.625,"5min":5.5949999999999998,"15min":3.3519999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26900000000000002},"max":{"1min":133.80099999999999,"5min":178.22200000000001,"15min":178.22200000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":8.8919999999999995,"5min":5.6079999999999997,"15min":3.359},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":133.648,"5min":180.43600000000001,"15min":180.43600000000001},"last":0.90900000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":8.4130000000000003,"5min":6.4509999999999996,"15min":3.5859999999999999},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":128.083,"5min":177.56299999999999,"15min":177.56299999999999},"last":0.875},{"interface":"front","average":{"1min":8.2870000000000008,"5min":5.3540000000000001,"15min":2.9510000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":128.11799999999999,"5min":177.59399999999999,"15min":177.59399999999999},"last":0.86899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.016,"15min":2.7530000000000001},"min":{"1min":0.39900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.5779999999999994,"5min":11.994999999999999,"15min":130.39599999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.151,"5min":1.581,"15min":2.3199999999999998},"min":{"1min":0.38300000000000001,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":5.359,"5min":9.8179999999999996,"15min":130.67699999999999},"last":0.84699999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:39:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.0510000000000002,"15min":2.7930000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.7590000000000003,"5min":10.195,"15min":121.613},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.3140000000000001,"5min":2.8149999999999999,"15min":3.1779999999999999},"min":{"1min":0.42699999999999999,"5min":0.372,"15min":0.33400000000000002},"max":{"1min":5.1849999999999996,"5min":87.001999999999995,"15min":121.559},"last":0.93600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019427,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2937100,"kb_used_data":1465668,"kb_used_omap":0,"kb_used_meta":1471360,"kb_avail":90808052,"statfs":{"total":95995035648,"available":92987445248,"internally_reserved":0,"allocated":1500844032,"data_stored":1499793944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1506672640},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:40:20 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.333,"15min":5.9059999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.22800000000000001},"max":{"1min":5.4569999999999999,"5min":31.366,"15min":287.58800000000002},"last":2.5089999999999999},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.9139999999999999,"15min":6.1349999999999998},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":5.468,"5min":31.373999999999999,"15min":287.57100000000003},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":8.7319999999999993,"5min":3.141,"15min":3.7349999999999999},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.22900000000000001},"max":{"1min":163.392,"5min":163.392,"15min":163.392},"last":2.6080000000000001},{"interface":"front","average":{"1min":8.9659999999999993,"5min":3.1749999999999998,"15min":3.8479999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":163.42400000000001,"5min":163.42400000000001,"15min":163.42400000000001},"last":2.544}]},{"osd":2,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":9.641,"5min":5.5220000000000002,"15min":4.8550000000000004},"min":{"1min":0.32400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":175.97900000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":9.4800000000000004,"5min":5.657,"15min":4.7969999999999997},"min":{"1min":0.32400000000000001,"5min":0.27400000000000002,"15min":0.26400000000000001},"max":{"1min":175.875,"5min":175.875,"15min":175.875},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.5409999999999999,"15min":5.6559999999999997},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":6.8920000000000003,"5min":94.984999999999999,"15min":161.19300000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.411,"5min":3.1480000000000001,"15min":5.3979999999999997},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.524,"5min":94.966999999999999,"15min":161.934},"last":2.5760000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.3819999999999999,"15min":1.028},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":1.1799999999999999,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.45300000000000001,"5min":1.518,"15min":1.1080000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":0.879,"5min":9.5009999999999994,"15min":9.5009999999999994},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:40:20 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":1.214,"15min":1.95},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.64400000000000002,"5min":9.4239999999999995,"15min":163.36699999999999},"last":2.4329999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":1.4770000000000001,"15min":2.2029999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":1.1599999999999999,"5min":10.705,"15min":163.381},"last":0.84099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:39:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4820000000000002,"15min":1.6950000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":3.8140000000000001,"5min":106.801,"15min":106.801},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.70599999999999996,"5min":2.6030000000000002,"15min":1.722},"min":{"1min":0.40000000000000002,"5min":0.36199999999999999,"15min":0.32900000000000001},"max":{"1min":3.3300000000000001,"5min":106.827,"15min":106.827},"last":2.476}]}]},{"osd":5,"up_from":38,"seq":163208757792,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2769028,"kb_used_data":1285436,"kb_used_omap":0,"kb_used_meta":1483584,"kb_avail":90976124,"statfs":{"total":95995035648,"available":93159550976,"internally_reserved":0,"allocated":1316286464,"data_stored":1315240820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519190016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:40:13 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":5.2380000000000004,"15min":6.2249999999999996},"min":{"1min":0.25,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":66.799000000000007,"5min":147.405,"15min":294.89699999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":4.673,"5min":6.3529999999999998,"15min":6.3769999999999998},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.215},"max":{"1min":66.614000000000004,"5min":146.31700000000001,"15min":294.30399999999997},"last":0.505}]},{"osd":1,"last update":"Mon Oct 9 06:40:00 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":13.241,"15min":6.9169999999999998},"min":{"1min":0.26400000000000001,"5min":0.22,"15min":0.22},"max":{"1min":2.9169999999999998,"5min":188.34100000000001,"15min":188.34100000000001},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":12.952999999999999,"15min":6.9160000000000004},"min":{"1min":0.27700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.2280000000000002,"5min":188.839,"15min":188.839},"last":3.512}]},{"osd":2,"last update":"Mon Oct 9 06:40:00 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":7.5,"15min":4.306},"min":{"1min":0.29799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.0630000000000002,"5min":120.613,"15min":120.613},"last":1.7370000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":7.2720000000000002,"15min":4.1529999999999996},"min":{"1min":0.307,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":2.9889999999999999,"5min":120.962,"15min":120.962},"last":0.53500000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:40:19 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":6.4219999999999997,"15min":6.6040000000000001},"min":{"1min":0.35699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.6280000000000001,"5min":220.41800000000001,"15min":220.41800000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":6.0060000000000002,"15min":6.3209999999999997},"min":{"1min":0.308,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.7789999999999999,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.91600000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:39:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":2.4340000000000002,"15min":1.478},"min":{"1min":0.29399999999999998,"5min":0.255,"15min":0.247},"max":{"1min":2.8740000000000001,"5min":79.043000000000006,"15min":79.043000000000006},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":2.6080000000000001,"15min":1.4790000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":2.8319999999999999,"5min":79.165999999999997,"15min":79.165999999999997},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.786,"15min":1.488},"min":{"1min":0.35199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":2.6219999999999999,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":1.835,"15min":1.575},"min":{"1min":0.34699999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.78500000000000003,"5min":80.781999999999996,"15min":80.781999999999996},"last":1.6839999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.4710000000000001,"15min":1.6779999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":2.9430000000000001,"5min":78.890000000000001,"15min":78.890000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":2.1589999999999998,"15min":1.5349999999999999},"min":{"1min":0.38300000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.859,"5min":79.138999999999996,"15min":79.138999999999996},"last":1.72}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2036203520,"data_stored":2036203520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2001403904,"data_stored":2001403904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1700724736,"data_stored":1700724736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1767702528,"data_stored":1767702528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1472724992,"data_stored":1472724992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1237188608,"data_stored":1237188608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":979501056,"data_stored":979501056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":730529792,"data_stored":730529792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:40:33.450 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:40:33.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pg_num 26 2023-10-09T06:40:33.569 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:33.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:33.681 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:33.734 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:33.772 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:33.984 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:33.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:34.093 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:34.124 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 49 16 66513 66497 84.7651 60 0.022464 0.0117922 2023-10-09T06:40:34.126 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:34.177 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:34.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:34.283 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:66694 14.1 e497/489 14:827cb143:::benchmark_data_smithi107_590976_object66693:head v 497'4130, mlcod=497'4128)", 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:34.324803+0000", 2023-10-09T06:40:34.330 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0037756600000000001, 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038066340000000001, 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324803+0000", 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:40:34.331 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324803+0000", 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324802+0000", 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324831+0000", 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9822e-05 2023-10-09T06:40:34.332 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324845+0000", 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3899e-05 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324854+0000", 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9290000000000004e-06 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.333 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:40:34.334 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324926+0000", 2023-10-09T06:40:34.334 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2028999999999998e-05 2023-10-09T06:40:34.334 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.324954+0000", 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7392000000000001e-05 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:66693 14.0 e497/489 14:0757349f:::benchmark_data_smithi107_590976_object66692:head v 497'4173, mlcod=497'4171)", 2023-10-09T06:40:34.335 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:34.325570+0000", 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0030084930000000001, 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031093760000000001, 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325570+0000", 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.336 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325570+0000", 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325569+0000", 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:40:34.337 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325592+0000", 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3558999999999999e-05 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325607+0000", 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4501999999999999e-05 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:40:34.338 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325614+0000", 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4070000000000004e-06 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325686+0000", 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1371999999999994e-05 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:40:34.339 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:34.325707+0000", 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0871000000000001e-05 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 2 2023-10-09T06:40:34.340 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:40:34.369 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:34.974 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:34.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:34.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:35.078 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:35.104 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:35.122 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 50 16 67401 67385 84.18 55.5 0.0164214 0.0118383 2023-10-09T06:40:35.148 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:35.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:35.253 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:35.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:34 smithi107 ceph-mon[220921]: pgmap v1541: 119 pgs: 119 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T06:40:35.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:34 smithi107 ceph-mon[212750]: pgmap v1541: 119 pgs: 119 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T06:40:35.281 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:34 smithi150 ceph-mon[194178]: pgmap v1541: 119 pgs: 119 active+clean; 3.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T06:40:35.287 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:35.339 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:35.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:35.445 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:35.482 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:35.589 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:40:35.820 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:35.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:35.922 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:40:35.934 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:35.962 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:35.963 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:36.010 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:36.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:36.115 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:36.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 51 16 68216 68200 83.5284 50.9375 0.00916155 0.011965 2023-10-09T06:40:36.143 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:36.199 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:36.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:36.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:36.306 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:36.396 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:36.757 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:36.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:36.866 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:36.899 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:36.954 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:36.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:37.061 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:37.122 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 52 16 69117 69101 83.0051 56.3125 0.0252417 0.0120406 2023-10-09T06:40:37.128 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:37.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:36 smithi150 ceph-mon[194178]: pgmap v1542: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:40:37.228 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:37.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:37.241 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:36 smithi107 ceph-mon[212750]: pgmap v1542: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:40:37.242 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:36 smithi107 ceph-mon[220921]: pgmap v1542: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-10-09T06:40:37.342 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:37.384 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:37.443 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:37.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:37.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:37.551 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:37.622 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:37.784 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:37.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:37.817 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:37.818 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:37.890 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:37.991 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:38.058 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:40:38.123 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 53 16 70127 70111 82.63 63.125 0.00990227 0.0120976 2023-10-09T06:40:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:40:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:40:38.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:40:39.124 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 54 16 71116 71100 82.2423 61.8125 0.0249388 0.0121527 2023-10-09T06:40:39.206 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:39.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:39.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:39.258 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: pgmap v1543: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[212750]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.259 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: pgmap v1543: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T06:40:39.260 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:39.260 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:40:39.260 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:40:39.260 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.260 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:38 smithi107 ceph-mon[220921]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:39.343 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: pgmap v1543: 119 pgs: 119 active+clean; 4.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2023-10-09T06:40:39.343 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:40:39.343 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1379037773' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:40:39.343 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: osdmap e498: 8 total, 8 up, 8 in 2023-10-09T06:40:39.343 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.344 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:38 smithi150 ceph-mon[194178]: failed to encode map e498 with expected crc 2023-10-09T06:40:39.346 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:39.409 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:39.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:39.516 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:39.589 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:40.124 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 55 16 72013 71997 81.7663 56.0625 0.00351383 0.0122175 2023-10-09T06:40:41.044 INFO:teuthology.orchestra.run.smithi107.stderr:set pool 14 pg_num to 26 2023-10-09T06:40:41.126 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 56 16 72952 72936 81.3524 58.6875 0.00639498 0.012287 2023-10-09T06:40:41.353 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:41.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:41 smithi107 ceph-mon[212750]: pgmap v1545: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T06:40:41.363 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:40:41.364 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:41 smithi107 ceph-mon[220921]: pgmap v1545: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T06:40:41.364 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:40:41.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:41 smithi150 ceph-mon[194178]: pgmap v1545: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-09T06:40:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T06:40:41.465 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:41.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:40:41.540 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:42.039 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:42.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:42.133 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 57 16 73708 73692 80.7537 47.25 0.0409271 0.0123642 2023-10-09T06:40:42.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:42.248 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:42.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:40:42.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:40:42.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:40:42.286 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:42] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2023-10-09T06:40:42.286 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:40:42.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:40:42.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:40:42.296 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:42.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:42.402 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/230228696' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T06:40:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: osdmap e499: 8 total, 8 up, 8 in 2023-10-09T06:40:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: failed to encode map e499 with expected crc 2023-10-09T06:40:42.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T06:40:42.440 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:42.562 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:42.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:42.648 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:40:42.648 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:40:42.648 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:40:42.648 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:40:42.648 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:40:42.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:42.699 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:42.743 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:42.752 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:42.753 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:42.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:42.776 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:42.798 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:42.818 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:42.837 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:42.855 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:42.859 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:42.876 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:42.886 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:42.905 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:42.906 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.906 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.906 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.906 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.907 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.907 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.907 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.907 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:40:42.986 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:42.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:43.091 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:43.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 58 16 73720 73704 79.375 0.75 0.83806 0.0124185 2023-10-09T06:40:43.129 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: pgmap v1547: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3027162365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: pgmap v1547: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:40:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:40:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:40:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3027162365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: pgmap v1547: 119 pgs: 119 active+clean; 4.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2023-10-09T06:40:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T06:40:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: osdmap e500: 8 total, 8 up, 8 in 2023-10-09T06:40:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: failed to encode map e500 with expected crc 2023-10-09T06:40:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3027162365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:44.079 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:40:44.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 59 16 74668 74652 79.0336 59.25 0.011243 0.0126484 2023-10-09T06:40:44.400 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e500 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e500 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:44 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e500 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: osdmap e501: 8 total, 8 up, 8 in 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:44.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:44 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:45.078 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:45 smithi107 ceph-mon[212750]: failed to encode map e501 with expected crc 2023-10-09T06:40:45.079 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:45 smithi107 ceph-mon[212750]: pgmap v1550: 129 pgs: 3 peering, 10 unknown, 116 active+clean; 4.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail 2023-10-09T06:40:45.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:40:45.127828+0000 min lat: 0.0010258 max lat: 1.15969 avg lat: 0.0126661 2023-10-09T06:40:45.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:40:45.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 60 15 75830 75815 78.9278 72.6875 0.00413992 0.0126661 2023-10-09T06:40:45.151 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:45.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:45.266 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:45.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:45 smithi107 ceph-mon[220921]: failed to encode map e501 with expected crc 2023-10-09T06:40:45.329 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:45 smithi107 ceph-mon[220921]: pgmap v1550: 129 pgs: 3 peering, 10 unknown, 116 active+clean; 4.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail 2023-10-09T06:40:45.332 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:45 smithi150 ceph-mon[194178]: failed to encode map e501 with expected crc 2023-10-09T06:40:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:45 smithi150 ceph-mon[194178]: pgmap v1550: 129 pgs: 3 peering, 10 unknown, 116 active+clean; 4.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail 2023-10-09T06:40:45.539 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:45.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:45.660 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:45.700 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:45.763 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:45.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:45.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:45.871 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:45.938 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:45.961 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:46.049 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:46.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:46.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:46.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:46 smithi107 ceph-mon[212750]: 14.7 scrub starts 2023-10-09T06:40:46.096 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:46 smithi107 ceph-mon[212750]: 14.7 scrub ok 2023-10-09T06:40:46.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 61 16 76988 76972 78.8193 72.3125 0.00838132 0.0126819 2023-10-09T06:40:46.163 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:46.201 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:46.263 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:46.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:46.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:46 smithi107 ceph-mon[220921]: 14.7 scrub starts 2023-10-09T06:40:46.354 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:46 smithi107 ceph-mon[220921]: 14.7 scrub ok 2023-10-09T06:40:46.369 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.396 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:46 smithi150 ceph-mon[194178]: 14.7 scrub starts 2023-10-09T06:40:46.396 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:46 smithi150 ceph-mon[194178]: 14.7 scrub ok 2023-10-09T06:40:46.399 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:46.453 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:46.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:46.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:46.598 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:46.667 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:46.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:46.775 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:46.807 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:46.863 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:46.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:46.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:46.970 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:47.004 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:47.057 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:47.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:47.104 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:40:47.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[212750]: pgmap v1551: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:47.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:47.106 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:47.106 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[220921]: pgmap v1551: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:47.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:47.107 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:47.127 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 62 16 78064 78048 78.6326 67.25 0.00309336 0.0126919 2023-10-09T06:40:47.164 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:47.190 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:47.232 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:47.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:47.275 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 6, 7, 3, 5, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:40:47.275 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:40:47.337 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:47.364 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:47 smithi150 ceph-mon[194178]: pgmap v1551: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:40:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:40:47.407 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:47.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:47.511 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:47.542 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:47.557 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:40:47.603 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:47.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:47.708 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:47.826 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:48.131 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 63 16 78776 78760 78.0862 44.5 0.0184352 0.0128015 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: 14.5 scrub starts 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: 14.1 scrub starts 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: 14.17 scrub starts 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: 14.5 scrub ok 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[212750]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: 14.5 scrub starts 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: 14.1 scrub starts 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: 14.17 scrub starts 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: 14.5 scrub ok 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:48 smithi107 ceph-mon[220921]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: 14.5 scrub starts 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: 14.1 scrub starts 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1362575295' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: osdmap e502: 8 total, 8 up, 8 in 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: 14.17 scrub starts 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: 14.5 scrub ok 2023-10-09T06:40:48.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:48 smithi150 ceph-mon[194178]: failed to encode map e502 with expected crc 2023-10-09T06:40:48.981 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:48.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:49.094 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:49.127 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:49.133 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 64 16 79679 79663 77.7454 56.4375 0.0117528 0.0128585 2023-10-09T06:40:49.187 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:49.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:49.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:49.247 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:49.297 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:49.345 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:49.358 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[212750]: pgmap v1553: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:49.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[212750]: 14.3 scrub starts 2023-10-09T06:40:49.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[212750]: 14.1 scrub ok 2023-10-09T06:40:49.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[212750]: 14.9 scrub starts 2023-10-09T06:40:49.359 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[212750]: 14.9 scrub ok 2023-10-09T06:40:49.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[220921]: pgmap v1553: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:49.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[220921]: 14.3 scrub starts 2023-10-09T06:40:49.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[220921]: 14.1 scrub ok 2023-10-09T06:40:49.360 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[220921]: 14.9 scrub starts 2023-10-09T06:40:49.361 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:49 smithi107 ceph-mon[220921]: 14.9 scrub ok 2023-10-09T06:40:49.394 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:49.397 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:49.401 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:49 smithi150 ceph-mon[194178]: pgmap v1553: 129 pgs: 3 peering, 3 unknown, 123 active+clean; 4.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:49.401 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:49 smithi150 ceph-mon[194178]: 14.3 scrub starts 2023-10-09T06:40:49.401 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:49 smithi150 ceph-mon[194178]: 14.1 scrub ok 2023-10-09T06:40:49.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:49 smithi150 ceph-mon[194178]: 14.9 scrub starts 2023-10-09T06:40:49.402 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:49 smithi150 ceph-mon[194178]: 14.9 scrub ok 2023-10-09T06:40:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:49.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:49.503 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:49.535 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:49.602 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:49.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:49.708 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:49.713 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:40:49.713 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:40:49.801 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:50.137 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 65 16 80753 80737 77.5786 67.125 0.00787211 0.0128857 2023-10-09T06:40:50.300 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:50 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:40:50.300 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:50 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:40:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:50 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T06:40:50.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:40:51.136 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 66 16 81263 81247 76.8865 31.875 0.00233646 0.0128608 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: pgmap v1554: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: 14.19 scrub starts 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[212750]: 14.19 scrub ok 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: pgmap v1554: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: 14.19 scrub starts 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:51 smithi107 ceph-mon[220921]: 14.19 scrub ok 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: pgmap v1554: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 34 KiB/s, 0 objects/s recovering 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: 14.19 scrub starts 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: osdmap e503: 8 total, 8 up, 8 in 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: failed to encode map e503 with expected crc 2023-10-09T06:40:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:51 smithi150 ceph-mon[194178]: 14.19 scrub ok 2023-10-09T06:40:51.791 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:40:51.890 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:51.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:51.994 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:52.021 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:52.062 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:52.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:52.137 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 67 16 81263 81247 75.7396 0 - 0.0128608 2023-10-09T06:40:52.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:40:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:40:52] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2023-10-09T06:40:52.168 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:52.199 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:52.253 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:52.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:52.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:52.359 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:40:52.390 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: 14.4 scrub starts 2023-10-09T06:40:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: 14.4 scrub ok 2023-10-09T06:40:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:52 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.474 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: 14.4 scrub starts 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: 14.4 scrub ok 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: osdmap e504: 8 total, 8 up, 8 in 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.475 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: failed to encode map e504 with expected crc 2023-10-09T06:40:52.476 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: 14.4 scrub starts 2023-10-09T06:40:52.476 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: 14.4 scrub ok 2023-10-09T06:40:52.476 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:52 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T06:40:52.522 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:40:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:52.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:40:52.635 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:52.709 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:52.933 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:52.959 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:52.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:52.987 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:40:53.067 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:53.117 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:53.137 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 68 16 81697 81681 75.025 13.5625 0.00561078 0.0132401 2023-10-09T06:40:53.204 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:53.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:53.253 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:40:53.284 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:40:53.313 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:53.314 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:40:53.342 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:53.346 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:40:53.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:40:53.393 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:53.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: pgmap v1557: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: 14.17 scrub starts 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: 14.17 scrub ok 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.404 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:53.404 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:40:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: pgmap v1557: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: 14.17 scrub starts 2023-10-09T06:40:53.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.411 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: 14.17 scrub ok 2023-10-09T06:40:53.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:40:53.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.412 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:53 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:53.431 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:40:53.460 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:40:53.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.464 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.464 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.464 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:40:53.501 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:53.526 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: pgmap v1557: 129 pgs: 129 active+clean; 4.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 22 MiB/s wr, 347 op/s; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:40:53.526 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: 14.17 scrub starts 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: osdmap e505: 8 total, 8 up, 8 in 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e505 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: 14.17 scrub ok 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: osdmap e506: 8 total, 8 up, 8 in 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: failed to encode map e506 with expected crc 2023-10-09T06:40:53.527 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:53 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:40:53.528 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:53.573 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:53.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:53.647 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:40:53.679 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:53.706 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:53.749 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:53.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:53.856 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:40:53.889 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:54.138 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 69 16 82151 82135 74.3482 28.375 0.00747708 0.013446 2023-10-09T06:40:54.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/446331598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1753463570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/446331598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1753463570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:40:54.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:40:54.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:54 smithi107 ceph-mon[220921]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: failed to encode map e506 with expected crc 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/446331598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1753463570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: osdmap e507: 8 total, 8 up, 8 in 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: failed to encode map e507 with expected crc 2023-10-09T06:40:54.656 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:54 smithi150 ceph-mon[194178]: failed to encode map e507 with expected crc 2023-10-09T06:40:55.138 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 70 16 83050 83034 74.0889 56.1875 0.024312 0.0134923 2023-10-09T06:40:55.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[212750]: pgmap v1560: 129 pgs: 1 remapped+peering, 128 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2023-10-09T06:40:55.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[212750]: failed to encode map e507 with expected crc 2023-10-09T06:40:55.543 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[212750]: 14.12 scrub starts 2023-10-09T06:40:55.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[220921]: pgmap v1560: 129 pgs: 1 remapped+peering, 128 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2023-10-09T06:40:55.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[220921]: failed to encode map e507 with expected crc 2023-10-09T06:40:55.547 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:55 smithi107 ceph-mon[220921]: 14.12 scrub starts 2023-10-09T06:40:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:55 smithi150 ceph-mon[194178]: pgmap v1560: 129 pgs: 1 remapped+peering, 128 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail 2023-10-09T06:40:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:55 smithi150 ceph-mon[194178]: failed to encode map e507 with expected crc 2023-10-09T06:40:55.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:55 smithi150 ceph-mon[194178]: 14.12 scrub starts 2023-10-09T06:40:55.669 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:40:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:55.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:40:55.780 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:55.818 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:55.903 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:55.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:56.010 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:40:56.052 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:56.127 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:40:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:56.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:40:56.138 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 71 16 84190 84174 74.0488 71.25 0.00409298 0.0134999 2023-10-09T06:40:56.231 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:56.301 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:56 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:40:56.503 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:56 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:40:56.504 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:56 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:40:57.138 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 72 16 85128 85112 73.8346 58.625 0.0126353 0.0135395 2023-10-09T06:40:57.492 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:57.493 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:57 smithi107 ceph-mon[212750]: pgmap v1562: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:40:57.494 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:57 smithi107 ceph-mon[220921]: pgmap v1562: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:40:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:57.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:57.595 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:40:57.628 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:57 smithi150 ceph-mon[194178]: pgmap v1562: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:40:57.634 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:57.694 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:40:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:57.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:40:57.805 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:57.884 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:58.140 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 73 16 86209 86193 73.7477 67.5625 0.0171069 0.0135554 2023-10-09T06:40:58.630 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:58.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:40:58.738 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:58.751 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:40:58.768 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:58.822 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:58.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:58 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:40:58.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:58 smithi107 ceph-mon[212750]: pgmap v1563: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 949 KiB/s, 14 objects/s recovering 2023-10-09T06:40:58.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:58 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:40:58.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:58 smithi107 ceph-mon[220921]: pgmap v1563: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 949 KiB/s, 14 objects/s recovering 2023-10-09T06:40:58.903 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:58 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:40:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:58 smithi150 ceph-mon[194178]: pgmap v1563: 129 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 127 active+clean; 5.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 7314/246336 objects misplaced (2.969%); 949 KiB/s, 14 objects/s recovering 2023-10-09T06:40:58.928 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:40:58.965 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:59.027 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:40:59.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:59.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:40:59.134 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:40:59.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 74 16 87085 87069 73.4911 54.75 0.0246587 0.0136029 2023-10-09T06:40:59.164 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:59.224 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:40:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:59.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:40:59.330 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:40:59.422 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:59.547 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:40:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:59.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:40:59.589 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:40:59.589 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:40:59.589 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:40:59.589 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:40:59.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:40:59 smithi107 ceph-mon[212750]: 14.12 scrub starts 2023-10-09T06:40:59.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:40:59 smithi107 ceph-mon[220921]: 14.12 scrub starts 2023-10-09T06:40:59.660 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:40:59.766 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:40:59.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:40:59 smithi150 ceph-mon[194178]: 14.12 scrub starts 2023-10-09T06:40:59.940 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:40:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:40:59.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:00.044 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:00.106 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:00.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 75 16 87950 87934 73.2321 54.0625 0.00660113 0.013651 2023-10-09T06:41:00.318 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:00.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:00.429 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:00.491 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:00.609 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:00.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:00.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:41:00.686 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:41:00.714 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:00.789 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:00.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[212750]: pgmap v1564: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 2497/260085 objects degraded (0.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2365345231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[220921]: pgmap v1564: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 2497/260085 objects degraded (0.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2365345231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:00 smithi150 ceph-mon[194178]: pgmap v1564: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T06:41:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:00 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 2497/260085 objects degraded (0.960%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2365345231' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:00.919 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:00.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:01.025 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:01.089 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:01.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 76 16 88905 88889 73.0539 59.6875 0.017865 0.0136825 2023-10-09T06:41:02.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: 14.2 deep-scrub starts 2023-10-09T06:41:02.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:02.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.014 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:02.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:02.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: 14.2 deep-scrub starts 2023-10-09T06:41:02.017 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:02.018 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:02.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 77 16 89735 89719 72.779 51.875 0.00263175 0.0137275 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: 14.2 deep-scrub starts 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: osdmap e508: 8 total, 8 up, 8 in 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: failed to encode map e508 with expected crc 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:02.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:02] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T06:41:02.411 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:02.457 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:41:02.564 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:02.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:02.675 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:02.739 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:02 smithi107 ceph-mon[212750]: pgmap v1566: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:41:02.739 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:02 smithi107 ceph-mon[220921]: pgmap v1566: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:41:02.844 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:02.967 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:02.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:03.087 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:03.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 78 16 90461 90445 72.4278 45.375 0.0156966 0.0138028 2023-10-09T06:41:03.152 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:03.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:02 smithi150 ceph-mon[194178]: pgmap v1566: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 2497/260085 objects degraded (0.960%); 9327/260085 objects misplaced (3.586%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T06:41:03.199 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:03.433 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:03.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:03.542 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:03.588 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:03.714 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:03.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:03.819 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:03.856 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:03.994 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:04.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:04.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:04.102 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:04.139 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 79 16 91501 91485 72.3337 65 0.0127939 0.0138205 2023-10-09T06:41:04.276 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:04.632 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:04.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:04.740 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:41:04.741 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:04.818 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:04.854 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:04.910 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:04 smithi107 ceph-mon[212750]: pgmap v1567: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 2497/266109 objects degraded (0.938%); 9327/266109 objects misplaced (3.505%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:04.911 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:04 smithi107 ceph-mon[220921]: pgmap v1567: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 2497/266109 objects degraded (0.938%); 9327/266109 objects misplaced (3.505%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:05.020 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:05.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:05.136 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:05.140 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:2023-10-09T06:41:05.140503+0000 min lat: 0.0010258 max lat: 2.25157 avg lat: 0.0138666 2023-10-09T06:41:05.140 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T06:41:05.140 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 80 16 92345 92329 72.089 52.75 0.0309464 0.0138666 2023-10-09T06:41:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:04 smithi150 ceph-mon[194178]: pgmap v1567: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 34 MiB/s wr, 543 op/s; 2497/266109 objects degraded (0.938%); 9327/266109 objects misplaced (3.505%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:05.230 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:05.400 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:05.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:05.505 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:05.587 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:06.140 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 81 16 93233 93217 71.8842 55.5 0.00412721 0.0138925 2023-10-09T06:41:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[212750]: pgmap v1568: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:06.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.654 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:06 smithi107 ceph-mon[220921]: pgmap v1568: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:06.793 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:06.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:06.899 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:06 smithi150 ceph-mon[194178]: pgmap v1568: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:06.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:06.983 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:06.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:07.088 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:07.125 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:07.145 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 82 16 93994 93978 71.5846 47.5625 0.0228223 0.0139633 2023-10-09T06:41:07.192 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:07.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:07.302 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:07.394 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:07.517 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:41:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/273327 objects degraded (0.914%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1840750597' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:07.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:07.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/273327 objects degraded (0.914%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:07.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1840750597' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:07.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:07 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/273327 objects degraded (0.914%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1840750597' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:07 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:41:08.146 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 83 16 94906 94890 71.4072 57 0.0248759 0.0139996 2023-10-09T06:41:08.630 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:08.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:08.742 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:08.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:08 smithi150 ceph-mon[194178]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:41:08.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:08 smithi150 ceph-mon[194178]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:41:08.786 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:08 smithi150 ceph-mon[194178]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.787 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:08 smithi150 ceph-mon[194178]: pgmap v1570: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:08.789 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:08.878 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:08.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[212750]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:41:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[212750]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:41:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[212750]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[212750]: pgmap v1570: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[220921]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:41:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[220921]: osdmap e509: 8 total, 8 up, 8 in 2023-10-09T06:41:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[220921]: failed to encode map e509 with expected crc 2023-10-09T06:41:08.904 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:08 smithi107 ceph-mon[220921]: pgmap v1570: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 2497/273327 objects degraded (0.914%); 8842/273327 objects misplaced (3.235%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:08.985 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:09.011 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:09.053 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:09.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:09.145 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 84 16 95715 95699 71.1592 50.5625 0.00559643 0.0140481 2023-10-09T06:41:09.159 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:09.203 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:09.249 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:09.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:09.354 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:09.415 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:09.704 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:41:09.762 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:09.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:09.871 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:10.013 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:10.097 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:10.148 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 85 16 96533 96517 70.9211 51.125 0.01277 0.0140947 2023-10-09T06:41:10.217 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:10.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:10.280 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:41:10.316 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:41:10.325 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:10.383 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:41:10.409 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:10.449 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:41:10.499 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:41:10.545 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:41:10.575 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:10.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:10.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:10.586 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:10.586 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:10.591 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:41:10.633 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:41:10.666 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.667 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:41:10.688 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:10.718 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:10.785 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:10.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:10.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:10.863 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:10 smithi107 ceph-mon[220921]: pgmap v1571: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:41:10.864 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1685830017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:10.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:10 smithi107 ceph-mon[212750]: pgmap v1571: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:41:10.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1685830017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:10.893 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:10.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:11.149 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 86 16 97348 97332 70.6883 50.9375 0.00733114 0.0141416 2023-10-09T06:41:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:10 smithi150 ceph-mon[194178]: pgmap v1571: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T06:41:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1685830017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:12.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 87 16 98424 98408 70.6487 67.25 0.00974781 0.0141505 2023-10-09T06:41:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:12] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.43.0" 2023-10-09T06:41:13.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:12 smithi107 ceph-mon[212750]: pgmap v1572: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:13.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/286278 objects degraded (0.872%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:13.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:12 smithi107 ceph-mon[220921]: pgmap v1572: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:13.009 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/286278 objects degraded (0.872%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:13.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 88 16 99220 99204 70.4112 49.75 0.00227233 0.0141804 2023-10-09T06:41:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:12 smithi150 ceph-mon[194178]: pgmap v1572: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 2497/286278 objects degraded (0.872%); 8397/286278 objects misplaced (2.933%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/286278 objects degraded (0.872%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:13.644 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:13.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:13.752 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:13.833 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:14.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout: 89 16 100104 100088 70.2409 55.25 0.00218405 0.0142294 2023-10-09T06:41:14.471 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:41:14.472 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:14.504 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:14.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:14.612 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:14.646 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:14.710 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:14.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:14.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:14.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:14 smithi150 ceph-mon[194178]: pgmap v1573: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 2497/292425 objects degraded (0.854%); 8397/292425 objects misplaced (2.872%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:14.923 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:14 smithi107 ceph-mon[220921]: pgmap v1573: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 2497/292425 objects degraded (0.854%); 8397/292425 objects misplaced (2.872%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:14.923 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:14 smithi107 ceph-mon[212750]: pgmap v1573: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 2497/292425 objects degraded (0.854%); 8397/292425 objects misplaced (2.872%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:14.945 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:15.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total time run: 90.0145 2023-10-09T06:41:15.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Total writes made: 100770 2023-10-09T06:41:15.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Write size: 65536 2023-10-09T06:41:15.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Object size: 65536 2023-10-09T06:41:15.150 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Bandwidth (MB/sec): 69.9679 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Bandwidth: 29.3353 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max bandwidth (MB/sec): 160.188 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min bandwidth (MB/sec): 0 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average IOPS: 1119 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev IOPS: 469.365 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max IOPS: 2563 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min IOPS: 0 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Average Latency(s): 0.0142878 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Stddev Latency(s): 0.0372927 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Max latency(s): 2.25157 2023-10-09T06:41:15.151 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Min latency(s): 0.0010258 2023-10-09T06:41:15.155 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T06:41:15.429 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:15.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:15.542 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:15.585 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:15.636 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:15.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:15.748 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:15.792 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:15.863 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:15.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:15.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:15.972 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:16.065 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:16.388 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:41:16.686 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[212750]: pgmap v1574: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3255819799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[220921]: pgmap v1574: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3255819799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:16.728 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:16 smithi150 ceph-mon[194178]: pgmap v1574: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3255819799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:16 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/299316 objects degraded (0.834%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/299316 objects degraded (0.834%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: osdmap e510: 8 total, 8 up, 8 in 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: failed to encode map e510 with expected crc 2023-10-09T06:41:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/299316 objects degraded (0.834%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:18.300 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:18.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:18.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:18.436 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:18.491 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:18.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:18.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:18.596 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:18.685 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.022 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:19.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:18 smithi107 ceph-mon[220921]: pgmap v1576: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:19.031 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:18 smithi107 ceph-mon[212750]: pgmap v1576: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:19.063 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:19.076 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:19.131 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:18 smithi150 ceph-mon[194178]: pgmap v1576: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 6.1 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 2497/299316 objects degraded (0.834%); 7944/299316 objects misplaced (2.654%); 1.9 MiB/s, 29 objects/s recovering 2023-10-09T06:41:19.162 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.218 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:19.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:19.354 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.409 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:19.518 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:19.547 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.597 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:19.702 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:19.732 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.762 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:41:19.788 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:19.894 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:19.930 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:19.965 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/778270843' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.965 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:19 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.966 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/778270843' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.966 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:19 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.986 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:19.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/778270843' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.991 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:19 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:41:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:19.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:20.093 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:20.132 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:20.199 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:20.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:20.308 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:20.359 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:20.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:20.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:20.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:20.517 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:41:20.528 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:20.707 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: pgmap v1577: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 879 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: failed to encode map e511 with expected crc 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: failed to encode map e511 with expected crc 2023-10-09T06:41:20.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[220921]: failed to encode map e511 with expected crc 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: pgmap v1577: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 879 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:41:20.908 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:20 smithi107 ceph-mon[212750]: failed to encode map e511 with expected crc 2023-10-09T06:41:21.047 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:21.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:21.154 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: pgmap v1577: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 879 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T06:41:21.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:41:21.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: osdmap e511: 8 total, 8 up, 8 in 2023-10-09T06:41:21.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: failed to encode map e511 with expected crc 2023-10-09T06:41:21.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: failed to encode map e511 with expected crc 2023-10-09T06:41:21.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:20 smithi150 ceph-mon[194178]: failed to encode map e511 with expected crc 2023-10-09T06:41:21.234 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:22.407 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:22] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-10-09T06:41:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:22 smithi150 ceph-mon[194178]: pgmap v1579: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 843 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:41:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/292608 objects degraded (0.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:23.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:22 smithi107 ceph-mon[220921]: pgmap v1579: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 843 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:41:23.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/292608 objects degraded (0.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:23.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:22 smithi107 ceph-mon[212750]: pgmap v1579: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 6.0 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 27 MiB/s wr, 843 op/s; 2497/292608 objects degraded (0.853%); 7432/292608 objects misplaced (2.540%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T06:41:23.156 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/292608 objects degraded (0.853%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:23.336 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:41:23.577 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:23.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:23.685 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:23.802 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:24.222 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:24.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:24.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:24.329 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:24.340 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:24 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.340 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:24 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:24 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.340 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:24 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:24 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:24 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:24.431 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:24.525 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:24.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:24.639 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:24.668 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:24.740 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:24.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:24.845 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:24.870 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:24.874 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:24.936 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:24.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:24.958 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:24.958 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:25.044 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:25.052 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:25.076 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:25.133 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:25.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:25.162 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:41:25.192 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:41:25.231 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:41:25.240 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:25.264 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:41:25.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:25 smithi150 ceph-mon[194178]: pgmap v1580: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 8.6 MiB/s wr, 814 op/s; 2497/286404 objects degraded (0.872%); 7432/286404 objects misplaced (2.595%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:41:25.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:41:25.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:41:25.272 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:25 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:25.277 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:25.305 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:41:25.337 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:25.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:25.348 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:41:25.395 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:41:25.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[220921]: pgmap v1580: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 8.6 MiB/s wr, 814 op/s; 2497/286404 objects degraded (0.872%); 7432/286404 objects misplaced (2.595%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:41:25.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:41:25.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:41:25.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:25.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[212750]: pgmap v1580: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 8.6 MiB/s wr, 814 op/s; 2497/286404 objects degraded (0.872%); 7432/286404 objects misplaced (2.595%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T06:41:25.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:41:25.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:41:25.410 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:25 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:25.439 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:41:25.447 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:25.462 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.463 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:41:25.516 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:26.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:26 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3012378030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:26.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:26 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3012378030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:26.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:26 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3012378030' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:27.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:27 smithi107 ceph-mon[212750]: pgmap v1581: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 114 B/s rd, 7.8 MiB/s wr, 999 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:41:27.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:27 smithi107 ceph-mon[220921]: pgmap v1581: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 114 B/s rd, 7.8 MiB/s wr, 999 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:41:27.351 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:27.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:27 smithi150 ceph-mon[194178]: pgmap v1581: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 114 B/s rd, 7.8 MiB/s wr, 999 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T06:41:27.458 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:27.516 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:41:27.528 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:27.953 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:27.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:27.994 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:28.058 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:28.098 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:28.155 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:28.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:28.261 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:28.343 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:28.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:28 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/279234 objects degraded (0.894%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:28.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:28 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/279234 objects degraded (0.894%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:28.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:28 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/279234 objects degraded (0.894%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:28.669 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:28.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:28.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:28.869 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:29.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:29 smithi107 ceph-mon[220921]: pgmap v1582: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 6.9 MiB/s wr, 890 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:41:29.126 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:29 smithi107 ceph-mon[212750]: pgmap v1582: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 6.9 MiB/s wr, 890 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:41:29.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:29 smithi150 ceph-mon[194178]: pgmap v1582: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 6.9 MiB/s wr, 890 op/s; 2497/279234 objects degraded (0.894%); 6816/279234 objects misplaced (2.441%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T06:41:29.580 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:41:30.020 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:30.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:30.128 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:30.172 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:30.278 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:30.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:30.381 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:30.464 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:30.475 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:30.593 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:30.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:30.709 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:30.760 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:30.829 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:30.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:30.939 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:30.970 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:31.037 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:31.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:31.142 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:31.148 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[212750]: pgmap v1583: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2497/271761 objects degraded (0.919%); 6816/271761 objects misplaced (2.508%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:31.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3495854764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.149 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.217 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:31.230 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:115182 14.15 e512/500 14:aa504ffc:::benchmark_data_smithi107_590976_object14409:head v 512'5787, mlcod=512'5785)", 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:31.322793+0000", 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.001055229, 2023-10-09T06:41:31.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010759809999999999, 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322793+0000", 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322793+0000", 2023-10-09T06:41:31.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322793+0000", 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322798+0000", 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1449999999999999e-06 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322806+0000", 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5429999999999994e-06 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322810+0000", 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9029999999999997e-06 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322844+0000", 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4187999999999999e-05 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322861+0000", 2023-10-09T06:41:31.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6725e-05 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:115183 14.19 e512/500 14:9c94fbe9:::benchmark_data_smithi107_590976_object14410:head v 512'5897, mlcod=512'5895)", 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:31.322910+0000", 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 0.00093810300000000003, 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00099976199999999996, 2023-10-09T06:41:31.329 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322910+0000", 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322910+0000", 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:31.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322910+0000", 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322915+0000", 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6289999999999998e-06 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322924+0000", 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3799999999999994e-06 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322929+0000", 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.904e-06 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:31.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.322980+0000", 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1082999999999999e-05 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:31.323005+0000", 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5306000000000001e-05 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: ], 2023-10-09T06:41:31.333 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 2 2023-10-09T06:41:31.334 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:41:31.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:31 smithi150 ceph-mon[194178]: pgmap v1583: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2497/271761 objects degraded (0.919%); 6816/271761 objects misplaced (2.508%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:31.351 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3495854764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.352 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:31 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[220921]: pgmap v1583: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2497/271761 objects degraded (0.919%); 6816/271761 objects misplaced (2.508%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T06:41:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3495854764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:31 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:41:31.704 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:31.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:31.808 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:31.840 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:31.963 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:31.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:32.070 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:32.115 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:32.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:41:32.195 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:41:32.209 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:32.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:32.220 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:32] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:41:32.220 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:32.220 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:32.220 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:32.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:41:32.222 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:32.318 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:32.369 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: osdmap e512: 8 total, 8 up, 8 in 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.370 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: failed to encode map e512 with expected crc 2023-10-09T06:41:32.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:41:32.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:41:32.371 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:32.453 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:32.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:32.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:32.591 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:32.672 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:32.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:32.687 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:32.781 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:32.820 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:32.927 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:32.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:33.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:33.073 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:33.187 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: pgmap v1585: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 2497/265572 objects degraded (0.940%); 6183/265572 objects misplaced (2.328%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.188 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:41:33.189 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:33 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/265572 objects degraded (0.940%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:33.343 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:33.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:33.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: pgmap v1585: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 2497/265572 objects degraded (0.940%); 6183/265572 objects misplaced (2.328%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.492 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/265572 objects degraded (0.940%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: pgmap v1585: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 2497/265572 objects degraded (0.940%); 6183/265572 objects misplaced (2.328%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2090845698' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: osdmap e513: 8 total, 8 up, 8 in 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: failed to encode map e513 with expected crc 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:41:33.493 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:33 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/265572 objects degraded (0.940%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:33.494 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:34.129 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:34.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:34.225 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:41:34.237 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:34.321 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:34.515 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:34.526 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:41:34.526 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:41:34.526 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:41:34.526 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:41:34.527 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:41:34.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:34.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:34.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:34.681 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:34.941 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:41:34.976 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:41:35.011 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:41:35.047 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:35.051 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:41:35.053 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:35.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:35.101 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:41:35.139 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:41:35.157 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:35.183 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:41:35.205 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:35.224 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:41:35.235 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.235 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.235 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.236 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.236 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.236 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.236 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.236 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:41:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:35 smithi150 ceph-mon[194178]: pgmap v1587: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2497/259818 objects degraded (0.961%); 6183/259818 objects misplaced (2.380%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:41:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2280980184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:35.461 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:35.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:35 smithi107 ceph-mon[212750]: pgmap v1587: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2497/259818 objects degraded (0.961%); 6183/259818 objects misplaced (2.380%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:41:35.462 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2280980184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:35.462 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:35 smithi107 ceph-mon[220921]: pgmap v1587: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2497/259818 objects degraded (0.961%); 6183/259818 objects misplaced (2.380%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T06:41:35.463 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2280980184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:35.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:35.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:35.475 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:41:35.575 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:35.650 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:35.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:41:36.016 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:36.213 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:36 smithi107 ceph-mon[212750]: 14.12 scrub starts 2023-10-09T06:41:36.230 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:36.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:36.350 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:36.428 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:36.596 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:36 smithi150 ceph-mon[194178]: 14.12 scrub starts 2023-10-09T06:41:36.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:36 smithi107 ceph-mon[220921]: 14.12 scrub starts 2023-10-09T06:41:37.583 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:37 smithi150 ceph-mon[194178]: pgmap v1588: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:37.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:37 smithi107 ceph-mon[212750]: pgmap v1588: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:37.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:37 smithi107 ceph-mon[220921]: pgmap v1588: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:37.848 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:37.848 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:38.152 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:38.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:38.261 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:38.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:38 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/253161 objects degraded (0.986%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:38.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:38 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/253161 objects degraded (0.986%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:38.328 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:38.394 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:38.429 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:38.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:38.550 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:38.591 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:38 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/253161 objects degraded (0.986%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:38.601 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:38.751 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:38.755 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:41:38.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:38.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:38.859 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:38.957 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:39.097 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:39.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:39.206 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:39.238 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:39 smithi107 ceph-mon[212750]: pgmap v1589: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:39.278 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:39.289 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:39.402 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:39.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:39.503 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:39 smithi150 ceph-mon[194178]: pgmap v1589: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:39.511 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:39.568 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:39 smithi107 ceph-mon[220921]: pgmap v1589: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2497/253161 objects degraded (0.986%); 5592/253161 objects misplaced (2.209%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T06:41:39.572 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:39.666 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:39.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:39.770 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:39.799 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:39.953 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:39.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:40.024 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:41:40.059 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:40.085 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:40.127 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:40.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:40.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:40.231 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:40.277 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:40.285 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:41:40.313 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1264000566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.314 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:40 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1264000566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:40 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1264000566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:40 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:41:40.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:40.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:40.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:40.686 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:41.057 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:41.057 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:41.153 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:41.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:41.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:41.294 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:41:41.309 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:41.379 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:41.381 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: pgmap v1590: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 2497/245634 objects degraded (1.017%); 5592/245634 objects misplaced (2.277%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:41:41.381 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:41:41.381 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:41:41.381 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.382 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.382 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:41 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:41:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:41.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:41.483 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:41.515 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:41.565 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: pgmap v1590: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 2497/245634 objects degraded (1.017%); 5592/245634 objects misplaced (2.277%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: pgmap v1590: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 5.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 2497/245634 objects degraded (1.017%); 5592/245634 objects misplaced (2.277%); 1.5 MiB/s, 23 objects/s recovering 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: osdmap e514: 8 total, 8 up, 8 in 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: failed to encode map e514 with expected crc 2023-10-09T06:41:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:41 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:41:41.805 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:41.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:41.872 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:41:41.914 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:41.950 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:42.039 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:42.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:42.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:42.184 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:42] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:41:42.193 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:42.280 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:42.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:42.315 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:41:42.386 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:42.406 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:42.419 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:42.590 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:42 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:41:42.650 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3454584436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.650 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:42 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3454584436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3454584436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:42 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:41:42.743 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:42.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:42.849 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:42.891 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:42.986 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:42.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:43.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.135 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:43.185 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:43.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:43.288 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.337 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:43.598 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: pgmap v1592: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2497/240006 objects degraded (1.040%); 4968/240006 objects misplaced (2.070%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:41:43.599 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.600 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:43 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/240006 objects degraded (1.040%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:43.623 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:43.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: pgmap v1592: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2497/240006 objects degraded (1.040%); 4968/240006 objects misplaced (2.070%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:43.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/240006 objects degraded (1.040%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: pgmap v1592: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 2497/240006 objects degraded (1.040%); 4968/240006 objects misplaced (2.070%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:41:43.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: osdmap e515: 8 total, 8 up, 8 in 2023-10-09T06:41:43.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: failed to encode map e515 with expected crc 2023-10-09T06:41:43.653 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:43 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/240006 objects degraded (1.040%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:43.731 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:43.758 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:43.853 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:43.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:43.960 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:44.038 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:44.168 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:44.168 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:44.256 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:44.293 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:44 smithi107 ceph-mon[212750]: 14.12 scrub starts 2023-10-09T06:41:44.318 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:44.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:44.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:44.430 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:44.499 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:44.600 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:44.602 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:44 smithi107 ceph-mon[220921]: 14.12 scrub starts 2023-10-09T06:41:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:44.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:44 smithi150 ceph-mon[194178]: 14.12 scrub starts 2023-10-09T06:41:44.703 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:44.787 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:44.908 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:44.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:44.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:45.017 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:45.021 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:41:45.056 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:45.108 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:41:45.122 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:45.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:45.143 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:41:45.186 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:41:45.223 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:41:45.231 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:45.261 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:41:45.294 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:45.295 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:41:45.337 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:41:45.378 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:41:45.382 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.382 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.382 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.382 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.383 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.383 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.383 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.383 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:41:45.404 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:45.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:45.514 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:45.546 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:45.588 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:45 smithi107 ceph-mon[212750]: pgmap v1594: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2497/234312 objects degraded (1.066%); 4968/234312 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:45.588 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2615327741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:45.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:45 smithi107 ceph-mon[220921]: pgmap v1594: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2497/234312 objects degraded (1.066%); 4968/234312 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:45.588 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2615327741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:45.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:45 smithi150 ceph-mon[194178]: pgmap v1594: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 2497/234312 objects degraded (1.066%); 4968/234312 objects misplaced (2.120%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T06:41:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2615327741' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:45.698 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:45.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:45.805 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:45.837 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:45.932 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:45.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:46.035 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:46.081 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:46.217 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:46.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:46.280 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:41:46.322 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:46.355 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:46 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:41:46.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:46 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:41:46.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:46 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:41:47.004 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:47.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:47.107 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:47.166 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:47.176 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:47.249 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:47.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:47.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[220921]: pgmap v1595: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:47.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:47.307 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:47.308 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:41:47 smithi150 conmon[174357]: logger=cleanup t=2023-10-09T06:41:47.255052395Z level=info msg="Completed cleanup jobs" duration=107.148082ms 2023-10-09T06:41:47.311 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[212750]: pgmap v1595: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:47.311 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:47.311 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:47.351 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:41:47.351 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:41:47.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:41:47.351 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:41:47.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103714 12.11 12.d344f871 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.041199+0000", 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.307135722, 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98892410399999997, 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103714 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.352 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041199+0000", 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041199+0000", 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041197+0000", 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041204+0000", 2023-10-09T06:41:47.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0389999999999999e-06 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041218+0000", 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4195e-05 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041226+0000", 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6969999999999993e-06 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041277+0000", 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1245000000000001e-05 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.041283+0000", 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6400000000000002e-06 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030075+0000", 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98879225000000004 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030123+0000", 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8118999999999998e-05 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103725 12.11 12.23a68a71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.046690+0000", 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.30164490100003, 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98346997199999997, 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103725 2023-10-09T06:41:47.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046690+0000", 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046690+0000", 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046689+0000", 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046694+0000", 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7350000000000001e-06 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046706+0000", 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1919000000000001e-05 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046712+0000", 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3909999999999997e-06 2023-10-09T06:41:47.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046745+0000", 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2954000000000003e-05 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.046757+0000", 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2184999999999999e-05 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030148+0000", 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98339104700000002 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030160+0000", 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2055000000000001e-05 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103736 12.11 12.ec215111 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.054373+0000", 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.29396141899997, 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97582208999999998, 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103736 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.364 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054373+0000", 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054373+0000", 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054372+0000", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054378+0000", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3099999999999997e-06 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054394+0000", 2023-10-09T06:41:47.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6249999999999999e-05 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054401+0000", 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4869999999999999e-06 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054446+0000", 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5059999999999999e-05 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.054452+0000", 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3459999999999999e-06 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030182+0000", 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97572939700000005 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030195+0000", 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3828999999999999e-05 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.369 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103775 12.11 12.aabba71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.370 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.098153+0000", 2023-10-09T06:41:47.370 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.25018140099996, 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93208458900000002, 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103775 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098153+0000", 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098153+0000", 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098152+0000", 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098158+0000", 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7300000000000002e-06 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098171+0000", 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3074000000000001e-05 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098183+0000", 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1693e-05 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098229+0000", 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6006e-05 2023-10-09T06:41:47.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098236+0000", 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2239999999999998e-06 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030227+0000", 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93199147900000001 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030238+0000", 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0624e-05 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103779 12.11 12.fc888191 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.098193+0000", 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.250141665, 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93207581500000003, 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.377 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103779 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098193+0000", 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098193+0000", 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098193+0000", 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098196+0000", 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1810000000000001e-06 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098209+0000", 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2666e-05 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098213+0000", 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7040000000000002e-06 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098285+0000", 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1574000000000003e-05 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.098289+0000", 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5569999999999997e-06 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030261+0000", 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93197201900000004 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030269+0000", 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5179999999999998e-06 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.382 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103818 12.11 12.c6fa9d31 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.145356+0000", 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.20297880700002, 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88493910399999998, 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103818 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145356+0000", 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145356+0000", 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145355+0000", 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145361+0000", 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8270000000000003e-06 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145376+0000", 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.554e-05 2023-10-09T06:41:47.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145383+0000", 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8109999999999997e-06 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145426+0000", 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2740000000000001e-05 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.145433+0000", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8089999999999993e-06 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030290+0000", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.88485678400000001 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030295+0000", 2023-10-09T06:41:47.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8640000000000004e-06 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103851 12.11 12.ef62f471 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.175433+0000", 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.17290190100005, 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.85488401700000005, 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103851 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175433+0000", 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175433+0000", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175431+0000", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175440+0000", 2023-10-09T06:41:47.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3270000000000001e-06 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175458+0000", 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7923000000000001e-05 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175465+0000", 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6159999999999996e-06 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175513+0000", 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7706e-05 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.175520+0000", 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.092e-06 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030313+0000", 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.85479341099999995 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030317+0000", 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.596e-06 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103924 12.11 12.9d770751 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.393 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.246817+0000", 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.101517339, 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.78351480799999995, 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103924 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246817+0000", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246817+0000", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246816+0000", 2023-10-09T06:41:47.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246824+0000", 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1640000000000001e-06 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246844+0000", 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.012e-05 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246853+0000", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3999999999999992e-06 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246904+0000", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0762e-05 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.246912+0000", 2023-10-09T06:41:47.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1699999999999997e-06 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030329+0000", 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.783417119 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030332+0000", 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4470000000000002e-06 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.398 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:103956 12.11 12.ce66e151 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.278004+0000", 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 539.07033096199996, 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75234504899999999, 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.399 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 103956 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278004+0000", 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278004+0000", 2023-10-09T06:41:47.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278002+0000", 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278011+0000", 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7690000000000002e-06 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278029+0000", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7609e-05 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278036+0000", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7589999999999997e-06 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278086+0000", 2023-10-09T06:41:47.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9826e-05 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.278094+0000", 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0220000000000004e-06 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030344+0000", 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75224972499999998 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030349+0000", 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9590000000000003e-06 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:104052 12.11 12.f9d73e91 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.383155+0000", 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 538.96517944300001, 2023-10-09T06:41:47.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.64720793899999995, 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 104052 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383155+0000", 2023-10-09T06:41:47.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383155+0000", 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383154+0000", 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383159+0000", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5329999999999996e-06 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383170+0000", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1533e-05 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383175+0000", 2023-10-09T06:41:47.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5349999999999998e-06 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383219+0000", 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4270999999999997e-05 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.383227+0000", 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2989999999999997e-06 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030360+0000", 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.64713239600000005 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030363+0000", 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4379999999999999e-06 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:104144 12.11 12.3d7e9dd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:32:48.483740+0000", 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 538.86459469900001, 2023-10-09T06:41:47.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.54663833699999997, 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 104144 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483740+0000", 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483740+0000", 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483739+0000", 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483747+0000", 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9160000000000001e-06 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483763+0000", 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6707000000000001e-05 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483771+0000", 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6499999999999996e-06 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483828+0000", 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6946e-05 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:48.483833+0000", 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.339e-06 2023-10-09T06:41:47.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030375+0000", 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.54654139099999999 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:32:49.030378+0000", 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8689999999999997e-06 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.415 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197860 12.14 12.69738a74 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.62282073400002, 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93658382699999998, 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197860 2023-10-09T06:41:47.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725517+0000", 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2739999999999999e-06 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725526+0000", 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7979999999999995e-06 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725530+0000", 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4449999999999998e-06 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725576+0000", 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6060999999999998e-05 2023-10-09T06:41:47.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725583+0000", 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5089999999999999e-06 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662088+0000", 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93650507699999996 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662098+0000", 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0195e-05 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197873 12.14 12.b12e1f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.61095673800003, 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92474845500000002, 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.421 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197873 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737377+0000", 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737383+0000", 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6949999999999997e-06 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737394+0000", 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0998e-05 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737401+0000", 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9929999999999998e-06 2023-10-09T06:41:47.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737462+0000", 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1781999999999999e-05 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737471+0000", 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0180000000000007e-06 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662119+0000", 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92464809999999997 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662127+0000", 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0849999999999999e-06 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.426 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197996 12.14 12.b91ea374 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.45825203200002, 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77207506199999998, 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197996 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890081+0000", 2023-10-09T06:41:47.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890088+0000", 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0820000000000001e-06 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890106+0000", 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8141000000000001e-05 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890112+0000", 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0440000000000001e-06 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890164+0000", 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1412000000000002e-05 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890172+0000", 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0730000000000005e-06 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662151+0000", 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77197900699999999 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662158+0000", 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9009999999999997e-06 2023-10-09T06:41:47.431 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198013 12.14 12.e99f1f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.44298886199999, 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75683585900000006, 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198013 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:41:47.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905344+0000", 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905352+0000", 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8949999999999998e-06 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905369+0000", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6959000000000001e-05 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905378+0000", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1300000000000007e-06 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905442+0000", 2023-10-09T06:41:47.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4079999999999996e-05 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905453+0000", 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1319e-05 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662175+0000", 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75672163800000003 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662182+0000", 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6850000000000004e-06 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198020 12.14 12.481c23d4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:41:47.437 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.43803112299997, 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75190084000000001, 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198020 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.438 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:41:47.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910302+0000", 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910309+0000", 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4549999999999998e-06 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910325+0000", 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5356999999999999e-05 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910332+0000", 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3660000000000003e-06 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910385+0000", 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2348000000000001e-05 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910392+0000", 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1810000000000004e-06 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662199+0000", 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75180681100000002 2023-10-09T06:41:47.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662205+0000", 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0079999999999999e-06 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198050 12.14 12.2fad4614 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:41:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 403.39898662000002, 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71287966800000002, 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198050 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:41:47.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949346+0000", 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949357+0000", 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0203999999999999e-05 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949379+0000", 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2098000000000001e-05 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949388+0000", 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7950000000000002e-06 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949444+0000", 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5989999999999998e-05 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949454+0000", 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2890000000000003e-06 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662221+0000", 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71276758399999995 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662228+0000", 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5769999999999999e-06 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 291.205359638, 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.50336221199999998, 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:41:47.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143027+0000", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1832000000000001e-05 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143033+0000", 2023-10-09T06:41:47.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5590000000000002e-06 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143038+0000", 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7360000000000001e-06 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144824+0000", 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017859289999999999 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144963+0000", 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000139577 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145090+0000", 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012642299999999999 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148188+0000", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030985750000000001 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.149512+0000", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013238119999999999 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646328+0000", 2023-10-09T06:41:47.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49681566199999999 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646337+0000", 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5510000000000005e-06 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98785 13.18 13:1808f816:::benchmark_data_smithi107_535588_object98784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:41:47.455 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 291.00742089099998, 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89694675400000001, 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98785 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340912+0000", 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:41:47.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340963+0000", 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0998999999999997e-05 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340979+0000", 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5761999999999999e-05 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340986+0000", 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0770000000000002e-06 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545392+0000", 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20440555299999999 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545401+0000", 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0270000000000001e-06 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237781+0000", 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69237996599999996 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237861+0000", 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0128000000000003e-05 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:41:47.461 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 291.00733514900003, 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.87606833799999995, 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:41:47.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340999+0000", 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:41:47.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341044+0000", 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4756999999999999e-05 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341054+0000", 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7010000000000008e-06 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341058+0000", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.459e-06 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341171+0000", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011276 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341177+0000", 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0839999999999997e-06 2023-10-09T06:41:47.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647137+0000", 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30595975600000003 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647149+0000", 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2629000000000001e-05 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.216997+0000", 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.56984790500000004 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.217068+0000", 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0773999999999997e-05 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:41:47.467 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:41:47.468 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:41:47.468 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:47.504 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:47.579 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:47.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:47 smithi150 ceph-mon[194178]: pgmap v1595: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:47.581 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:41:47.582 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:41:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:47.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:47.684 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:47.732 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:47.925 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:47.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:48.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:48.098 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:48.179 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:41:48.183 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:48.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:48.301 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:48.376 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:48.471 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/228363 objects degraded (1.093%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2325052068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/228363 objects degraded (1.093%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2325052068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.478 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:48.580 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:48 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/228363 objects degraded (1.093%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:48 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2325052068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:41:48.657 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:48.680 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:41:48.706 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:48.953 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:41:49.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[212750]: pgmap v1596: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:49.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:41:49.602 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[212750]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:41:49.603 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[212750]: failed to encode map e516 with expected crc 2023-10-09T06:41:49.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[220921]: pgmap v1596: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:49.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:41:49.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[220921]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:41:49.603 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:49 smithi107 ceph-mon[220921]: failed to encode map e516 with expected crc 2023-10-09T06:41:49.648 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:49 smithi150 ceph-mon[194178]: pgmap v1596: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 2497/228363 objects degraded (1.093%); 4317/228363 objects misplaced (1.890%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:41:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:49 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:41:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:49 smithi150 ceph-mon[194178]: osdmap e516: 8 total, 8 up, 8 in 2023-10-09T06:41:49.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:49 smithi150 ceph-mon[194178]: failed to encode map e516 with expected crc 2023-10-09T06:41:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:49.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:49.758 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:49.785 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:41:49.829 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:50.038 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:50.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:50.148 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:50.190 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:50.378 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:50.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:50.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:50.393 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:50 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:41:50.433 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:41:50.433 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:41:50.434 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:41:50.434 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:41:50.485 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:50.531 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:50.582 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:50.655 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:50.657 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:50 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:41:50.658 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:50 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:41:50.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:50.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:50.759 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:50.821 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:50.876 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:50.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:50.980 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:51.050 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:51.302 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:41:51.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:51 smithi107 ceph-mon[212750]: pgmap v1598: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2497/221778 objects degraded (1.126%); 4317/221778 objects misplaced (1.947%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:41:51.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:51.364 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:51 smithi107 ceph-mon[220921]: pgmap v1598: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2497/221778 objects degraded (1.126%); 4317/221778 objects misplaced (1.947%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:41:51.364 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:51.365 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:41:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:51 smithi150 ceph-mon[194178]: pgmap v1598: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2497/221778 objects degraded (1.126%); 4317/221778 objects misplaced (1.947%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:41:51.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:41:51.797 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:52.374 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:52.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:52.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:41:52.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.377 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:41:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:41:52] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[212750]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:52 smithi107 ceph-mon[220921]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3864713116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: osdmap e517: 8 total, 8 up, 8 in 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: failed to encode map e517 with expected crc 2023-10-09T06:41:52.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:52 smithi150 ceph-mon[194178]: failed to encode map e517 with expected crc 2023-10-09T06:41:53.132 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:53.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:53.239 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:53.306 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:53.388 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:53 smithi107 ceph-mon[212750]: pgmap v1600: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2497/216783 objects degraded (1.152%); 3591/216783 objects misplaced (1.656%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:41:53.388 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:53 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/216783 objects degraded (1.152%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:53.610 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:41:53.611 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:41:53.611 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:41:53.611 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:41:53.611 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:41:53.631 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:53 smithi150 ceph-mon[194178]: pgmap v1600: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2497/216783 objects degraded (1.152%); 3591/216783 objects misplaced (1.656%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:41:53.631 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:53 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/216783 objects degraded (1.152%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:53.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:53 smithi107 ceph-mon[220921]: pgmap v1600: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2497/216783 objects degraded (1.152%); 3591/216783 objects misplaced (1.656%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:41:53.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:53 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/216783 objects degraded (1.152%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:41:53.933 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:53.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:54.038 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.109 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:54.221 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:54.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:54.336 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:41:54.356 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:41:54.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:54.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:41:54.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:54.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:41:54.564 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:54.626 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:54.677 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:41:54.691 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:54.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:54.719 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:41:54.752 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:41:54.793 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:41:54.801 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:54.827 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:41:54.872 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:41:54.890 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:54.892 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:54.912 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:41:54.938 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:41:54.959 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:41:54.960 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:41:55.027 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:55.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:55.137 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.221 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:55.341 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:55.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:55.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:55.439 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:41:55.451 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:55.493 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:55.537 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:55.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:55.640 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:55.663 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[212750]: pgmap v1601: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2497/211518 objects degraded (1.181%); 3591/211518 objects misplaced (1.698%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:55.663 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2739564594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:55.664 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:41:55.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[220921]: pgmap v1601: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2497/211518 objects degraded (1.181%); 3591/211518 objects misplaced (1.698%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:55.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2739564594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:55.664 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:41:55.666 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:55.708 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:55.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:55.814 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:41:55.862 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:55 smithi150 ceph-mon[194178]: pgmap v1601: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 4.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 2497/211518 objects degraded (1.181%); 3591/211518 objects misplaced (1.698%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:41:55.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2739564594' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:41:55.863 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:41:55.869 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:55.948 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:41:56.121 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:41:56.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:56.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:41:56.229 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:56.354 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[220921]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:56 smithi150 ceph-mon[194178]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1166506816' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:41:56.655 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: osdmap e518: 8 total, 8 up, 8 in 2023-10-09T06:41:56.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.656 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:56 smithi107 ceph-mon[212750]: failed to encode map e518 with expected crc 2023-10-09T06:41:56.666 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:56.666 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:56.931 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:56.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:57.039 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:41:57.046 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:41:57.065 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:57.115 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:41:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:57.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:41:57.137 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:57.221 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:41:57.298 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:57.455 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:41:57.456 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:41:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:57.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:41:57.550 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:57 smithi107 ceph-mon[212750]: pgmap v1603: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:41:57.550 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:57 smithi107 ceph-mon[220921]: pgmap v1603: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:41:57.564 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:41:57.603 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:57.676 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:41:57.680 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:57 smithi150 ceph-mon[194178]: pgmap v1603: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T06:41:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:57.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:41:57.781 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:41:57.832 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:57.938 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:41:58.326 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: pgmap v1605: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:41:58.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 2497/205035 objects degraded (1.218%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: pgmap v1605: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:41:58.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:58 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 2497/205035 objects degraded (1.218%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:41:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: osdmap e519: 8 total, 8 up, 8 in 2023-10-09T06:41:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: failed to encode map e519 with expected crc 2023-10-09T06:41:58.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: pgmap v1605: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 4.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2497/205035 objects degraded (1.218%); 2749/205035 objects misplaced (1.341%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T06:41:58.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:58 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 2497/205035 objects degraded (1.218%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T06:41:58.905 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:41:58.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:58.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:41:59.012 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:59.060 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:59.141 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:59.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:59.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:41:59.296 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:59.611 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:41:59.611 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:41:59.630 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:41:59.632 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:59 smithi150 ceph-mon[194178]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:59 smithi150 ceph-mon[194178]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:59 smithi150 ceph-mon[194178]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:41:59.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:59 smithi150 ceph-mon[194178]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.633 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:41:59 smithi150 ceph-mon[194178]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.633 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[212750]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.635 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[212750]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:59.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:41:59.643 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[220921]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.643 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[220921]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.643 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[220921]: osdmap e520: 8 total, 8 up, 8 in 2023-10-09T06:41:59.643 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[220921]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.643 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:41:59 smithi107 ceph-mon[220921]: failed to encode map e520 with expected crc 2023-10-09T06:41:59.739 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:41:59.783 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:41:59.865 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:41:59.889 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:41:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:41:59.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:41:59.993 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:00.022 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:00.237 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:00.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:00.348 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:00.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:00.436 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:42:00.467 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:00.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:00.574 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:00.618 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:00.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:00 smithi107 ceph-mon[220921]: pgmap v1607: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 4.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/198108 objects degraded (0.538%); 3772/198108 objects misplaced (1.904%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:42:00.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:00 smithi107 ceph-mon[212750]: pgmap v1607: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 4.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/198108 objects degraded (0.538%); 3772/198108 objects misplaced (1.904%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:42:00.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:00 smithi150 ceph-mon[194178]: pgmap v1607: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 4.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/198108 objects degraded (0.538%); 3772/198108 objects misplaced (1.904%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T06:42:00.707 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:00.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:00.811 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:00.896 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:00.972 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:01.487 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:42:01.487 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:01.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:01.599 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:01.654 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:01.706 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:01.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:01.810 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:01.834 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:01 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/688018031' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.834 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:01 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.836 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:01.870 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:01 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/688018031' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.870 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:01 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.870 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:01 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/688018031' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.870 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:01 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:01.876 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:01.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:01.979 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:02.053 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:02.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:02] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:42:02.722 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:42:02.722 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:42:02.746 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: pgmap v1609: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 3.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1066/193659 objects degraded (0.550%); 3772/193659 objects misplaced (1.948%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:02.747 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: pgmap v1609: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 3.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1066/193659 objects degraded (0.550%); 3772/193659 objects misplaced (1.948%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:42:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: osdmap e521: 8 total, 8 up, 8 in 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: failed to encode map e521 with expected crc 2023-10-09T06:42:02.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: pgmap v1609: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 127 active+clean; 3.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1066/193659 objects degraded (0.550%); 3772/193659 objects misplaced (1.948%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T06:42:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:02.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:03.032 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:03.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:03.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:03.137 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:03.163 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:03.479 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:03.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:03.535 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:42:03.589 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:03.650 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:03 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1066/193659 objects degraded (0.550%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:03.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:03 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1066/193659 objects degraded (0.550%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:03.716 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:03.881 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:03.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:03.911 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:03 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1066/193659 objects degraded (0.550%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:03.989 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:04.026 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:42:04.092 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:04.216 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:04.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:04.327 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:04.389 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:04.504 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:04.504 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:42:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:04.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:04.552 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:42:04.595 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:42:04.616 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:04.642 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:42:04.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:04 smithi107 ceph-mon[212750]: pgmap v1610: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 1066/188061 objects degraded (0.567%); 3328/188061 objects misplaced (1.770%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:42:04.652 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4075586427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:04.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:04 smithi107 ceph-mon[220921]: pgmap v1610: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 1066/188061 objects degraded (0.567%); 3328/188061 objects misplaced (1.770%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:42:04.653 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4075586427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:04.681 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:42:04.690 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:04.716 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:42:04.749 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:04.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:04.852 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:04.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:04 smithi150 ceph-mon[194178]: pgmap v1610: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 1066/188061 objects degraded (0.567%); 3328/188061 objects misplaced (1.770%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T06:42:04.888 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4075586427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:04.893 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:04.907 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:42:04.944 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:42:04.963 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:42:04.964 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:42:05.104 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:05.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:05.210 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:05.245 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:05.338 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:05.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:05.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:05.443 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:05.495 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:05.549 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:42:05.558 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:05.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:05.666 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:05.692 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:42:05.741 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:05.800 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2041622226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:05.800 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:05 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:05.803 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2041622226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:05.806 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:05 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180927 12.d 12:b6ff457f:::benchmark_data_smithi107_479504_object81316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 447.06341713299997, 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.35203043499999997, 2023-10-09T06:42:05.815 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180927 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.816 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748411+0000", 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.817 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748415+0000", 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6779999999999999e-06 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748424+0000", 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.818 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748429+0000", 2023-10-09T06:42:05.819 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3270000000000002e-06 2023-10-09T06:42:05.819 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.819 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.820 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.820 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748471+0000", 2023-10-09T06:42:05.820 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2400000000000001e-05 2023-10-09T06:42:05.820 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748563+0000", 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2088000000000003e-05 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748643+0000", 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9326000000000006e-05 2023-10-09T06:42:05.821 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.750567+0000", 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019239070000000001 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.822 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.751954+0000", 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013873869999999999 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100346+0000", 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.34839218799999999 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100401+0000", 2023-10-09T06:42:05.823 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4484999999999998e-05 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100442+0000", 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1430000000000001e-05 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180945 12.d 12:b29197fc:::benchmark_data_smithi107_479504_object81334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:42:05.824 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 447.03246858199998, 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32532718199999999, 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180945 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.825 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779359+0000", 2023-10-09T06:42:05.826 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779366+0000", 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8349999999999998e-06 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779380+0000", 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3978999999999999e-05 2023-10-09T06:42:05.827 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779386+0000", 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2960000000000004e-06 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779426+0000", 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0438000000000001e-05 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.828 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779510+0000", 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3070000000000003e-05 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779582+0000", 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2076999999999994e-05 2023-10-09T06:42:05.829 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.782438+0000", 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002856408 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.786572+0000", 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041337539999999999 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.830 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104626+0000", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31805433399999999 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104660+0000", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3751000000000003e-05 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104687+0000", 2023-10-09T06:42:05.831 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7487999999999998e-05 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98783 13.e 13:7e44f156:::benchmark_data_smithi107_535588_object98782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 309.67291971200001, 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23612880999999999, 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.832 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98783 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:42:05.833 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138907+0000", 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:05.834 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138985+0000", 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8372000000000001e-05 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138998+0000", 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.254e-05 2023-10-09T06:42:05.835 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139004+0000", 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1290000000000001e-06 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139052+0000", 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8241000000000001e-05 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.836 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139170+0000", 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117379 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139302+0000", 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013261800000000001 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.837 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145027+0000", 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057249759999999997 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148343+0000", 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033158129999999999 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374933+0000", 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22659017100000001 2023-10-09T06:42:05.838 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374992+0000", 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8931000000000002e-05 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.375038+0000", 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5528000000000002e-05 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.839 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 309.67280559599999, 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30779935400000002, 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.840 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:42:05.841 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139054+0000", 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000001e-05 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.842 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139062+0000", 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139066+0000", 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.843 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140814+0000", 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001748793 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140929+0000", 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.141055+0000", 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126563 2023-10-09T06:42:05.844 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.146811+0000", 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005755418 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446811+0000", 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29999972899999999 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.845 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446822+0000", 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1856e-05 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:42:05.846 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 309.55736327099999, 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93419914299999995, 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.847 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254463+0000", 2023-10-09T06:42:05.848 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254519+0000", 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6208999999999997e-05 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254547+0000", 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8109000000000001e-05 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.849 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254562+0000", 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5073e-05 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254724+0000", 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161974 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:42:05.850 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254729+0000", 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497899+0000", 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24316990899999999 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497910+0000", 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0874e-05 2023-10-09T06:42:05.851 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188593+0000", 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69068296699999998 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188665+0000", 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1802000000000001e-05 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.852 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98796 13.7 13:e44dc813:::benchmark_data_smithi107_535588_object98795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 309.45571366399997, 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84800129700000004, 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98796 2023-10-09T06:42:05.853 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.854 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356113+0000", 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4719000000000002e-05 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.855 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356184+0000", 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6277e-05 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356192+0000", 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9789999999999993e-06 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.856 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498051+0000", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14185953700000001 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498058+0000", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2799999999999998e-06 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188683+0000", 2023-10-09T06:42:05.857 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.690624601 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188792+0000", 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108497 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188939+0000", 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014728699999999999 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.858 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.191074+0000", 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021353840000000002 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.203068+0000", 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011993634 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.859 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204019+0000", 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000951625 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204076+0000", 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6569e-05 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204116+0000", 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0346999999999998e-05 2023-10-09T06:42:05.860 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98805 13.0 13:054831b9:::benchmark_data_smithi107_535588_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 309.12560972, 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.528813278, 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.861 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98805 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.862 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686214+0000", 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999952 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686248+0000", 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4001000000000002e-05 2023-10-09T06:42:05.863 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686269+0000", 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0539000000000001e-05 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686278+0000", 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2469999999999997e-06 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.864 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686339+0000", 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0735e-05 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686348+0000", 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181836+0000", 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49548772400000002 2023-10-09T06:42:05.865 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181965+0000", 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000129448 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.182343+0000", 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037732500000000002 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.866 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.184651+0000", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023081030000000002 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.199956+0000", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015305375 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214937+0000", 2023-10-09T06:42:05.867 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014981213 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214993+0000", 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5884999999999998e-05 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.215032+0000", 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9060999999999998e-05 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.868 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207733 13.0 13:0665e03e:::benchmark_data_smithi107_535588_object101626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 179.09622727799999, 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26422807999999998, 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.869 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207733 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:42:05.870 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715600+0000", 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715606+0000", 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.871 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715622+0000", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6038000000000001e-05 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715630+0000", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2320000000000004e-06 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715686+0000", 2023-10-09T06:42:05.872 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6475999999999999e-05 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715825+0000", 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013932800000000001 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716377+0000", 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055127400000000003 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.873 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.722227+0000", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0058503299999999999 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.964701+0000", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24247376800000001 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979740+0000", 2023-10-09T06:42:05.874 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015039132 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979785+0000", 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5225999999999997e-05 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979830+0000", 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4431000000000002e-05 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.875 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211410 13.7 13:e70c994c:::benchmark_data_smithi107_535588_object105303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.14238250400001, 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267478086, 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211410 2023-10-09T06:42:05.876 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.877 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669444+0000", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669454+0000", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8539999999999992e-06 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669466+0000", 2023-10-09T06:42:05.878 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1838000000000001e-05 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669472+0000", 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0059999999999998e-06 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669508+0000", 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.879 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669585+0000", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7124000000000002e-05 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670141+0000", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055599200000000005 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.674991+0000", 2023-10-09T06:42:05.880 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048495869999999998 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936590+0000", 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159882800000001 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936837+0000", 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024747099999999998 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.881 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936885+0000", 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8071000000000001e-05 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936924+0000", 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8995000000000002e-05 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.882 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211418 13.17 13:e9af5116:::benchmark_data_smithi107_535588_object105311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.13942352999999, 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.264762895, 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211418 2023-10-09T06:42:05.883 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.884 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672404+0000", 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672414+0000", 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0522e-05 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.885 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672428+0000", 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3747999999999999e-05 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672435+0000", 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.878e-06 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672490+0000", 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5414999999999997e-05 2023-10-09T06:42:05.886 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672574+0000", 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4127000000000003e-05 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673020+0000", 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044588000000000001 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.887 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676720+0000", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036994749999999998 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936560+0000", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25984011699999998 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937086+0000", 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000526558 2023-10-09T06:42:05.888 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937128+0000", 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1659999999999998e-05 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937168+0000", 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9950999999999997e-05 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.889 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211420 13.17 13:ef772928:::benchmark_data_smithi107_535588_object105313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.138587591, 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265498495, 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:42:05.890 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211420 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.891 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673244+0000", 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7500000000000001e-06 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.892 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673253+0000", 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2239999999999995e-06 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673257+0000", 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4170000000000002e-06 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.893 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675010+0000", 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017534130000000001 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675083+0000", 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2701999999999996e-05 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.894 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675377+0000", 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029354000000000001 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676887+0000", 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015100999999999999 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.895 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936617+0000", 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.259729969 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938661+0000", 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020442260000000001 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938704+0000", 2023-10-09T06:42:05.896 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3467000000000002e-05 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938740+0000", 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5256999999999998e-05 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.897 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:6728 14.f 14:fd78970a:::benchmark_data_smithi107_590976_object6727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 137.959967119, 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24803666199999999, 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6728 2023-10-09T06:42:05.898 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.899 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851861+0000", 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851941+0000", 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0459999999999999e-05 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.900 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851954+0000", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605999999999999e-05 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851959+0000", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8440000000000003e-06 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852021+0000", 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1607999999999997e-05 2023-10-09T06:42:05.901 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852156+0000", 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135234 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852317+0000", 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016078500000000001 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.902 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852928+0000", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00061110299999999999 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.854096+0000", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011680460000000001 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099820+0000", 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24572355200000001 2023-10-09T06:42:05.903 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099856+0000", 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6050999999999998e-05 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099898+0000", 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2388000000000001e-05 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.904 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:53446 14.f 14:fdbc8938:::benchmark_data_smithi107_590976_object53445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 105.169918989, 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24989693599999999, 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.905 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 53446 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.906 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641908+0000", 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642000+0000", 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2066999999999998e-05 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.907 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642019+0000", 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8488999999999999e-05 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642027+0000", 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2989999999999997e-06 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642077+0000", 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0349999999999997e-05 2023-10-09T06:42:05.908 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642204+0000", 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126422 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642347+0000", 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142903 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.909 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.644067+0000", 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017200099999999999 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.889696+0000", 2023-10-09T06:42:05.910 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245629034 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891721+0000", 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020251079999999999 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.911 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891762+0000", 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1298000000000003e-05 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.912 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891807+0000", 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.46e-05 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.913 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93215 14.f 14:fff2cbd4:::benchmark_data_smithi107_590976_object93214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 59.765349692999997, 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247730006, 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.914 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93215 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.915 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.916 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046477+0000", 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046588+0000", 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110818 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046609+0000", 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0880999999999999e-05 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046617+0000", 2023-10-09T06:42:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9980000000000003e-06 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046668+0000", 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0525000000000002e-05 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046819+0000", 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015156699999999999 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.919 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046943+0000", 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000123709 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050044+0000", 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031011939999999998 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.920 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.291715+0000", 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24167045500000001 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294145+0000", 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024305669999999998 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294182+0000", 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6606999999999997e-05 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.921 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294209+0000", 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7245000000000001e-05 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.922 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93221 14.f 14:f8a7a189:::benchmark_data_smithi107_590976_object93220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 59.761356532999997, 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245878864, 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.923 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93221 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.924 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.925 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050471+0000", 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.926 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050529+0000", 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7805e-05 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050544+0000", 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5617999999999999e-05 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.927 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050550+0000", 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8390000000000003e-06 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050596+0000", 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5843000000000001e-05 2023-10-09T06:42:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050742+0000", 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014604700000000001 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050897+0000", 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015521900000000001 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.929 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.053948+0000", 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030502860000000001 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.293048+0000", 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.239100441 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.930 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296266+0000", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032179029999999998 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296321+0000", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5438e-05 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296351+0000", 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9584999999999999e-05 2023-10-09T06:42:05.931 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114305 14.10 e511/504 14:0e814216:::benchmark_data_smithi107_590976_object13532:head v 511'5874, mlcod=495'2568)", 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 35.720403226000002, 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26106436700000002, 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.932 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.933 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091424+0000", 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091437+0000", 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3426e-05 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091458+0000", 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.039e-05 2023-10-09T06:42:05.934 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091468+0000", 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0360000000000001e-05 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091572+0000", 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103453 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.935 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091599+0000", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7172999999999999e-05 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352452+0000", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26085362600000001 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352490+0000", 2023-10-09T06:42:05.936 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7518999999999998e-05 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114306 14.10 e511/504 14:0c753e0c:::benchmark_data_smithi107_590976_object13533:head v 511'5875, mlcod=495'2568)", 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 35.720329176999996, 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26101210699999999, 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.937 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.938 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091499+0000", 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091507+0000", 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5369999999999997e-06 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.939 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091524+0000", 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6308999999999999e-05 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091533+0000", 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4569999999999994e-06 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.940 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345321+0000", 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25378754399999998 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345349+0000", 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7840000000000001e-05 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352493+0000", 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071447630000000002 2023-10-09T06:42:05.941 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352512+0000", 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8321e-05 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:114323 14.f 14:f062eb52:::benchmark_data_smithi107_590976_object13550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:42:05.942 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 35.698293200000002, 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24122175000000001, 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 114323 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.943 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113533+0000", 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:05.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113543+0000", 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2560000000000008e-06 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.342992+0000", 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22944947199999999 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343004+0000", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1386000000000001e-05 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343062+0000", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8409000000000003e-05 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343203+0000", 2023-10-09T06:42:05.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014109999999999999 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343349+0000", 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000146308 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.348700+0000", 2023-10-09T06:42:05.947 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053506509999999997 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.351340+0000", 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026400450000000002 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354664+0000", 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033237610000000002 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.948 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354715+0000", 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1208000000000002e-05 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354757+0000", 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2179999999999999e-05 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.949 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:129858 14.f 14:fe543c35:::benchmark_data_smithi107_590976_object29085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:49.385395+0000", 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 16.426433529000001, 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16614863999999999, 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.950 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 129858 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385395+0000", 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385395+0000", 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385394+0000", 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385400+0000", 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4949999999999999e-06 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385409+0000", 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4040000000000001e-06 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385413+0000", 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2130000000000002e-06 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385447+0000", 2023-10-09T06:42:05.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3899000000000002e-05 2023-10-09T06:42:05.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.954 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.954 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385521+0000", 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3558000000000003e-05 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.385589+0000", 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8307999999999997e-05 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.390068+0000", 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0044790510000000004 2023-10-09T06:42:05.955 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.392164+0000", 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020959569999999999 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.394446+0000", 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022817290000000001 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.956 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.551496+0000", 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.15705002700000001 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.551544+0000", 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7970999999999999e-05 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:129860 14.f 14:f2cec47b:::benchmark_data_smithi107_590976_object29087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e516)", 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:49.386457+0000", 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 16.425371787, 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.16525812400000001, 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 129860 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.958 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386457+0000", 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386457+0000", 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.959 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386456+0000", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386460+0000", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9870000000000001e-06 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386467+0000", 2023-10-09T06:42:05.960 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4459999999999998e-06 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.386470+0000", 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3119999999999998e-06 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.387631+0000", 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001161067 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.961 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.387695+0000", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3842000000000003e-05 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.387758+0000", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2586e-05 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.392139+0000", 2023-10-09T06:42:05.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043813630000000001 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.393697+0000", 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001558369 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.551637+0000", 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.157939736 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.551682+0000", 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4539000000000002e-05 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:49.551715+0000", 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3481e-05 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.964 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:42:05.965 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:42:05.965 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:42:05.965 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2041622226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:05.965 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:05 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:42:06.643 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[212750]: pgmap v1612: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:06 smithi107 ceph-mon[220921]: pgmap v1612: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: osdmap e522: 8 total, 8 up, 8 in 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: failed to encode map e522 with expected crc 2023-10-09T06:42:06.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:06 smithi150 ceph-mon[194178]: pgmap v1612: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T06:42:07.194 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:07.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:07.305 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:07.335 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:07.381 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:07.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:07.487 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:07.529 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:07.583 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:07.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:07.690 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:07.759 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:07 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1066/182316 objects degraded (0.585%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:07.925 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:07 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1066/182316 objects degraded (0.585%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:07.925 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:07 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1066/182316 objects degraded (0.585%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:07.965 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:42:07.965 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:42:07.993 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:08.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:08.076 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:08.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:08.139 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:08.198 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:08.203 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:08.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:08.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:08.315 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:08.400 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:08.535 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:08.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:08.643 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:08.761 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:08.968 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:08.973 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:08 smithi107 ceph-mon[220921]: pgmap v1613: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:42:08.974 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:08 smithi107 ceph-mon[212750]: pgmap v1613: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:42:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:08.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:09.089 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:09.121 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:08 smithi150 ceph-mon[194178]: pgmap v1613: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 1066/182316 objects degraded (0.585%); 3328/182316 objects misplaced (1.825%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T06:42:09.125 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:09.211 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:09.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:09.318 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:09.395 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:09.684 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:42:10.752 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:10.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:10.863 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:10.941 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:11.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:10 smithi107 ceph-mon[212750]: pgmap v1614: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/175452 objects degraded (0.608%); 2850/175452 objects misplaced (1.624%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:42:11.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:10 smithi107 ceph-mon[220921]: pgmap v1614: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/175452 objects degraded (0.608%); 2850/175452 objects misplaced (1.624%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:42:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:10 smithi150 ceph-mon[194178]: pgmap v1614: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 1066/175452 objects degraded (0.608%); 2850/175452 objects misplaced (1.624%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T06:42:11.959 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:11.976 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:11.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:12.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:12.170 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:12.209 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:12] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:42:12.334 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:12.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:12.447 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:12.488 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:12.510 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:12.539 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:12.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:12.645 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:12.733 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:12.847 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:12.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:12 smithi107 ceph-mon[212750]: pgmap v1615: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1066/170229 objects degraded (0.626%); 2850/170229 objects misplaced (1.674%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:42:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:12 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1066/170229 objects degraded (0.626%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:12.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:12 smithi107 ceph-mon[220921]: pgmap v1615: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1066/170229 objects degraded (0.626%); 2850/170229 objects misplaced (1.674%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:42:12.903 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:12 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1066/170229 objects degraded (0.626%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:12 smithi150 ceph-mon[194178]: pgmap v1615: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 1066/170229 objects degraded (0.626%); 2850/170229 objects misplaced (1.674%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T06:42:12.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:12 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1066/170229 objects degraded (0.626%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:12.955 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:12.987 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:13.047 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:13.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:13.154 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:13.238 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:13.426 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:148727 14.b 14:d69d98c4:::benchmark_data_smithi107_590976_object47954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:13.420390+0000", 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0057728470000000002, 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058039850000000002, 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:42:13.427 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 148727 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420390+0000", 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.428 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420390+0000", 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420389+0000", 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:13.429 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420396+0000", 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420406+0000", 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0589e-05 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420414+0000", 2023-10-09T06:42:13.430 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4989999999999997e-06 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.425881+0000", 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054670819999999998 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.425954+0000", 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3497000000000005e-05 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.431 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.426049+0000", 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4853999999999997e-05 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:148730 14.16 14:6f63cbdf:::benchmark_data_smithi107_590976_object47957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:13.420426+0000", 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0057370620000000002, 2023-10-09T06:42:13.432 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058636490000000003, 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 148730 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420426+0000", 2023-10-09T06:42:13.433 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420426+0000", 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420426+0000", 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:13.434 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420428+0000", 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7800000000000001e-06 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420435+0000", 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2110000000000003e-06 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.435 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420440+0000", 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1950000000000002e-06 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420478+0000", 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7871999999999999e-05 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:13.436 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420614+0000", 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000136024 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.421091+0000", 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00047759300000000001 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.426265+0000", 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0051740370000000003 2023-10-09T06:42:13.437 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.426273+0000", 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7950000000000008e-06 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:148731 14.13 14.c6662fd3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:42:13.438 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:13.420457+0000", 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0057058330000000004, 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059166679999999999, 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "queued for pg", 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 148731 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.439 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420457+0000", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420457+0000", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420457+0000", 2023-10-09T06:42:13.440 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420460+0000", 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.258e-06 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420467+0000", 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0790000000000003e-06 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.441 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420473+0000", 2023-10-09T06:42:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7180000000000001e-06 2023-10-09T06:42:13.442 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.444 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:148736 14.0 14:0451bfb9:::benchmark_data_smithi107_590976_object47963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:42:13.445 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:13.420923+0000", 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 0.0052400190000000003, 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055080939999999998, 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:13.446 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 148736 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420923+0000", 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:13.447 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420923+0000", 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420922+0000", 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420928+0000", 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2179999999999998e-06 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.448 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420937+0000", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8579999999999995e-06 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.420944+0000", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4930000000000003e-06 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.421000+0000", 2023-10-09T06:42:13.449 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6192e-05 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.421099+0000", 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8658000000000003e-05 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.421566+0000", 2023-10-09T06:42:13.450 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00046765199999999999 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.426228+0000", 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0046612210000000001 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:13.426260+0000", 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2651999999999998e-05 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:13.451 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: ], 2023-10-09T06:42:13.452 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 4 2023-10-09T06:42:13.452 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:42:13.728 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:42:14.104 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:14.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:14.109 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:42:14.111 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:42:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:42:14.209 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:14.280 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:15.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: pgmap v1616: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 1066/164028 objects degraded (0.650%); 2344/164028 objects misplaced (1.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:15.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:15.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[212750]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.030 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: pgmap v1616: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 1066/164028 objects degraded (0.650%); 2344/164028 objects misplaced (1.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:15.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:15.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:42:15.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:42:15.031 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.032 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.032 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.032 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.032 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:14 smithi107 ceph-mon[220921]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: pgmap v1616: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 1066/164028 objects degraded (0.650%); 2344/164028 objects misplaced (1.429%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3445176806' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:42:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: osdmap e523: 8 total, 8 up, 8 in 2023-10-09T06:42:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:14 smithi150 ceph-mon[194178]: failed to encode map e523 with expected crc 2023-10-09T06:42:15.648 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:15.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:15.756 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:15.830 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:16.140 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:15 smithi107 ceph-mon[212750]: 14.3 scrub starts 2023-10-09T06:42:16.141 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:15 smithi107 ceph-mon[220921]: 14.3 scrub starts 2023-10-09T06:42:16.142 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:42:16.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:15 smithi150 ceph-mon[194178]: 14.3 scrub starts 2023-10-09T06:42:16.326 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:16.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:16.449 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:16.537 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:16.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:16 smithi107 ceph-mon[212750]: pgmap v1618: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:16.805 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:16.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:16 smithi107 ceph-mon[220921]: pgmap v1618: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:16.805 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:16.964 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:16.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:17.069 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:16 smithi150 ceph-mon[194178]: pgmap v1618: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:17.104 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:17.110 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:17.196 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:17.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:17.306 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:17.347 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:17.426 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:17.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:17.535 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:17.572 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:17.657 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:17.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:17.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:17.765 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:17.812 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:17.877 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:17.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:17.983 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:18.001 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:17 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1066/157839 objects degraded (0.675%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:18.003 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:17 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1066/157839 objects degraded (0.675%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:18.086 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:17 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1066/157839 objects degraded (0.675%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:18.186 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:42:18.261 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:18.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:18.374 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:18.406 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:18.410 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:18.468 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:18.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:18.573 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:18.692 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:18.706 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:42:18.731 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:42:18.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:18 smithi107 ceph-mon[212750]: pgmap v1619: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:18.860 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:18 smithi107 ceph-mon[212750]: 14.6 scrub starts 2023-10-09T06:42:18.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:18 smithi107 ceph-mon[220921]: pgmap v1619: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:18.860 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:18 smithi107 ceph-mon[220921]: 14.6 scrub starts 2023-10-09T06:42:18.992 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:19.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:19.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:19.112 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:18 smithi150 ceph-mon[194178]: pgmap v1619: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 1066/157839 objects degraded (0.675%); 2344/157839 objects misplaced (1.485%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T06:42:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:18 smithi150 ceph-mon[194178]: 14.6 scrub starts 2023-10-09T06:42:19.166 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:19.253 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:19.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:19.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:19.360 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:19.372 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:42:19.372 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.373 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.374 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.374 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.374 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.374 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:42:19.444 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:20.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:19 smithi107 ceph-mon[212750]: from='client.47234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:20.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:19 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/645851118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:19 smithi107 ceph-mon[220921]: from='client.47234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:20.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:19 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/645851118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:19 smithi150 ceph-mon[194178]: from='client.47234 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:19 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/645851118' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:21.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:20 smithi107 ceph-mon[220921]: pgmap v1620: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1066/150366 objects degraded (0.709%); 1680/150366 objects misplaced (1.117%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:21.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:20 smithi107 ceph-mon[212750]: pgmap v1620: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1066/150366 objects degraded (0.709%); 1680/150366 objects misplaced (1.117%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:20 smithi150 ceph-mon[194178]: pgmap v1620: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 1066/150366 objects degraded (0.709%); 1680/150366 objects misplaced (1.117%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:21.325 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:21.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:21.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:21.488 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:21.540 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:21.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:21.647 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:21.677 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:21.742 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:21.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:21.847 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:21.911 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:22.016 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:22] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-10-09T06:42:22.179 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1619,"stamp":"2023-10-09T06:42:17.617987+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3447003064,"num_objects":52613,"num_object_clones":0,"num_object_copies":157839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1066,"num_objects_misplaced":2344,"num_objects_unfound":0,"num_objects_dirty":52613,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131969,"num_write_kb":5352990,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3377,"num_bytes_recovered":217786316,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":179745,"ondisk_log_size":179745,"up":387,"acting":386,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20522532,"kb_used_data":10767652,"kb_used_omap":20,"kb_used_meta":9754603,"kb_avail":729438684,"statfs":{"total":767960285184,"available":746945212416,"internally_reserved":0,"allocated":11026075648,"data_stored":11012836003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9988714126},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,4,3,0,0,0,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-534708224,"num_objects":-8159,"num_object_clones":0,"num_object_copies":-24477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-984,"num_objects_unfound":0,"num_objects_dirty":-8159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8159,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32243712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007035"},"pg_stats":[{"pgid":"14.17","version":"522'6813","reported_seq":6981,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.520988+0000","last_change":"2023-10-09T06:40:52.372943+0000","last_active":"2023-10-09T06:42:12.520988+0000","last_peered":"2023-10-09T06:42:12.520988+0000","last_clean":"2023-10-09T06:42:12.520988+0000","last_became_active":"2023-10-09T06:40:43.185643+0000","last_became_peered":"2023-10-09T06:40:43.185643+0000","last_unstale":"2023-10-09T06:42:12.520988+0000","last_undegraded":"2023-10-09T06:42:12.520988+0000","last_fullsized":"2023-10-09T06:42:12.520988+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4783","last_scrub_stamp":"2023-10-09T06:40:52.372858+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:52.372858+0000","objects_scrubbed":2511,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:22.222358+0000","scrub_duration":0.27153366200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2030,"num_write_kb":35712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":816,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.514856+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:42:01.514856+0000","last_peered":"2023-10-09T06:42:01.514856+0000","last_clean":"2023-10-09T06:42:01.514856+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:42:01.514856+0000","last_undegraded":"2023-10-09T06:42:01.514856+0000","last_fullsized":"2023-10-09T06:42:01.514856+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":501,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470519+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:42:01.470519+0000","last_peered":"2023-10-09T06:42:01.470519+0000","last_clean":"2023-10-09T06:42:01.470519+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:42:01.470519+0000","last_undegraded":"2023-10-09T06:42:01.470519+0000","last_fullsized":"2023-10-09T06:42:01.470519+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":225,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521842+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:42:05.521842+0000","last_peered":"2023-10-09T06:42:05.521842+0000","last_clean":"2023-10-09T06:42:05.521842+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:42:05.521842+0000","last_undegraded":"2023-10-09T06:42:05.521842+0000","last_fullsized":"2023-10-09T06:42:05.521842+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"522'6756","reported_seq":6822,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.842512+0000","last_change":"2023-10-09T06:41:49.571613+0000","last_active":"2023-10-09T06:42:12.842512+0000","last_peered":"2023-10-09T06:42:12.842512+0000","last_clean":"2023-10-09T06:42:12.842512+0000","last_became_active":"2023-10-09T06:40:43.089707+0000","last_became_peered":"2023-10-09T06:40:43.089707+0000","last_unstale":"2023-10-09T06:42:12.842512+0000","last_undegraded":"2023-10-09T06:42:12.842512+0000","last_fullsized":"2023-10-09T06:42:12.842512+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284736,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4504,"num_write_kb":194560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":777,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470813+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:42:01.470813+0000","last_peered":"2023-10-09T06:42:01.470813+0000","last_clean":"2023-10-09T06:42:01.470813+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:42:01.470813+0000","last_undegraded":"2023-10-09T06:42:01.470813+0000","last_fullsized":"2023-10-09T06:42:01.470813+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":519,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:03.179343+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:42:03.179343+0000","last_peered":"2023-10-09T06:42:03.179343+0000","last_clean":"2023-10-09T06:42:03.179343+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:42:03.179343+0000","last_undegraded":"2023-10-09T06:42:03.179343+0000","last_fullsized":"2023-10-09T06:42:03.179343+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":524,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516522+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:42:05.516522+0000","last_peered":"2023-10-09T06:42:05.516522+0000","last_clean":"2023-10-09T06:42:05.516522+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:42:05.516522+0000","last_undegraded":"2023-10-09T06:42:05.516522+0000","last_fullsized":"2023-10-09T06:42:05.516522+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"523'6846","reported_seq":6899,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.809003+0000","last_change":"2023-10-09T06:40:43.168550+0000","last_active":"2023-10-09T06:42:14.809003+0000","last_peered":"2023-10-09T06:42:14.809003+0000","last_clean":"2023-10-09T06:42:14.809003+0000","last_became_active":"2023-10-09T06:40:43.166693+0000","last_became_peered":"2023-10-09T06:40:43.166693+0000","last_unstale":"2023-10-09T06:42:14.809003+0000","last_undegraded":"2023-10-09T06:42:14.809003+0000","last_fullsized":"2023-10-09T06:42:14.809003+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104005632,"num_objects":1587,"num_object_clones":0,"num_object_copies":4761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4585,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":769,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470382+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:42:01.470382+0000","last_peered":"2023-10-09T06:42:01.470382+0000","last_clean":"2023-10-09T06:42:01.470382+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:42:01.470382+0000","last_undegraded":"2023-10-09T06:42:01.470382+0000","last_fullsized":"2023-10-09T06:42:01.470382+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":244,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.748829+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:42:13.748829+0000","last_peered":"2023-10-09T06:42:13.748829+0000","last_clean":"2023-10-09T06:42:13.748829+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:42:13.748829+0000","last_undegraded":"2023-10-09T06:42:13.748829+0000","last_fullsized":"2023-10-09T06:42:13.748829+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":511,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502473+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:42:01.502473+0000","last_peered":"2023-10-09T06:42:01.502473+0000","last_clean":"2023-10-09T06:42:01.502473+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:42:01.502473+0000","last_undegraded":"2023-10-09T06:42:01.502473+0000","last_fullsized":"2023-10-09T06:42:01.502473+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":630,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512381+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:42:01.512381+0000","last_peered":"2023-10-09T06:42:01.512381+0000","last_clean":"2023-10-09T06:42:01.512381+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:42:01.512381+0000","last_undegraded":"2023-10-09T06:42:01.512381+0000","last_fullsized":"2023-10-09T06:42:01.512381+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516078+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:42:05.516078+0000","last_peered":"2023-10-09T06:42:05.516078+0000","last_clean":"2023-10-09T06:42:05.516078+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:42:05.516078+0000","last_undegraded":"2023-10-09T06:42:05.516078+0000","last_fullsized":"2023-10-09T06:42:05.516078+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521416+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:42:05.521416+0000","last_peered":"2023-10-09T06:42:05.521416+0000","last_clean":"2023-10-09T06:42:05.521416+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:42:05.521416+0000","last_undegraded":"2023-10-09T06:42:05.521416+0000","last_fullsized":"2023-10-09T06:42:05.521416+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":667,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512531+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:42:01.512531+0000","last_peered":"2023-10-09T06:42:01.512531+0000","last_clean":"2023-10-09T06:42:01.512531+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:42:01.512531+0000","last_undegraded":"2023-10-09T06:42:01.512531+0000","last_fullsized":"2023-10-09T06:42:01.512531+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.478761+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:42:01.478761+0000","last_peered":"2023-10-09T06:42:01.478761+0000","last_clean":"2023-10-09T06:42:01.478761+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:42:01.478761+0000","last_undegraded":"2023-10-09T06:42:01.478761+0000","last_fullsized":"2023-10-09T06:42:01.478761+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521898+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:42:05.521898+0000","last_peered":"2023-10-09T06:42:05.521898+0000","last_clean":"2023-10-09T06:42:05.521898+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:42:05.521898+0000","last_undegraded":"2023-10-09T06:42:05.521898+0000","last_fullsized":"2023-10-09T06:42:05.521898+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"523'7199","reported_seq":7250,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.795307+0000","last_change":"2023-10-09T06:40:43.182904+0000","last_active":"2023-10-09T06:42:14.795307+0000","last_peered":"2023-10-09T06:42:14.795307+0000","last_clean":"2023-10-09T06:42:14.795307+0000","last_became_active":"2023-10-09T06:40:43.180666+0000","last_became_peered":"2023-10-09T06:40:43.180666+0000","last_unstale":"2023-10-09T06:42:14.795307+0000","last_undegraded":"2023-10-09T06:42:14.795307+0000","last_fullsized":"2023-10-09T06:42:14.795307+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112492544,"num_objects":1717,"num_object_clones":0,"num_object_copies":5151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4809,"num_write_kb":208800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":652,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515798+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:42:05.515798+0000","last_peered":"2023-10-09T06:42:05.515798+0000","last_clean":"2023-10-09T06:42:05.515798+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:42:05.515798+0000","last_undegraded":"2023-10-09T06:42:05.515798+0000","last_fullsized":"2023-10-09T06:42:05.515798+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":403,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512763+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:42:01.512763+0000","last_peered":"2023-10-09T06:42:01.512763+0000","last_clean":"2023-10-09T06:42:01.512763+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:42:01.512763+0000","last_undegraded":"2023-10-09T06:42:01.512763+0000","last_fullsized":"2023-10-09T06:42:01.512763+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":460,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.493693+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:42:01.493693+0000","last_peered":"2023-10-09T06:42:01.493693+0000","last_clean":"2023-10-09T06:42:01.493693+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:42:01.493693+0000","last_undegraded":"2023-10-09T06:42:01.493693+0000","last_fullsized":"2023-10-09T06:42:01.493693+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"523'6964","reported_seq":7079,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.064154+0000","last_change":"2023-10-09T06:40:50.305746+0000","last_active":"2023-10-09T06:42:14.064154+0000","last_peered":"2023-10-09T06:42:14.064154+0000","last_clean":"2023-10-09T06:42:14.064154+0000","last_became_active":"2023-10-09T06:40:43.093584+0000","last_became_peered":"2023-10-09T06:40:43.093584+0000","last_unstale":"2023-10-09T06:42:14.064154+0000","last_undegraded":"2023-10-09T06:42:14.064154+0000","last_fullsized":"2023-10-09T06:42:14.064154+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4832","last_scrub_stamp":"2023-10-09T06:40:50.305626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:50.305626+0000","objects_scrubbed":2516,"log_size":6964,"log_dups_size":0,"ondisk_log_size":6964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:57.896131+0000","scrub_duration":0.56208410600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102694912,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2141,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":632,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:03.179443+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:42:03.179443+0000","last_peered":"2023-10-09T06:42:03.179443+0000","last_clean":"2023-10-09T06:42:03.179443+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:42:03.179443+0000","last_undegraded":"2023-10-09T06:42:03.179443+0000","last_fullsized":"2023-10-09T06:42:03.179443+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":528,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470844+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:42:01.470844+0000","last_peered":"2023-10-09T06:42:01.470844+0000","last_clean":"2023-10-09T06:42:01.470844+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:42:01.470844+0000","last_undegraded":"2023-10-09T06:42:01.470844+0000","last_fullsized":"2023-10-09T06:42:01.470844+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":492,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471228+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:42:01.471228+0000","last_peered":"2023-10-09T06:42:01.471228+0000","last_clean":"2023-10-09T06:42:01.471228+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:42:01.471228+0000","last_undegraded":"2023-10-09T06:42:01.471228+0000","last_fullsized":"2023-10-09T06:42:01.471228+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":797,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515686+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:42:05.515686+0000","last_peered":"2023-10-09T06:42:05.515686+0000","last_clean":"2023-10-09T06:42:05.515686+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:42:05.515686+0000","last_undegraded":"2023-10-09T06:42:05.515686+0000","last_fullsized":"2023-10-09T06:42:05.515686+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":371,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512087+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:42:01.512087+0000","last_peered":"2023-10-09T06:42:01.512087+0000","last_clean":"2023-10-09T06:42:01.512087+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:42:01.512087+0000","last_undegraded":"2023-10-09T06:42:01.512087+0000","last_fullsized":"2023-10-09T06:42:01.512087+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524012+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:42:05.524012+0000","last_peered":"2023-10-09T06:42:05.524012+0000","last_clean":"2023-10-09T06:42:05.524012+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:42:05.524012+0000","last_undegraded":"2023-10-09T06:42:05.524012+0000","last_fullsized":"2023-10-09T06:42:05.524012+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":794,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.514944+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:42:01.514944+0000","last_peered":"2023-10-09T06:42:01.514944+0000","last_clean":"2023-10-09T06:42:01.514944+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:42:01.514944+0000","last_undegraded":"2023-10-09T06:42:01.514944+0000","last_fullsized":"2023-10-09T06:42:01.514944+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":522,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516196+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:42:05.516196+0000","last_peered":"2023-10-09T06:42:05.516196+0000","last_clean":"2023-10-09T06:42:05.516196+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:42:05.516196+0000","last_undegraded":"2023-10-09T06:42:05.516196+0000","last_fullsized":"2023-10-09T06:42:05.516196+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512617+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:42:01.512617+0000","last_peered":"2023-10-09T06:42:01.512617+0000","last_clean":"2023-10-09T06:42:01.512617+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:42:01.512617+0000","last_undegraded":"2023-10-09T06:42:01.512617+0000","last_fullsized":"2023-10-09T06:42:01.512617+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":850,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488843+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:42:01.488843+0000","last_peered":"2023-10-09T06:42:01.488843+0000","last_clean":"2023-10-09T06:42:01.488843+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:42:01.488843+0000","last_undegraded":"2023-10-09T06:42:01.488843+0000","last_fullsized":"2023-10-09T06:42:01.488843+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":365,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.511954+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:42:01.511954+0000","last_peered":"2023-10-09T06:42:01.511954+0000","last_clean":"2023-10-09T06:42:01.511954+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:42:01.511954+0000","last_undegraded":"2023-10-09T06:42:01.511954+0000","last_fullsized":"2023-10-09T06:42:01.511954+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":199,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516187+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:42:05.516187+0000","last_peered":"2023-10-09T06:42:05.516187+0000","last_clean":"2023-10-09T06:42:05.516187+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:42:05.516187+0000","last_undegraded":"2023-10-09T06:42:05.516187+0000","last_fullsized":"2023-10-09T06:42:05.516187+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":636,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.479266+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:42:01.479266+0000","last_peered":"2023-10-09T06:42:01.479266+0000","last_clean":"2023-10-09T06:42:01.479266+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:42:01.479266+0000","last_undegraded":"2023-10-09T06:42:01.479266+0000","last_fullsized":"2023-10-09T06:42:01.479266+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":600,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.500807+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:42:01.500807+0000","last_peered":"2023-10-09T06:42:01.500807+0000","last_clean":"2023-10-09T06:42:01.500807+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:42:01.500807+0000","last_undegraded":"2023-10-09T06:42:01.500807+0000","last_fullsized":"2023-10-09T06:42:01.500807+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:09.105989+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:42:09.105989+0000","last_peered":"2023-10-09T06:42:09.105989+0000","last_clean":"2023-10-09T06:42:09.105989+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:42:09.105989+0000","last_undegraded":"2023-10-09T06:42:09.105989+0000","last_fullsized":"2023-10-09T06:42:09.105989+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"522'6885","reported_seq":6961,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.772008+0000","last_change":"2023-10-09T06:41:45.688515+0000","last_active":"2023-10-09T06:42:12.772008+0000","last_peered":"2023-10-09T06:42:12.772008+0000","last_clean":"2023-10-09T06:42:12.772008+0000","last_became_active":"2023-10-09T06:40:43.094441+0000","last_became_peered":"2023-10-09T06:40:43.094441+0000","last_unstale":"2023-10-09T06:42:12.772008+0000","last_undegraded":"2023-10-09T06:42:12.772008+0000","last_fullsized":"2023-10-09T06:42:12.772008+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6885,"log_dups_size":0,"ondisk_log_size":6885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107839488,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4573,"num_write_kb":199008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":649,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512796+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:42:01.512796+0000","last_peered":"2023-10-09T06:42:01.512796+0000","last_clean":"2023-10-09T06:42:01.512796+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:42:01.512796+0000","last_undegraded":"2023-10-09T06:42:01.512796+0000","last_fullsized":"2023-10-09T06:42:01.512796+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":407,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.515436+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:42:01.515436+0000","last_peered":"2023-10-09T06:42:01.515436+0000","last_clean":"2023-10-09T06:42:01.515436+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:42:01.515436+0000","last_undegraded":"2023-10-09T06:42:01.515436+0000","last_fullsized":"2023-10-09T06:42:01.515436+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":305,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.515196+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:42:01.515196+0000","last_peered":"2023-10-09T06:42:01.515196+0000","last_clean":"2023-10-09T06:42:01.515196+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:42:01.515196+0000","last_undegraded":"2023-10-09T06:42:01.515196+0000","last_fullsized":"2023-10-09T06:42:01.515196+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"523'7010","reported_seq":7064,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.699077+0000","last_change":"2023-10-09T06:40:47.767571+0000","last_active":"2023-10-09T06:42:14.699077+0000","last_peered":"2023-10-09T06:42:14.699077+0000","last_clean":"2023-10-09T06:42:14.699077+0000","last_became_active":"2023-10-09T06:40:43.165023+0000","last_became_peered":"2023-10-09T06:40:43.165023+0000","last_unstale":"2023-10-09T06:42:14.699077+0000","last_undegraded":"2023-10-09T06:42:14.699077+0000","last_fullsized":"2023-10-09T06:42:14.699077+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109051904,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":910,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523812+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:42:05.523812+0000","last_peered":"2023-10-09T06:42:05.523812+0000","last_clean":"2023-10-09T06:42:05.523812+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:42:05.523812+0000","last_undegraded":"2023-10-09T06:42:05.523812+0000","last_fullsized":"2023-10-09T06:42:05.523812+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":369,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512610+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:42:01.512610+0000","last_peered":"2023-10-09T06:42:01.512610+0000","last_clean":"2023-10-09T06:42:01.512610+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:42:01.512610+0000","last_undegraded":"2023-10-09T06:42:01.512610+0000","last_fullsized":"2023-10-09T06:42:01.512610+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":519,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470550+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:42:01.470550+0000","last_peered":"2023-10-09T06:42:01.470550+0000","last_clean":"2023-10-09T06:42:01.470550+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:42:01.470550+0000","last_undegraded":"2023-10-09T06:42:01.470550+0000","last_fullsized":"2023-10-09T06:42:01.470550+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"523'6944","reported_seq":6998,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.543195+0000","last_change":"2023-10-09T06:40:43.082575+0000","last_active":"2023-10-09T06:42:14.543195+0000","last_peered":"2023-10-09T06:42:14.543195+0000","last_clean":"2023-10-09T06:42:14.543195+0000","last_became_active":"2023-10-09T06:40:43.080150+0000","last_became_peered":"2023-10-09T06:40:43.080150+0000","last_unstale":"2023-10-09T06:42:14.543195+0000","last_undegraded":"2023-10-09T06:42:14.543195+0000","last_fullsized":"2023-10-09T06:42:14.543195+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6944,"log_dups_size":0,"ondisk_log_size":6944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106397696,"num_objects":1623,"num_object_clones":0,"num_object_copies":4869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4653,"num_write_kb":200864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":823,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502440+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:42:01.502440+0000","last_peered":"2023-10-09T06:42:01.502440+0000","last_clean":"2023-10-09T06:42:01.502440+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:42:01.502440+0000","last_undegraded":"2023-10-09T06:42:01.502440+0000","last_fullsized":"2023-10-09T06:42:01.502440+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":425,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512537+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:42:01.512537+0000","last_peered":"2023-10-09T06:42:01.512537+0000","last_clean":"2023-10-09T06:42:01.512537+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:42:01.512537+0000","last_undegraded":"2023-10-09T06:42:01.512537+0000","last_fullsized":"2023-10-09T06:42:01.512537+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":498,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.501934+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:42:01.501934+0000","last_peered":"2023-10-09T06:42:01.501934+0000","last_clean":"2023-10-09T06:42:01.501934+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:42:01.501934+0000","last_undegraded":"2023-10-09T06:42:01.501934+0000","last_fullsized":"2023-10-09T06:42:01.501934+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"523'6979","reported_seq":7097,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.076254+0000","last_change":"2023-10-09T06:40:47.766598+0000","last_active":"2023-10-09T06:42:14.076254+0000","last_peered":"2023-10-09T06:42:14.076254+0000","last_clean":"2023-10-09T06:42:14.076254+0000","last_became_active":"2023-10-09T06:40:43.101954+0000","last_became_peered":"2023-10-09T06:40:43.101954+0000","last_unstale":"2023-10-09T06:42:14.076254+0000","last_undegraded":"2023-10-09T06:42:14.076254+0000","last_fullsized":"2023-10-09T06:42:14.076254+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4698","last_scrub_stamp":"2023-10-09T06:40:47.766356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.766356+0000","objects_scrubbed":2441,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:35.858607+0000","scrub_duration":1.024576309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109707264,"num_objects":1674,"num_object_clones":0,"num_object_copies":5022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2291,"num_write_kb":48768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":646,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471023+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:42:01.471023+0000","last_peered":"2023-10-09T06:42:01.471023+0000","last_clean":"2023-10-09T06:42:01.471023+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:42:01.471023+0000","last_undegraded":"2023-10-09T06:42:01.471023+0000","last_fullsized":"2023-10-09T06:42:01.471023+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524086+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:42:05.524086+0000","last_peered":"2023-10-09T06:42:05.524086+0000","last_clean":"2023-10-09T06:42:05.524086+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:42:05.524086+0000","last_undegraded":"2023-10-09T06:42:05.524086+0000","last_fullsized":"2023-10-09T06:42:05.524086+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":487,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516704+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:42:05.516704+0000","last_peered":"2023-10-09T06:42:05.516704+0000","last_clean":"2023-10-09T06:42:05.516704+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:42:05.516704+0000","last_undegraded":"2023-10-09T06:42:05.516704+0000","last_fullsized":"2023-10-09T06:42:05.516704+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"523'6877","reported_seq":6930,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.792707+0000","last_change":"2023-10-09T06:40:43.164619+0000","last_active":"2023-10-09T06:42:14.792707+0000","last_peered":"2023-10-09T06:42:14.792707+0000","last_clean":"2023-10-09T06:42:14.792707+0000","last_became_active":"2023-10-09T06:40:43.159937+0000","last_became_peered":"2023-10-09T06:40:43.159937+0000","last_unstale":"2023-10-09T06:42:14.792707+0000","last_undegraded":"2023-10-09T06:42:14.792707+0000","last_fullsized":"2023-10-09T06:42:14.792707+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108199936,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4597,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":874,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.511866+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:42:01.511866+0000","last_peered":"2023-10-09T06:42:01.511866+0000","last_clean":"2023-10-09T06:42:01.511866+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:42:01.511866+0000","last_undegraded":"2023-10-09T06:42:01.511866+0000","last_fullsized":"2023-10-09T06:42:01.511866+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":252,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522492+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:42:05.522492+0000","last_peered":"2023-10-09T06:42:05.522492+0000","last_clean":"2023-10-09T06:42:05.522492+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:42:05.522492+0000","last_undegraded":"2023-10-09T06:42:05.522492+0000","last_fullsized":"2023-10-09T06:42:05.522492+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":502,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470694+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:42:01.470694+0000","last_peered":"2023-10-09T06:42:01.470694+0000","last_clean":"2023-10-09T06:42:01.470694+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:42:01.470694+0000","last_undegraded":"2023-10-09T06:42:01.470694+0000","last_fullsized":"2023-10-09T06:42:01.470694+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"522'6934","reported_seq":7091,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.537802+0000","last_change":"2023-10-09T06:40:45.736296+0000","last_active":"2023-10-09T06:42:12.537802+0000","last_peered":"2023-10-09T06:42:12.537802+0000","last_clean":"2023-10-09T06:42:12.537802+0000","last_became_active":"2023-10-09T06:40:43.168245+0000","last_became_peered":"2023-10-09T06:40:43.168245+0000","last_unstale":"2023-10-09T06:42:12.537802+0000","last_undegraded":"2023-10-09T06:42:12.537802+0000","last_fullsized":"2023-10-09T06:42:12.537802+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4658","last_scrub_stamp":"2023-10-09T06:40:45.736230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:45.736230+0000","objects_scrubbed":2357,"log_size":6934,"log_dups_size":0,"ondisk_log_size":6934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:31.145769+0000","scrub_duration":0.58060363000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111280128,"num_objects":1698,"num_object_clones":0,"num_object_copies":5094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2285,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":558,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521493+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:42:05.521493+0000","last_peered":"2023-10-09T06:42:05.521493+0000","last_clean":"2023-10-09T06:42:05.521493+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:42:05.521493+0000","last_undegraded":"2023-10-09T06:42:05.521493+0000","last_fullsized":"2023-10-09T06:42:05.521493+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":479,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.501145+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:42:01.501145+0000","last_peered":"2023-10-09T06:42:01.501145+0000","last_clean":"2023-10-09T06:42:01.501145+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:42:01.501145+0000","last_undegraded":"2023-10-09T06:42:01.501145+0000","last_fullsized":"2023-10-09T06:42:01.501145+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":499,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471164+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:42:01.471164+0000","last_peered":"2023-10-09T06:42:01.471164+0000","last_clean":"2023-10-09T06:42:01.471164+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:42:01.471164+0000","last_undegraded":"2023-10-09T06:42:01.471164+0000","last_fullsized":"2023-10-09T06:42:01.471164+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"522'6844","reported_seq":7006,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.857141+0000","last_change":"2023-10-09T06:40:51.573557+0000","last_active":"2023-10-09T06:42:12.857141+0000","last_peered":"2023-10-09T06:42:12.857141+0000","last_clean":"2023-10-09T06:42:12.857141+0000","last_became_active":"2023-10-09T06:40:43.081959+0000","last_became_peered":"2023-10-09T06:40:43.081959+0000","last_unstale":"2023-10-09T06:42:12.857141+0000","last_undegraded":"2023-10-09T06:42:12.857141+0000","last_fullsized":"2023-10-09T06:42:12.857141+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'4739","last_scrub_stamp":"2023-10-09T06:40:51.573471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:51.573471+0000","objects_scrubbed":2521,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:30.924217+0000","scrub_duration":0.34142884899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2105,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":859,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488921+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:42:01.488921+0000","last_peered":"2023-10-09T06:42:01.488921+0000","last_clean":"2023-10-09T06:42:01.488921+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:42:01.488921+0000","last_undegraded":"2023-10-09T06:42:01.488921+0000","last_fullsized":"2023-10-09T06:42:01.488921+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":503,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470454+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:42:01.470454+0000","last_peered":"2023-10-09T06:42:01.470454+0000","last_clean":"2023-10-09T06:42:01.470454+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:42:01.470454+0000","last_undegraded":"2023-10-09T06:42:01.470454+0000","last_fullsized":"2023-10-09T06:42:01.470454+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":482,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471180+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:42:01.471180+0000","last_peered":"2023-10-09T06:42:01.471180+0000","last_clean":"2023-10-09T06:42:01.471180+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:42:01.471180+0000","last_undegraded":"2023-10-09T06:42:01.471180+0000","last_fullsized":"2023-10-09T06:42:01.471180+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"523'6831","reported_seq":6945,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.792551+0000","last_change":"2023-10-09T06:40:47.192403+0000","last_active":"2023-10-09T06:42:14.792551+0000","last_peered":"2023-10-09T06:42:14.792551+0000","last_clean":"2023-10-09T06:42:14.792551+0000","last_became_active":"2023-10-09T06:40:43.074962+0000","last_became_peered":"2023-10-09T06:40:43.074962+0000","last_unstale":"2023-10-09T06:42:14.792551+0000","last_undegraded":"2023-10-09T06:42:14.792551+0000","last_fullsized":"2023-10-09T06:42:14.792551+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4664","last_scrub_stamp":"2023-10-09T06:40:47.192137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.192137+0000","objects_scrubbed":2418,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:24.365070+0000","scrub_duration":0.455376642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103677952,"num_objects":1582,"num_object_clones":0,"num_object_copies":4746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2170,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":484,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.501068+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:42:01.501068+0000","last_peered":"2023-10-09T06:42:01.501068+0000","last_clean":"2023-10-09T06:42:01.501068+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:42:01.501068+0000","last_undegraded":"2023-10-09T06:42:01.501068+0000","last_fullsized":"2023-10-09T06:42:01.501068+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521348+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:42:05.521348+0000","last_peered":"2023-10-09T06:42:05.521348+0000","last_clean":"2023-10-09T06:42:05.521348+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:42:05.521348+0000","last_undegraded":"2023-10-09T06:42:05.521348+0000","last_fullsized":"2023-10-09T06:42:05.521348+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":478,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488614+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:42:01.488614+0000","last_peered":"2023-10-09T06:42:01.488614+0000","last_clean":"2023-10-09T06:42:01.488614+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:42:01.488614+0000","last_undegraded":"2023-10-09T06:42:01.488614+0000","last_fullsized":"2023-10-09T06:42:01.488614+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"523'7252","reported_seq":7304,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.795191+0000","last_change":"2023-10-09T06:40:43.157665+0000","last_active":"2023-10-09T06:42:14.795191+0000","last_peered":"2023-10-09T06:42:14.795191+0000","last_clean":"2023-10-09T06:42:14.795191+0000","last_became_active":"2023-10-09T06:40:43.080919+0000","last_became_peered":"2023-10-09T06:40:43.080919+0000","last_unstale":"2023-10-09T06:42:14.795191+0000","last_undegraded":"2023-10-09T06:42:14.795191+0000","last_fullsized":"2023-10-09T06:42:14.795191+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7252,"log_dups_size":0,"ondisk_log_size":7252,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108625920,"num_objects":1657,"num_object_clones":0,"num_object_copies":4971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4861,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":814,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502532+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:42:01.502532+0000","last_peered":"2023-10-09T06:42:01.502532+0000","last_clean":"2023-10-09T06:42:01.502532+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:42:01.502532+0000","last_undegraded":"2023-10-09T06:42:01.502532+0000","last_fullsized":"2023-10-09T06:42:01.502532+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":531,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.531261+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:42:05.531261+0000","last_peered":"2023-10-09T06:42:05.531261+0000","last_clean":"2023-10-09T06:42:05.531261+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:42:05.531261+0000","last_undegraded":"2023-10-09T06:42:05.531261+0000","last_fullsized":"2023-10-09T06:42:05.531261+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521773+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:42:05.521773+0000","last_peered":"2023-10-09T06:42:05.521773+0000","last_clean":"2023-10-09T06:42:05.521773+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:42:05.521773+0000","last_undegraded":"2023-10-09T06:42:05.521773+0000","last_fullsized":"2023-10-09T06:42:05.521773+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":465,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.527420+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:42:05.527420+0000","last_peered":"2023-10-09T06:42:05.527420+0000","last_clean":"2023-10-09T06:42:05.527420+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:42:05.527420+0000","last_undegraded":"2023-10-09T06:42:05.527420+0000","last_fullsized":"2023-10-09T06:42:05.527420+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"522'9188","reported_seq":9229,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.698836+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:42:12.698836+0000","last_peered":"2023-10-09T06:42:12.698836+0000","last_clean":"2023-10-09T06:42:12.698836+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:42:12.698836+0000","last_undegraded":"2023-10-09T06:42:12.698836+0000","last_fullsized":"2023-10-09T06:42:12.698836+0000","mapping_epoch":489,"log_start":"496'3600","ondisk_log_start":"496'3600","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219545600,"num_objects":3350,"num_object_clones":0,"num_object_copies":10050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9188,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":703,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515942+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:42:05.515942+0000","last_peered":"2023-10-09T06:42:05.515942+0000","last_clean":"2023-10-09T06:42:05.515942+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:42:05.515942+0000","last_undegraded":"2023-10-09T06:42:05.515942+0000","last_fullsized":"2023-10-09T06:42:05.515942+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":254,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.493459+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:42:01.493459+0000","last_peered":"2023-10-09T06:42:01.493459+0000","last_clean":"2023-10-09T06:42:01.493459+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:42:01.493459+0000","last_undegraded":"2023-10-09T06:42:01.493459+0000","last_fullsized":"2023-10-09T06:42:01.493459+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":503,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471148+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:42:01.471148+0000","last_peered":"2023-10-09T06:42:01.471148+0000","last_clean":"2023-10-09T06:42:01.471148+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:42:01.471148+0000","last_undegraded":"2023-10-09T06:42:01.471148+0000","last_fullsized":"2023-10-09T06:42:01.471148+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":495,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.531184+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:42:05.531184+0000","last_peered":"2023-10-09T06:42:05.531184+0000","last_clean":"2023-10-09T06:42:05.531184+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:42:05.531184+0000","last_undegraded":"2023-10-09T06:42:05.531184+0000","last_fullsized":"2023-10-09T06:42:05.531184+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"522'9130","reported_seq":9172,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.863972+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:42:12.863972+0000","last_peered":"2023-10-09T06:42:12.863972+0000","last_clean":"2023-10-09T06:42:12.863972+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:42:12.863972+0000","last_undegraded":"2023-10-09T06:42:12.863972+0000","last_fullsized":"2023-10-09T06:42:12.863972+0000","mapping_epoch":489,"log_start":"495'2700","ondisk_log_start":"495'2700","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023640,"num_objects":3282,"num_object_clones":0,"num_object_copies":9846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3282,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9130,"num_write_kb":397121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":864,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.511990+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:42:01.511990+0000","last_peered":"2023-10-09T06:42:01.511990+0000","last_clean":"2023-10-09T06:42:01.511990+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:42:01.511990+0000","last_undegraded":"2023-10-09T06:42:01.511990+0000","last_fullsized":"2023-10-09T06:42:01.511990+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":517,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521633+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:42:05.521633+0000","last_peered":"2023-10-09T06:42:05.521633+0000","last_clean":"2023-10-09T06:42:05.521633+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:42:05.521633+0000","last_undegraded":"2023-10-09T06:42:05.521633+0000","last_fullsized":"2023-10-09T06:42:05.521633+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":503,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488465+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:42:01.488465+0000","last_peered":"2023-10-09T06:42:01.488465+0000","last_clean":"2023-10-09T06:42:01.488465+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:42:01.488465+0000","last_undegraded":"2023-10-09T06:42:01.488465+0000","last_fullsized":"2023-10-09T06:42:01.488465+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":525,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488667+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:42:01.488667+0000","last_peered":"2023-10-09T06:42:01.488667+0000","last_clean":"2023-10-09T06:42:01.488667+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:42:01.488667+0000","last_undegraded":"2023-10-09T06:42:01.488667+0000","last_fullsized":"2023-10-09T06:42:01.488667+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"522'9369","reported_seq":9410,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.677642+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:42:12.677642+0000","last_peered":"2023-10-09T06:42:12.677642+0000","last_clean":"2023-10-09T06:42:12.677642+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:42:12.677642+0000","last_undegraded":"2023-10-09T06:42:12.677642+0000","last_fullsized":"2023-10-09T06:42:12.677642+0000","mapping_epoch":489,"log_start":"496'3200","ondisk_log_start":"496'3200","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9369,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":331,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512035+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:42:01.512035+0000","last_peered":"2023-10-09T06:42:01.512035+0000","last_clean":"2023-10-09T06:42:01.512035+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:42:01.512035+0000","last_undegraded":"2023-10-09T06:42:01.512035+0000","last_fullsized":"2023-10-09T06:42:01.512035+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":845,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.515271+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:42:01.515271+0000","last_peered":"2023-10-09T06:42:01.515271+0000","last_clean":"2023-10-09T06:42:01.515271+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:42:01.515271+0000","last_undegraded":"2023-10-09T06:42:01.515271+0000","last_fullsized":"2023-10-09T06:42:01.515271+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":236,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749436+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:42:13.749436+0000","last_peered":"2023-10-09T06:42:13.749436+0000","last_clean":"2023-10-09T06:42:13.749436+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:42:13.749436+0000","last_undegraded":"2023-10-09T06:42:13.749436+0000","last_fullsized":"2023-10-09T06:42:13.749436+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":484,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502378+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:42:01.502378+0000","last_peered":"2023-10-09T06:42:01.502378+0000","last_clean":"2023-10-09T06:42:01.502378+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:42:01.502378+0000","last_undegraded":"2023-10-09T06:42:01.502378+0000","last_fullsized":"2023-10-09T06:42:01.502378+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"523'9494","reported_seq":9537,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.825384+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:42:14.825384+0000","last_peered":"2023-10-09T06:42:14.825384+0000","last_clean":"2023-10-09T06:42:14.825384+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:42:14.825384+0000","last_undegraded":"2023-10-09T06:42:14.825384+0000","last_fullsized":"2023-10-09T06:42:14.825384+0000","mapping_epoch":489,"log_start":"496'3400","ondisk_log_start":"496'3400","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6094,"log_dups_size":0,"ondisk_log_size":6094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9494,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":834,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.527301+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:42:05.527301+0000","last_peered":"2023-10-09T06:42:05.527301+0000","last_clean":"2023-10-09T06:42:05.527301+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:42:05.527301+0000","last_undegraded":"2023-10-09T06:42:05.527301+0000","last_fullsized":"2023-10-09T06:42:05.527301+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":385,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.515082+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:42:01.515082+0000","last_peered":"2023-10-09T06:42:01.515082+0000","last_clean":"2023-10-09T06:42:01.515082+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:42:01.515082+0000","last_undegraded":"2023-10-09T06:42:01.515082+0000","last_fullsized":"2023-10-09T06:42:01.515082+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":200,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515865+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:42:05.515865+0000","last_peered":"2023-10-09T06:42:05.515865+0000","last_clean":"2023-10-09T06:42:05.515865+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:42:05.515865+0000","last_undegraded":"2023-10-09T06:42:05.515865+0000","last_fullsized":"2023-10-09T06:42:05.515865+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516605+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:42:05.516605+0000","last_peered":"2023-10-09T06:42:05.516605+0000","last_clean":"2023-10-09T06:42:05.516605+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:42:05.516605+0000","last_undegraded":"2023-10-09T06:42:05.516605+0000","last_fullsized":"2023-10-09T06:42:05.516605+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"523'7001","reported_seq":7115,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.098691+0000","last_change":"2023-10-09T06:40:48.261698+0000","last_active":"2023-10-09T06:42:14.098691+0000","last_peered":"2023-10-09T06:42:14.098691+0000","last_clean":"2023-10-09T06:42:14.098691+0000","last_became_active":"2023-10-09T06:40:43.090550+0000","last_became_peered":"2023-10-09T06:40:43.090550+0000","last_unstale":"2023-10-09T06:42:14.098691+0000","last_undegraded":"2023-10-09T06:42:14.098691+0000","last_fullsized":"2023-10-09T06:42:14.098691+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4775","last_scrub_stamp":"2023-10-09T06:40:48.261467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:48.261467+0000","objects_scrubbed":2451,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:45.927807+0000","scrub_duration":0.48199046000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2231,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":844,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502385+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:42:01.502385+0000","last_peered":"2023-10-09T06:42:01.502385+0000","last_clean":"2023-10-09T06:42:01.502385+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:42:01.502385+0000","last_undegraded":"2023-10-09T06:42:01.502385+0000","last_fullsized":"2023-10-09T06:42:01.502385+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":385,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512687+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:42:01.512687+0000","last_peered":"2023-10-09T06:42:01.512687+0000","last_clean":"2023-10-09T06:42:01.512687+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:42:01.512687+0000","last_undegraded":"2023-10-09T06:42:01.512687+0000","last_fullsized":"2023-10-09T06:42:01.512687+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":521,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.531114+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:42:05.531114+0000","last_peered":"2023-10-09T06:42:05.531114+0000","last_clean":"2023-10-09T06:42:05.531114+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:42:05.531114+0000","last_undegraded":"2023-10-09T06:42:05.531114+0000","last_fullsized":"2023-10-09T06:42:05.531114+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":509,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516819+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:42:05.516819+0000","last_peered":"2023-10-09T06:42:05.516819+0000","last_clean":"2023-10-09T06:42:05.516819+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:42:05.516819+0000","last_undegraded":"2023-10-09T06:42:05.516819+0000","last_fullsized":"2023-10-09T06:42:05.516819+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"522'9392","reported_seq":9435,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.142545+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:42:13.142545+0000","last_peered":"2023-10-09T06:42:13.142545+0000","last_clean":"2023-10-09T06:42:13.142545+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:42:13.142545+0000","last_undegraded":"2023-10-09T06:42:13.142545+0000","last_fullsized":"2023-10-09T06:42:13.142545+0000","mapping_epoch":489,"log_start":"497'4000","ondisk_log_start":"497'4000","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5392,"log_dups_size":0,"ondisk_log_size":5392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216793088,"num_objects":3308,"num_object_clones":0,"num_object_copies":9924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":845,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471087+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:42:01.471087+0000","last_peered":"2023-10-09T06:42:01.471087+0000","last_clean":"2023-10-09T06:42:01.471087+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:42:01.471087+0000","last_undegraded":"2023-10-09T06:42:01.471087+0000","last_fullsized":"2023-10-09T06:42:01.471087+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":256,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522947+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:42:05.522947+0000","last_peered":"2023-10-09T06:42:05.522947+0000","last_clean":"2023-10-09T06:42:05.522947+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:42:05.522947+0000","last_undegraded":"2023-10-09T06:42:05.522947+0000","last_fullsized":"2023-10-09T06:42:05.522947+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":488,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.492725+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:42:01.492725+0000","last_peered":"2023-10-09T06:42:01.492725+0000","last_clean":"2023-10-09T06:42:01.492725+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:42:01.492725+0000","last_undegraded":"2023-10-09T06:42:01.492725+0000","last_fullsized":"2023-10-09T06:42:01.492725+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":245,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521272+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:42:05.521272+0000","last_peered":"2023-10-09T06:42:05.521272+0000","last_clean":"2023-10-09T06:42:05.521272+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:42:05.521272+0000","last_undegraded":"2023-10-09T06:42:05.521272+0000","last_fullsized":"2023-10-09T06:42:05.521272+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"523'9417","reported_seq":9458,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.810979+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:42:14.810979+0000","last_peered":"2023-10-09T06:42:14.810979+0000","last_clean":"2023-10-09T06:42:14.810979+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:42:14.810979+0000","last_undegraded":"2023-10-09T06:42:14.810979+0000","last_fullsized":"2023-10-09T06:42:14.810979+0000","mapping_epoch":489,"log_start":"496'3400","ondisk_log_start":"496'3400","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6017,"log_dups_size":0,"ondisk_log_size":6017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214892544,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9417,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":878,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524157+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:42:05.524157+0000","last_peered":"2023-10-09T06:42:05.524157+0000","last_clean":"2023-10-09T06:42:05.524157+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:42:05.524157+0000","last_undegraded":"2023-10-09T06:42:05.524157+0000","last_fullsized":"2023-10-09T06:42:05.524157+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":506,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.502615+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:42:01.502615+0000","last_peered":"2023-10-09T06:42:01.502615+0000","last_clean":"2023-10-09T06:42:01.502615+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:42:01.502615+0000","last_undegraded":"2023-10-09T06:42:01.502615+0000","last_fullsized":"2023-10-09T06:42:01.502615+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":491,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516430+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:42:05.516430+0000","last_peered":"2023-10-09T06:42:05.516430+0000","last_clean":"2023-10-09T06:42:05.516430+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:42:05.516430+0000","last_undegraded":"2023-10-09T06:42:05.516430+0000","last_fullsized":"2023-10-09T06:42:05.516430+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":485,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470494+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:42:01.470494+0000","last_peered":"2023-10-09T06:42:01.470494+0000","last_clean":"2023-10-09T06:42:01.470494+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:42:01.470494+0000","last_undegraded":"2023-10-09T06:42:01.470494+0000","last_fullsized":"2023-10-09T06:42:01.470494+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"523'6969","reported_seq":7021,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.660593+0000","last_change":"2023-10-09T06:40:43.188245+0000","last_active":"2023-10-09T06:42:14.660593+0000","last_peered":"2023-10-09T06:42:14.660593+0000","last_clean":"2023-10-09T06:42:14.660593+0000","last_became_active":"2023-10-09T06:40:43.186495+0000","last_became_peered":"2023-10-09T06:40:43.186495+0000","last_unstale":"2023-10-09T06:42:14.660593+0000","last_undegraded":"2023-10-09T06:42:14.660593+0000","last_fullsized":"2023-10-09T06:42:14.660593+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6969,"log_dups_size":0,"ondisk_log_size":6969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99549184,"num_objects":1519,"num_object_clones":0,"num_object_copies":4557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4689,"num_write_kb":198656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":805,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521414+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:42:05.521414+0000","last_peered":"2023-10-09T06:42:05.521414+0000","last_clean":"2023-10-09T06:42:05.521414+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:42:05.521414+0000","last_undegraded":"2023-10-09T06:42:05.521414+0000","last_fullsized":"2023-10-09T06:42:05.521414+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":492,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516841+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:42:05.516841+0000","last_peered":"2023-10-09T06:42:05.516841+0000","last_clean":"2023-10-09T06:42:05.516841+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:42:05.516841+0000","last_undegraded":"2023-10-09T06:42:05.516841+0000","last_fullsized":"2023-10-09T06:42:05.516841+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":513,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.488412+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:42:01.488412+0000","last_peered":"2023-10-09T06:42:01.488412+0000","last_clean":"2023-10-09T06:42:01.488412+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:42:01.488412+0000","last_undegraded":"2023-10-09T06:42:01.488412+0000","last_fullsized":"2023-10-09T06:42:01.488412+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"522'6869","reported_seq":9303,"reported_epoch":522,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:13.172112+0000","last_change":"2023-10-09T06:41:56.489849+0000","last_active":"2023-10-09T06:42:13.172112+0000","last_peered":"2023-10-09T06:42:13.172112+0000","last_clean":"2023-10-09T06:40:50.553480+0000","last_became_active":"2023-10-09T06:40:53.875917+0000","last_became_peered":"2023-10-09T06:40:53.875917+0000","last_unstale":"2023-10-09T06:42:13.172112+0000","last_undegraded":"2023-10-09T06:40:53.725278+0000","last_fullsized":"2023-10-09T06:40:53.643650+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109772800,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1066,"num_objects_misplaced":2344,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4589,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":792,"num_bytes_recovered":51904512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-30},{"shards":"1,2,3","objects":639}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":735,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.521935+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:42:05.521935+0000","last_peered":"2023-10-09T06:42:05.521935+0000","last_clean":"2023-10-09T06:42:05.521935+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:42:05.521935+0000","last_undegraded":"2023-10-09T06:42:05.521935+0000","last_fullsized":"2023-10-09T06:42:05.521935+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":200,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470826+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:42:01.470826+0000","last_peered":"2023-10-09T06:42:01.470826+0000","last_clean":"2023-10-09T06:42:01.470826+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:42:01.470826+0000","last_undegraded":"2023-10-09T06:42:01.470826+0000","last_fullsized":"2023-10-09T06:42:01.470826+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":529,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471253+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:42:01.471253+0000","last_peered":"2023-10-09T06:42:01.471253+0000","last_clean":"2023-10-09T06:42:01.471253+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:42:01.471253+0000","last_undegraded":"2023-10-09T06:42:01.471253+0000","last_fullsized":"2023-10-09T06:42:01.471253+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"522'6960","reported_seq":17143,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.158951+0000","last_change":"2023-10-09T06:41:58.466796+0000","last_active":"2023-10-09T06:42:13.158951+0000","last_peered":"2023-10-09T06:42:13.158951+0000","last_clean":"2023-10-09T06:42:13.158951+0000","last_became_active":"2023-10-09T06:41:58.457422+0000","last_became_peered":"2023-10-09T06:41:58.457422+0000","last_unstale":"2023-10-09T06:42:13.158951+0000","last_undegraded":"2023-10-09T06:42:13.158951+0000","last_fullsized":"2023-10-09T06:42:13.158951+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6960,"log_dups_size":0,"ondisk_log_size":6960,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106659840,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4670,"num_write_kb":201504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2529,"num_bytes_recovered":165740544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":200,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.471184+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:42:01.471184+0000","last_peered":"2023-10-09T06:42:01.471184+0000","last_clean":"2023-10-09T06:42:01.471184+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:42:01.471184+0000","last_undegraded":"2023-10-09T06:42:01.471184+0000","last_fullsized":"2023-10-09T06:42:01.471184+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":539,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470504+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:42:01.470504+0000","last_peered":"2023-10-09T06:42:01.470504+0000","last_clean":"2023-10-09T06:42:01.470504+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:42:01.470504+0000","last_undegraded":"2023-10-09T06:42:01.470504+0000","last_fullsized":"2023-10-09T06:42:01.470504+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"523'7053","reported_seq":7109,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.515272+0000","last_change":"2023-10-09T06:40:43.203964+0000","last_active":"2023-10-09T06:42:14.515272+0000","last_peered":"2023-10-09T06:42:14.515272+0000","last_clean":"2023-10-09T06:42:14.515272+0000","last_became_active":"2023-10-09T06:40:43.185473+0000","last_became_peered":"2023-10-09T06:40:43.185473+0000","last_unstale":"2023-10-09T06:42:14.515272+0000","last_undegraded":"2023-10-09T06:42:14.515272+0000","last_fullsized":"2023-10-09T06:42:14.515272+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106233856,"num_objects":1621,"num_object_clones":0,"num_object_copies":4863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4733,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":878,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.512538+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:42:01.512538+0000","last_peered":"2023-10-09T06:42:01.512538+0000","last_clean":"2023-10-09T06:42:01.512538+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:42:01.512538+0000","last_undegraded":"2023-10-09T06:42:01.512538+0000","last_fullsized":"2023-10-09T06:42:01.512538+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":227,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522112+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:42:05.522112+0000","last_peered":"2023-10-09T06:42:05.522112+0000","last_clean":"2023-10-09T06:42:05.522112+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:42:05.522112+0000","last_undegraded":"2023-10-09T06:42:05.522112+0000","last_fullsized":"2023-10-09T06:42:05.522112+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.753590+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:42:13.753590+0000","last_peered":"2023-10-09T06:42:13.753590+0000","last_clean":"2023-10-09T06:42:13.753590+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:42:13.753590+0000","last_undegraded":"2023-10-09T06:42:13.753590+0000","last_fullsized":"2023-10-09T06:42:13.753590+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"522'7023","reported_seq":7085,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:12.834830+0000","last_change":"2023-10-09T06:41:43.517660+0000","last_active":"2023-10-09T06:42:12.834830+0000","last_peered":"2023-10-09T06:42:12.834830+0000","last_clean":"2023-10-09T06:42:12.834830+0000","last_became_active":"2023-10-09T06:40:43.111680+0000","last_became_peered":"2023-10-09T06:40:43.111680+0000","last_unstale":"2023-10-09T06:42:12.834830+0000","last_undegraded":"2023-10-09T06:42:12.834830+0000","last_fullsized":"2023-10-09T06:42:12.834830+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108363776,"num_objects":1654,"num_object_clones":0,"num_object_copies":4962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4712,"num_write_kb":203680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":645,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.493955+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:42:01.493955+0000","last_peered":"2023-10-09T06:42:01.493955+0000","last_clean":"2023-10-09T06:42:01.493955+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:42:01.493955+0000","last_undegraded":"2023-10-09T06:42:01.493955+0000","last_fullsized":"2023-10-09T06:42:01.493955+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":522,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516002+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:42:05.516002+0000","last_peered":"2023-10-09T06:42:05.516002+0000","last_clean":"2023-10-09T06:42:05.516002+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:42:05.516002+0000","last_undegraded":"2023-10-09T06:42:05.516002+0000","last_fullsized":"2023-10-09T06:42:05.516002+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":482,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-09T06:42:01.470262+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:42:01.470262+0000","last_peered":"2023-10-09T06:42:01.470262+0000","last_clean":"2023-10-09T06:42:01.470262+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:42:01.470262+0000","last_undegraded":"2023-10-09T06:42:01.470262+0000","last_fullsized":"2023-10-09T06:42:01.470262+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":3446407192,"num_objects":52589,"num_object_clones":0,"num_object_copies":157767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1066,"num_objects_misplaced":2344,"num_objects_unfound":0,"num_objects_dirty":52589,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131908,"num_write_kb":5352961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3323,"num_bytes_recovered":217776128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10862342144,"data_stored":10862329928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174699,"ondisk_log_size":174699,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234539,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241956,"kb_used_data":749148,"kb_used_omap":0,"kb_used_meta":1492800,"kb_avail":91503196,"statfs":{"total":95995035648,"available":93699272704,"internally_reserved":0,"allocated":767127552,"data_stored":766061392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1528627200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:15 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.552,"15min":3.6890000000000001},"min":{"1min":0.24299999999999999,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":4.7699999999999996,"5min":8.8819999999999997,"15min":215.97800000000001},"last":0.216},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.585,"15min":3.7509999999999999},"min":{"1min":0.27100000000000002,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":5.4009999999999998,"5min":10.449999999999999,"15min":215.577},"last":0.27000000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":12.045999999999999,"5min":4.665,"15min":5.8719999999999999},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":169.51900000000001,"5min":169.51900000000001,"15min":215.61699999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":11.603999999999999,"5min":4.6269999999999998,"15min":4.867},"min":{"1min":0.28899999999999998,"5min":0.28100000000000003,"15min":0.23300000000000001},"max":{"1min":169.101,"5min":169.101,"15min":216.22200000000001},"last":2.528}]},{"osd":2,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":6.4640000000000004,"15min":5.8150000000000004},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":30.117000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":2.6469999999999998},{"interface":"front","average":{"1min":2.5870000000000002,"5min":6.5140000000000002,"15min":5.7960000000000003},"min":{"1min":0.35499999999999998,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":30.245999999999999,"5min":228.291,"15min":228.291},"last":1.3680000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":5.6219999999999999,"5min":6.7199999999999998,"15min":5.774},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":43.341000000000001,"5min":175.40600000000001,"15min":215.922},"last":2.1989999999999998},{"interface":"front","average":{"1min":5.242,"5min":6.5490000000000004,"15min":5.7400000000000002},"min":{"1min":0.373,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":41.433999999999997,"5min":175.30099999999999,"15min":215.94900000000001},"last":6.2670000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":4.3609999999999998,"15min":3.1480000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":2.8199999999999998,"5min":225.041,"15min":225.041},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":4.1680000000000001,"15min":3.0139999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6059999999999999,"5min":225.244,"15min":225.244},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":4.9210000000000003,"15min":3.3799999999999999},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.879,"5min":219.923,"15min":219.923},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.628,"5min":4.6020000000000003,"15min":3.3199999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":1.8600000000000001,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.64300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":4.5449999999999999,"15min":3.4620000000000002},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.317},"max":{"1min":2.23,"5min":222.179,"15min":222.179},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":4.4829999999999997,"15min":3.294},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.4989999999999997,"5min":223.261,"15min":223.261},"last":1.893}]}]},{"osd":6,"up_from":46,"seq":197568496175,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2112720,"kb_used_data":917128,"kb_used_omap":0,"kb_used_meta":1195584,"kb_avail":91632432,"statfs":{"total":95995035648,"available":93831610368,"internally_reserved":0,"allocated":939139072,"data_stored":938092138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1224278016},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.5740000000000001,"15min":2.4359999999999999},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.2199999999999998,"5min":23.125,"15min":75.061999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.341,"5min":4.056,"15min":3.5150000000000001},"min":{"1min":0.33500000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":4.2329999999999997,"5min":212.096,"15min":212.096},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:41:11 2023","interfaces":[{"interface":"back","average":{"1min":7.7789999999999999,"5min":5.1950000000000003,"15min":5.0140000000000002},"min":{"1min":0.27300000000000002,"5min":0.26500000000000001,"15min":0.20899999999999999},"max":{"1min":157.852,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":7.931,"5min":5.2270000000000003,"15min":5.0720000000000001},"min":{"1min":0.30399999999999999,"5min":0.222,"15min":0.222},"max":{"1min":157.87100000000001,"5min":219.94800000000001,"15min":292.904},"last":0.56899999999999995}]},{"osd":2,"last update":"Mon Oct 9 06:41:11 2023","interfaces":[{"interface":"back","average":{"1min":8.6869999999999994,"5min":6.0250000000000004,"15min":3.7469999999999999},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":158,"5min":185.654,"15min":185.654},"last":1.27},{"interface":"front","average":{"1min":8.5180000000000007,"5min":5.8179999999999996,"15min":3.5550000000000002},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":158.017,"5min":185.67699999999999,"15min":185.67699999999999},"last":1.2769999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:41:30 2023","interfaces":[{"interface":"back","average":{"1min":8.7029999999999994,"5min":4.9080000000000004,"15min":4.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":161.60300000000001,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":9.0820000000000007,"5min":4.899,"15min":4.0170000000000003},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.318},"max":{"1min":161.56299999999999,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.42399999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.5780000000000001,"15min":1.512},"min":{"1min":0.253,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":3.8140000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":3.7669999999999999,"15min":2.1259999999999999},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.223},"max":{"1min":2.4039999999999999,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.33500000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.677,"15min":1.302},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.7789999999999999,"5min":9.984,"15min":12.747999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":2.0089999999999999,"15min":1.4870000000000001},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.4340000000000002,"5min":12.244999999999999,"15min":16.536999999999999},"last":0.37}]},{"osd":7,"last update":"Mon Oct 9 06:41:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":2.8340000000000001,"15min":2.6749999999999998},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5220000000000002,"5min":165.37,"15min":165.37},"last":0.379},{"interface":"front","average":{"1min":1.1479999999999999,"5min":3.2170000000000001,"15min":2.9929999999999999},"min":{"1min":0.36899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.6369999999999996,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.39400000000000002}]}]},{"osd":1,"up_from":110,"seq":472446402912,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2929104,"kb_used_data":1855112,"kb_used_omap":4,"kb_used_meta":1073979,"kb_avail":90816048,"statfs":{"total":95995035648,"available":92995633152,"internally_reserved":0,"allocated":1899634688,"data_stored":1897418764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1099755406},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.5410000000000004,"5min":3.9950000000000001,"15min":4.9109999999999996},"min":{"1min":0.21299999999999999,"5min":0.182,"15min":0.182},"max":{"1min":102.941,"5min":110.479,"15min":269.79199999999997},"last":0.23200000000000001},{"interface":"front","average":{"1min":10.712999999999999,"5min":4.6639999999999997,"15min":5.2030000000000003},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":100.16800000000001,"5min":110.59999999999999,"15min":271.20999999999998},"last":2.3620000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.3420000000000001,"15min":2.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.184},"max":{"1min":8.0129999999999999,"5min":8.0129999999999999,"15min":138.452},"last":3.0110000000000001},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.5449999999999999,"15min":2.8530000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":17.555,"5min":17.555,"15min":138.47499999999999},"last":0.26300000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.7000000000000002,"5min":2.5379999999999998,"15min":2.7080000000000002},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":13.523999999999999,"5min":13.523999999999999,"15min":31.062000000000001},"last":2.6880000000000002},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.5129999999999999,"15min":1.8799999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.22700000000000001},"max":{"1min":8.4220000000000006,"5min":8.4220000000000006,"15min":27.657},"last":0.30299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.013999999999999,"5min":3.927,"15min":3.2400000000000002},"min":{"1min":0.316,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":180.59700000000001,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":9.3659999999999997,"5min":3.6040000000000001,"15min":2.919},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":180.70599999999999,"5min":180.70599999999999,"15min":180.70599999999999},"last":2.3039999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.6459999999999999,"15min":2.5630000000000002},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":7.9420000000000002,"5min":15.83,"15min":119.408},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.988,"15min":2.7850000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":8.3819999999999997,"15min":119.483},"last":2.3330000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.7230000000000008,"5min":3.883,"15min":3.4409999999999998},"min":{"1min":0.33700000000000002,"5min":0.317,"15min":0.317},"max":{"1min":183.12100000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":2.2679999999999998},{"interface":"front","average":{"1min":10.446,"5min":4.3179999999999996,"15min":3.8639999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":183.13900000000001,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.67600000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.129,"5min":4.5419999999999998,"15min":4.3570000000000002},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":178.74000000000001,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":8.9540000000000006,"5min":3.8029999999999999,"15min":3.7109999999999999},"min":{"1min":0.379,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":178.75299999999999,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.48099999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024346,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2794880,"kb_used_data":1683896,"kb_used_omap":5,"kb_used_meta":1110970,"kb_avail":90950272,"statfs":{"total":95995035648,"available":93133078528,"internally_reserved":0,"allocated":1724309504,"data_stored":1721710306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1137633914},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:41:51 2023","interfaces":[{"interface":"back","average":{"1min":13.958,"5min":8.4000000000000004,"15min":4.8179999999999996},"min":{"1min":0.192,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":172.762,"5min":201.381,"15min":201.381},"last":6.4580000000000002},{"interface":"front","average":{"1min":13.670999999999999,"5min":8.4559999999999995,"15min":5.0940000000000003},"min":{"1min":0.27300000000000002,"5min":0.193,"15min":0.187},"max":{"1min":172.773,"5min":201.44200000000001,"15min":201.44200000000001},"last":3.4470000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":3.3239999999999998,"15min":3.5099999999999998},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":4.5289999999999999,"5min":97.650000000000006,"15min":100.425},"last":1.8759999999999999},{"interface":"front","average":{"1min":2.698,"5min":3.7210000000000001,"15min":3.9260000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":21.096,"5min":96.082999999999998,"15min":99.769999999999996},"last":3.1459999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:41:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":3.4279999999999999,"15min":4.891},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":6.5389999999999997,"5min":95.555000000000007,"15min":208.11199999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.0720000000000001,"5min":3.4319999999999999,"15min":4.7839999999999998},"min":{"1min":0.29299999999999998,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.2329999999999997,"5min":97.605000000000004,"15min":208.25899999999999},"last":3.4729999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:41:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":6.0880000000000001,"15min":3.0510000000000002},"min":{"1min":0.316,"5min":0.309,"15min":0.27700000000000002},"max":{"1min":14.183,"5min":200.654,"15min":200.654},"last":6.2519999999999998},{"interface":"front","average":{"1min":1.889,"5min":6.4589999999999996,"15min":3.3969999999999998},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.2619999999999996,"5min":200.86600000000001,"15min":200.86600000000001},"last":1.851}]},{"osd":5,"last update":"Mon Oct 9 06:41:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.794,"15min":2.0760000000000001},"min":{"1min":0.30199999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":14.263,"5min":80.885999999999996,"15min":80.885999999999996},"last":6.2990000000000004},{"interface":"front","average":{"1min":1.9410000000000001,"5min":2.5270000000000001,"15min":2.1909999999999998},"min":{"1min":0.40400000000000003,"5min":0.32900000000000001,"15min":0.31900000000000001},"max":{"1min":6.3659999999999997,"5min":80.438999999999993,"15min":80.438999999999993},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:41:22 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":3.1269999999999998,"15min":3.21},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":6.4130000000000003,"5min":118.056,"15min":120.572},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.423,"5min":3.0790000000000002,"15min":2.9380000000000002},"min":{"1min":0.35799999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.308999999999999,"5min":118.274,"15min":120.64400000000001},"last":6.3280000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:41:22 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.863,"15min":2.4399999999999999},"min":{"1min":0.436,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.2939999999999996,"5min":16.911000000000001,"15min":122.018},"last":1.8149999999999999},{"interface":"front","average":{"1min":2.2850000000000001,"5min":2.3100000000000001,"15min":2.8380000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.0069999999999997,"5min":9.9190000000000005,"15min":122.32299999999999},"last":0.38800000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814215,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484896,"kb_used_data":1471640,"kb_used_omap":7,"kb_used_meta":1013176,"kb_avail":91260256,"statfs":{"total":95995035648,"available":93450502144,"internally_reserved":0,"allocated":1506959360,"data_stored":1504793632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1037492305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":6.5650000000000004,"15min":6.5469999999999997},"min":{"1min":0.254,"5min":0.189,"15min":0.189},"max":{"1min":14.374000000000001,"5min":214.595,"15min":214.595},"last":1.659},{"interface":"front","average":{"1min":3.786,"5min":6.1200000000000001,"15min":6.226},"min":{"1min":0.221,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":17.053000000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":1.6479999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.6360000000000001,"15min":4.3070000000000004},"min":{"1min":0.217,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.773999999999999,"5min":80.141000000000005,"15min":140.452},"last":1.474},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.665,"15min":4.5149999999999997},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":11.930999999999999,"5min":79.474000000000004,"15min":139.75299999999999},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:41:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":2.7650000000000001,"15min":3.464},"min":{"1min":0.27400000000000002,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":29.093,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.089,"5min":2.7120000000000002,"15min":3.7669999999999999},"min":{"1min":0.37,"5min":0.27700000000000002,"15min":0.25700000000000001},"max":{"1min":29.126999999999999,"5min":79.614000000000004,"15min":91.685000000000002},"last":1.7190000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":2.036,"15min":2.3069999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.5679999999999996,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2120000000000002,"15min":2.54},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":4.4539999999999997,"5min":82.872,"15min":82.872},"last":0.313}]},{"osd":5,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.994,"15min":2.3570000000000002},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7040000000000006,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1720000000000002,"15min":2.4950000000000001},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":4.4109999999999996,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.34999999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3039999999999998,"15min":3.2010000000000001},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.312},"max":{"1min":4.4880000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.376},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.5209999999999999,"15min":3.1850000000000001},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":6.3689999999999998,"5min":80.364999999999995,"15min":122.517},"last":1.5449999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":8.9900000000000002,"5min":3.8100000000000001,"15min":3.298},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":132.06399999999999,"5min":132.06399999999999,"15min":132.06399999999999},"last":1.518},{"interface":"front","average":{"1min":8.7509999999999994,"5min":3.4420000000000002,"15min":2.9460000000000002},"min":{"1min":0.35899999999999999,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":132.196,"5min":132.196,"15min":132.196},"last":0.42999999999999999}]}]},{"osd":3,"up_from":149,"seq":639950127412,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2436184,"kb_used_data":1527120,"kb_used_omap":2,"kb_used_meta":909053,"kb_avail":91308968,"statfs":{"total":95995035648,"available":93500383232,"internally_reserved":0,"allocated":1563770880,"data_stored":1561730233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":930870325},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.7939999999999996,"5min":2.8290000000000002,"15min":4.0720000000000001},"min":{"1min":0.217,"5min":0.216,"15min":0.19},"max":{"1min":84.817999999999998,"5min":84.817999999999998,"15min":161.821},"last":2.0920000000000001},{"interface":"front","average":{"1min":6.6589999999999998,"5min":2.964,"15min":3.9300000000000002},"min":{"1min":0.246,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":84.635000000000005,"5min":84.635000000000005,"15min":163.52000000000001},"last":10.412000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.5890000000000004,"5min":3.1419999999999999,"15min":3.9790000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21199999999999999},"max":{"1min":83.412000000000006,"5min":83.412000000000006,"15min":184.012},"last":2.2389999999999999},{"interface":"front","average":{"1min":5.8739999999999997,"5min":3.1219999999999999,"15min":4.734},"min":{"1min":0.32100000000000001,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":83.394999999999996,"5min":83.394999999999996,"15min":184.703},"last":2.5270000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":2.9430000000000001,"15min":3.9510000000000001},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.0199999999999996,"5min":118.93899999999999,"15min":182.44900000000001},"last":0.878},{"interface":"front","average":{"1min":2.081,"5min":3.0259999999999998,"15min":4.2759999999999998},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":6.7329999999999997,"5min":118.97199999999999,"15min":182.56399999999999},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":3.3820000000000001,"15min":3.3319999999999999},"min":{"1min":0.29099999999999998,"5min":0.28899999999999998,"15min":0.26900000000000002},"max":{"1min":5.2679999999999998,"5min":133.80099999999999,"15min":178.22200000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":3.4340000000000002,"15min":3.3959999999999999},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":7.2690000000000001,"5min":133.648,"15min":180.43600000000001},"last":0.57999999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":3.5329999999999999,"15min":3.6320000000000001},"min":{"1min":0.374,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":7.3600000000000003,"5min":128.083,"15min":177.56299999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.6220000000000001,"5min":3.359,"15min":3.0099999999999998},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":7.0060000000000002,"5min":128.11799999999999,"15min":177.59399999999999},"last":0.89900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.9810000000000001,"15min":2.7400000000000002},"min":{"1min":0.40100000000000002,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":7.3159999999999998,"5min":10.505000000000001,"15min":130.39599999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.8029999999999999,"15min":2.3809999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.9859999999999998,"5min":9.8179999999999996,"15min":130.67699999999999},"last":0.91500000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.657,"5min":2.641,"15min":3.04},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":99.248000000000005,"5min":99.248000000000005,"15min":121.613},"last":0.41199999999999998},{"interface":"front","average":{"1min":6.7199999999999998,"5min":2.8660000000000001,"15min":3.4860000000000002},"min":{"1min":0.41299999999999998,"5min":0.372,"15min":0.33400000000000002},"max":{"1min":99.457999999999998,"5min":99.457999999999998,"15min":121.559},"last":0.65100000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019448,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2936856,"kb_used_data":1461264,"kb_used_omap":0,"kb_used_meta":1475456,"kb_avail":90808296,"statfs":{"total":95995035648,"available":92987695104,"internally_reserved":0,"allocated":1496334336,"data_stored":1495279587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1510866944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:24 2023","interfaces":[{"interface":"back","average":{"1min":6.7699999999999996,"5min":2.1699999999999999,"15min":6.298},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":108.01300000000001,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.23400000000000001},{"interface":"front","average":{"1min":6.5819999999999999,"5min":2.702,"15min":6.4980000000000002},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":107.96299999999999,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.40500000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":5.7619999999999996,"5min":3.9820000000000002,"15min":4.1699999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":69.644000000000005,"5min":163.392,"15min":163.392},"last":0.34799999999999998},{"interface":"front","average":{"1min":6.1920000000000002,"5min":4.085,"15min":4.2439999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":69.572000000000003,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.317}]},{"osd":2,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":5.1680000000000001,"15min":4.3579999999999997},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3180000000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.25,"5min":5.4630000000000001,"15min":4.3079999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.26400000000000001},"max":{"1min":3.3140000000000001,"5min":175.875,"15min":175.875},"last":2.1139999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:41:29 2023","interfaces":[{"interface":"back","average":{"1min":7.0590000000000002,"5min":4.6420000000000003,"15min":5.9500000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":108.19799999999999,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.499},{"interface":"front","average":{"1min":7.5979999999999999,"5min":4.3659999999999997,"15min":5.7699999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":112.21899999999999,"5min":112.21899999999999,"15min":161.934},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.006,"15min":0.96299999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":3.1070000000000002,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.198,"15min":1.087},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.7599999999999998,"5min":8.4600000000000009,"15min":9.5009999999999994},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.1899999999999999,"15min":1.9490000000000001},"min":{"1min":0.27300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4649999999999999,"5min":9.4239999999999995,"15min":163.36699999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.3700000000000001,"15min":2.1629999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":2.23,"5min":10.705,"15min":163.381},"last":0.39400000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:41:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.101,"15min":1.651},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.94999999999999996,"5min":5.2539999999999996,"15min":106.801},"last":0.435},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.1559999999999999,"15min":1.694},"min":{"1min":0.34399999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.403,"5min":9.4749999999999996,"15min":106.827},"last":0.46000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757813,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2585936,"kb_used_data":1102344,"kb_used_omap":0,"kb_used_meta":1483584,"kb_avail":91159216,"statfs":{"total":95995035648,"available":93347037184,"internally_reserved":0,"allocated":1128800256,"data_stored":1127749951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519190016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0049999999999999,"5min":5.3049999999999997,"15min":6.0259999999999998},"min":{"1min":0.23000000000000001,"5min":0.21099999999999999,"15min":0.20799999999999999},"max":{"1min":45.753999999999998,"5min":147.405,"15min":294.89699999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":4.3419999999999996,"5min":6.4969999999999999,"15min":6.1820000000000004},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.215},"max":{"1min":44.658000000000001,"5min":146.31700000000001,"15min":294.30399999999997},"last":0.36299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":10.257999999999999,"15min":7.2409999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.22},"max":{"1min":5.8120000000000003,"5min":153.91300000000001,"15min":188.34100000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.2349999999999999,"5min":10.214,"15min":7.2910000000000004},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1369999999999996,"5min":154.023,"15min":188.839},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":5.0970000000000004,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.49,"5min":100.46599999999999,"15min":120.613},"last":0.88600000000000001},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.2919999999999998,"15min":3.9540000000000002},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.4610000000000003,"5min":100.56999999999999,"15min":120.962},"last":0.53400000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:41:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8799999999999999,"5min":6.9649999999999999,"15min":6.6870000000000003},"min":{"1min":0.28399999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":45.499000000000002,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":4.5350000000000001,"5min":6.7210000000000001,"15min":6.4930000000000003},"min":{"1min":0.34200000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":44.395000000000003,"5min":220.45500000000001,"15min":220.45500000000001},"last":3.7879999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.091,"15min":1.3799999999999999},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.247},"max":{"1min":2.7320000000000002,"5min":9.9290000000000003,"15min":79.043000000000006},"last":0.311},{"interface":"front","average":{"1min":1,"5min":1.298,"15min":1.4039999999999999},"min":{"1min":0.378,"5min":0.251,"15min":0.251},"max":{"1min":3.3180000000000001,"5min":8.3970000000000002,"15min":79.165999999999997},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.806,"15min":1.486},"min":{"1min":0.34200000000000003,"5min":0.251,"15min":0.251},"max":{"1min":3.7490000000000001,"5min":79.013999999999996,"15min":79.013999999999996},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.8069999999999999,"15min":1.569},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.7809999999999999,"5min":80.781999999999996,"15min":80.781999999999996},"last":0.40699999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.3700000000000001,"15min":1.5449999999999999},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":4.3280000000000003,"5min":11.146000000000001,"15min":78.890000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.3049999999999999,"15min":1.395},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.359,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.50700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1693974528,"data_stored":1693974528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1865940992,"data_stored":1865940992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1477443584,"data_stored":1477443584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1556152320,"data_stored":1556152320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1476333568,"data_stored":1476329496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1112608768,"data_stored":1112604696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":923078656,"data_stored":923074584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":756809728,"data_stored":756809728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:42:22.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:42:22.204 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:22.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:22.317 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:22.318 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:22.348 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:22.413 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:22.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:22.520 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:22.630 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:22.737 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:22.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:22.845 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:22.863 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:22 smithi107 ceph-mon[212750]: pgmap v1621: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 1066/145137 objects degraded (0.734%); 1680/145137 objects misplaced (1.158%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:22.864 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:22 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1066/145137 objects degraded (0.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:22.866 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:22 smithi107 ceph-mon[220921]: pgmap v1621: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 1066/145137 objects degraded (0.734%); 1680/145137 objects misplaced (1.158%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:22.866 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:22 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1066/145137 objects degraded (0.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:22.874 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:22.880 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:42:22.881 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:42:22.924 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:22.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:23.036 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:23.105 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:22 smithi150 ceph-mon[194178]: pgmap v1621: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 3.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 836 op/s; 1066/145137 objects degraded (0.734%); 1680/145137 objects misplaced (1.158%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T06:42:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:22 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1066/145137 objects degraded (0.734%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:23.705 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:23.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:23.781 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:42:23.818 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:23.861 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:23.905 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:23.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:23.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:24.010 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:24.067 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:24.889 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:24.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:24.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:25.070 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:24 smithi107 ceph-mon[220921]: pgmap v1622: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1047/139371 objects degraded (0.751%); 1047/139371 objects misplaced (0.751%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T06:42:25.071 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:24 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:42:25.073 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:24 smithi107 ceph-mon[212750]: pgmap v1622: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1047/139371 objects degraded (0.751%); 1047/139371 objects misplaced (0.751%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T06:42:25.073 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:24 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:42:25.076 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:24 smithi150 ceph-mon[194178]: pgmap v1622: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 1047/139371 objects degraded (0.751%); 1047/139371 objects misplaced (0.751%); 4.0 MiB/s, 65 objects/s recovering 2023-10-09T06:42:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:24 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:42:25.470 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:42:25.574 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:25.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:25.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:25.695 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:25.731 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:25.754 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:25.809 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:25 smithi107 ceph-mon[212750]: 14.8 scrub starts 2023-10-09T06:42:25.846 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:25.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:25.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:25.946 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:25.955 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:25.987 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:25 smithi150 ceph-mon[194178]: 14.8 scrub starts 2023-10-09T06:42:25.991 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:26.049 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:26.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:26.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:25 smithi107 ceph-mon[220921]: 14.8 scrub starts 2023-10-09T06:42:26.162 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:26.243 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:26.635 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:26.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:26.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:26.749 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:26.801 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:26.925 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:26.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:27.036 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:27.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:26 smithi150 ceph-mon[194178]: pgmap v1623: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:42:27.076 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:26 smithi150 ceph-mon[194178]: 14.0 scrub starts 2023-10-09T06:42:27.082 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:27.117 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:26 smithi107 ceph-mon[212750]: pgmap v1623: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:42:27.117 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:26 smithi107 ceph-mon[212750]: 14.0 scrub starts 2023-10-09T06:42:27.118 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:26 smithi107 ceph-mon[220921]: pgmap v1623: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.4 MiB/s, 55 objects/s recovering 2023-10-09T06:42:27.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:26 smithi107 ceph-mon[220921]: 14.0 scrub starts 2023-10-09T06:42:27.170 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:27.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:27.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:27.274 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:27.320 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:27.830 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:42:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3256128735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 1047/133566 objects degraded (0.784%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3256128735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:27 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:27 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 1047/133566 objects degraded (0.784%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:28.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3256128735' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:28.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:27 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 1047/133566 objects degraded (0.784%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:28.335 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:28.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:28.451 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:28.503 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:28.608 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:28.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:28.722 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:28.748 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: pgmap v1624: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.3 MiB/s, 54 objects/s recovering 2023-10-09T06:42:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:42:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: failed to encode map e524 with expected crc 2023-10-09T06:42:28.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:42:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: failed to encode map e524 with expected crc 2023-10-09T06:42:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: failed to encode map e524 with expected crc 2023-10-09T06:42:28.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:28 smithi150 ceph-mon[194178]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: pgmap v1624: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.3 MiB/s, 54 objects/s recovering 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[212750]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: pgmap v1624: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 1047/133566 objects degraded (0.784%); 1047/133566 objects misplaced (0.784%); 3.3 MiB/s, 54 objects/s recovering 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: osdmap e524: 8 total, 8 up, 8 in 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:28 smithi107 ceph-mon[220921]: failed to encode map e524 with expected crc 2023-10-09T06:42:29.465 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:29.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:29.576 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:29.651 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:29.697 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:29.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:29.801 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:29.855 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:30.284 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:30.295 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:30.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:30.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:30.464 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:30.547 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:30.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:30.652 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:30.758 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[212750]: pgmap v1626: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 615/126696 objects degraded (0.485%); 615/126696 objects misplaced (0.485%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[220921]: pgmap v1626: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 615/126696 objects degraded (0.485%); 615/126696 objects misplaced (0.485%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:30.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:31.021 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:31.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:31.133 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:30 smithi150 ceph-mon[194178]: pgmap v1626: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 615/126696 objects degraded (0.485%); 615/126696 objects misplaced (0.485%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:31.225 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:31.405 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:42:31.446 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:42:31.447 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:42:31.511 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:42:31.537 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:42:31.584 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:42:31.604 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:31.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:31.627 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:42:31.662 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:42:31.699 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:42:31.714 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:31.734 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:42:31.749 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.750 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:42:31.762 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:31.831 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:31.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:31.939 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:31.984 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:32.043 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:32.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:32.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:32.061 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:31 smithi150 ceph-mon[194178]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:32.061 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3908701522' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:32.062 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:32.062 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[212750]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3908701522' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:32] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[220921]: from='client.37254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3908701522' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:32.159 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.196 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:32.249 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:32.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:32.268 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:42:32.357 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:32.392 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:32.459 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:32.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:32.565 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:32.591 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:32.632 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:32.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:32.737 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:32.768 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:32.821 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:32.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:32.926 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:32.953 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:33.007 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:33.019 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:32 smithi150 ceph-mon[194178]: pgmap v1627: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 615/122223 objects degraded (0.503%); 615/122223 objects misplaced (0.503%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:33.019 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:32 smithi150 ceph-mon[194178]: 14.18 scrub starts 2023-10-09T06:42:33.019 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:32 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 615/122223 objects degraded (0.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:33.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:33.121 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[212750]: pgmap v1627: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 615/122223 objects degraded (0.503%); 615/122223 objects misplaced (0.503%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[212750]: 14.18 scrub starts 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 615/122223 objects degraded (0.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[220921]: pgmap v1627: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 615/122223 objects degraded (0.503%); 615/122223 objects misplaced (0.503%); 1.9 MiB/s, 75 objects/s recovering 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[220921]: 14.18 scrub starts 2023-10-09T06:42:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:32 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 615/122223 objects degraded (0.503%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:33.179 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:33.736 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:33.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:33.841 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:33.905 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:33.954 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1626,"stamp":"2023-10-09T06:42:29.625187+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2766673848,"num_objects":42232,"num_object_clones":0,"num_object_copies":126696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":615,"num_objects_misplaced":615,"num_objects_unfound":0,"num_objects_dirty":42232,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142350,"num_write_kb":5352990,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4467,"num_bytes_recovered":259663820,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":186226,"ondisk_log_size":186226,"up":387,"acting":386,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18528424,"kb_used_data":8698664,"kb_used_omap":20,"kb_used_meta":9829419,"kb_avail":731432792,"statfs":{"total":767960285184,"available":748987179008,"internally_reserved":0,"allocated":8907431936,"data_stored":8894180859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10065325710},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-517079040,"num_objects":-7890,"num_object_clones":0,"num_object_copies":-23670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-451,"num_objects_misplaced":-1065,"num_objects_unfound":0,"num_objects_dirty":-7890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7890,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":758,"num_bytes_recovered":20119552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006658"},"pg_stats":[{"pgid":"14.17","version":"523'7197","reported_seq":7366,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.592672+0000","last_change":"2023-10-09T06:40:52.372943+0000","last_active":"2023-10-09T06:42:27.592672+0000","last_peered":"2023-10-09T06:42:27.592672+0000","last_clean":"2023-10-09T06:42:27.592672+0000","last_became_active":"2023-10-09T06:40:43.185643+0000","last_became_peered":"2023-10-09T06:40:43.185643+0000","last_unstale":"2023-10-09T06:42:27.592672+0000","last_undegraded":"2023-10-09T06:42:27.592672+0000","last_fullsized":"2023-10-09T06:42:27.592672+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4783","last_scrub_stamp":"2023-10-09T06:40:52.372858+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:52.372858+0000","objects_scrubbed":2511,"log_size":7197,"log_dups_size":0,"ondisk_log_size":7197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:22.222358+0000","scrub_duration":0.27153366200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2414,"num_write_kb":35712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":817,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524575+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:42:05.524575+0000","last_peered":"2023-10-09T06:42:05.524575+0000","last_clean":"2023-10-09T06:42:05.524575+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:42:05.524575+0000","last_undegraded":"2023-10-09T06:42:05.524575+0000","last_fullsized":"2023-10-09T06:42:05.524575+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":502,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522605+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:42:05.522605+0000","last_peered":"2023-10-09T06:42:05.522605+0000","last_clean":"2023-10-09T06:42:05.522605+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:42:05.522605+0000","last_undegraded":"2023-10-09T06:42:05.522605+0000","last_fullsized":"2023-10-09T06:42:05.522605+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":226,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749451+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:42:13.749451+0000","last_peered":"2023-10-09T06:42:13.749451+0000","last_clean":"2023-10-09T06:42:13.749451+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:42:13.749451+0000","last_undegraded":"2023-10-09T06:42:13.749451+0000","last_fullsized":"2023-10-09T06:42:13.749451+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"523'7109","reported_seq":7177,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.780662+0000","last_change":"2023-10-09T06:41:49.571613+0000","last_active":"2023-10-09T06:42:27.780662+0000","last_peered":"2023-10-09T06:42:27.780662+0000","last_clean":"2023-10-09T06:42:27.780662+0000","last_became_active":"2023-10-09T06:40:43.089707+0000","last_became_peered":"2023-10-09T06:40:43.089707+0000","last_unstale":"2023-10-09T06:42:27.780662+0000","last_undegraded":"2023-10-09T06:42:27.780662+0000","last_fullsized":"2023-10-09T06:42:27.780662+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7109,"log_dups_size":0,"ondisk_log_size":7109,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":194560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":778,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522608+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:42:05.522608+0000","last_peered":"2023-10-09T06:42:05.522608+0000","last_clean":"2023-10-09T06:42:05.522608+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:42:05.522608+0000","last_undegraded":"2023-10-09T06:42:05.522608+0000","last_fullsized":"2023-10-09T06:42:05.522608+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":521,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.743363+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:42:13.743363+0000","last_peered":"2023-10-09T06:42:13.743363+0000","last_clean":"2023-10-09T06:42:13.743363+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:42:13.743363+0000","last_undegraded":"2023-10-09T06:42:13.743363+0000","last_fullsized":"2023-10-09T06:42:13.743363+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":525,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.735712+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:42:13.735712+0000","last_peered":"2023-10-09T06:42:13.735712+0000","last_clean":"2023-10-09T06:42:13.735712+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:42:13.735712+0000","last_undegraded":"2023-10-09T06:42:13.735712+0000","last_fullsized":"2023-10-09T06:42:13.735712+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"523'7105","reported_seq":7158,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.791321+0000","last_change":"2023-10-09T06:40:43.168550+0000","last_active":"2023-10-09T06:42:24.791321+0000","last_peered":"2023-10-09T06:42:24.791321+0000","last_clean":"2023-10-09T06:42:24.791321+0000","last_became_active":"2023-10-09T06:40:43.166693+0000","last_became_peered":"2023-10-09T06:40:43.166693+0000","last_unstale":"2023-10-09T06:42:24.791321+0000","last_undegraded":"2023-10-09T06:42:24.791321+0000","last_fullsized":"2023-10-09T06:42:24.791321+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7105,"log_dups_size":0,"ondisk_log_size":7105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87031808,"num_objects":1328,"num_object_clones":0,"num_object_copies":3984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":770,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522550+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:42:05.522550+0000","last_peered":"2023-10-09T06:42:05.522550+0000","last_clean":"2023-10-09T06:42:05.522550+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:42:05.522550+0000","last_undegraded":"2023-10-09T06:42:05.522550+0000","last_fullsized":"2023-10-09T06:42:05.522550+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":245,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.106461+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:42:14.106461+0000","last_peered":"2023-10-09T06:42:14.106461+0000","last_clean":"2023-10-09T06:42:14.106461+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:42:14.106461+0000","last_undegraded":"2023-10-09T06:42:14.106461+0000","last_fullsized":"2023-10-09T06:42:14.106461+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":513,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.759159+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:42:13.759159+0000","last_peered":"2023-10-09T06:42:13.759159+0000","last_clean":"2023-10-09T06:42:13.759159+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:42:13.759159+0000","last_undegraded":"2023-10-09T06:42:13.759159+0000","last_fullsized":"2023-10-09T06:42:13.759159+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":631,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525226+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:42:05.525226+0000","last_peered":"2023-10-09T06:42:05.525226+0000","last_clean":"2023-10-09T06:42:05.525226+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:42:05.525226+0000","last_undegraded":"2023-10-09T06:42:05.525226+0000","last_fullsized":"2023-10-09T06:42:05.525226+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":199,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516078+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:42:05.516078+0000","last_peered":"2023-10-09T06:42:05.516078+0000","last_clean":"2023-10-09T06:42:05.516078+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:42:05.516078+0000","last_undegraded":"2023-10-09T06:42:05.516078+0000","last_fullsized":"2023-10-09T06:42:05.516078+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749048+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:42:13.749048+0000","last_peered":"2023-10-09T06:42:13.749048+0000","last_clean":"2023-10-09T06:42:13.749048+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:42:13.749048+0000","last_undegraded":"2023-10-09T06:42:13.749048+0000","last_fullsized":"2023-10-09T06:42:13.749048+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":668,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525484+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:42:05.525484+0000","last_peered":"2023-10-09T06:42:05.525484+0000","last_clean":"2023-10-09T06:42:05.525484+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:42:05.525484+0000","last_undegraded":"2023-10-09T06:42:05.525484+0000","last_fullsized":"2023-10-09T06:42:05.525484+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":210,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.543535+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:42:05.543535+0000","last_peered":"2023-10-09T06:42:05.543535+0000","last_clean":"2023-10-09T06:42:05.543535+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:42:05.543535+0000","last_undegraded":"2023-10-09T06:42:05.543535+0000","last_fullsized":"2023-10-09T06:42:05.543535+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.753653+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:42:13.753653+0000","last_peered":"2023-10-09T06:42:13.753653+0000","last_clean":"2023-10-09T06:42:13.753653+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:42:13.753653+0000","last_undegraded":"2023-10-09T06:42:13.753653+0000","last_fullsized":"2023-10-09T06:42:13.753653+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"523'7492","reported_seq":7543,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.748473+0000","last_change":"2023-10-09T06:40:43.182904+0000","last_active":"2023-10-09T06:42:24.748473+0000","last_peered":"2023-10-09T06:42:24.748473+0000","last_clean":"2023-10-09T06:42:24.748473+0000","last_became_active":"2023-10-09T06:40:43.180666+0000","last_became_peered":"2023-10-09T06:40:43.180666+0000","last_unstale":"2023-10-09T06:42:24.748473+0000","last_undegraded":"2023-10-09T06:42:24.748473+0000","last_fullsized":"2023-10-09T06:42:24.748473+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7492,"log_dups_size":0,"ondisk_log_size":7492,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93290496,"num_objects":1424,"num_object_clones":0,"num_object_copies":4272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5102,"num_write_kb":208800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":652,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515798+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:42:05.515798+0000","last_peered":"2023-10-09T06:42:05.515798+0000","last_clean":"2023-10-09T06:42:05.515798+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:42:05.515798+0000","last_undegraded":"2023-10-09T06:42:05.515798+0000","last_fullsized":"2023-10-09T06:42:05.515798+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":404,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525463+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:42:05.525463+0000","last_peered":"2023-10-09T06:42:05.525463+0000","last_clean":"2023-10-09T06:42:05.525463+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:42:05.525463+0000","last_undegraded":"2023-10-09T06:42:05.525463+0000","last_fullsized":"2023-10-09T06:42:05.525463+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":462,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.741795+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:42:13.741795+0000","last_peered":"2023-10-09T06:42:13.741795+0000","last_clean":"2023-10-09T06:42:13.741795+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:42:13.741795+0000","last_undegraded":"2023-10-09T06:42:13.741795+0000","last_fullsized":"2023-10-09T06:42:13.741795+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"524'7327","reported_seq":7443,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T06:42:28.999663+0000","last_change":"2023-10-09T06:40:50.305746+0000","last_active":"2023-10-09T06:42:28.999663+0000","last_peered":"2023-10-09T06:42:28.999663+0000","last_clean":"2023-10-09T06:42:28.999663+0000","last_became_active":"2023-10-09T06:40:43.093584+0000","last_became_peered":"2023-10-09T06:40:43.093584+0000","last_unstale":"2023-10-09T06:42:28.999663+0000","last_undegraded":"2023-10-09T06:42:28.999663+0000","last_fullsized":"2023-10-09T06:42:28.999663+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4832","last_scrub_stamp":"2023-10-09T06:40:50.305626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:50.305626+0000","objects_scrubbed":2516,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:57.896131+0000","scrub_duration":0.56208410600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78905344,"num_objects":1204,"num_object_clones":0,"num_object_copies":3612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2504,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":634,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737549+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:42:13.737549+0000","last_peered":"2023-10-09T06:42:13.737549+0000","last_clean":"2023-10-09T06:42:13.737549+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:42:13.737549+0000","last_undegraded":"2023-10-09T06:42:13.737549+0000","last_fullsized":"2023-10-09T06:42:13.737549+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":529,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523200+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:42:05.523200+0000","last_peered":"2023-10-09T06:42:05.523200+0000","last_clean":"2023-10-09T06:42:05.523200+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:42:05.523200+0000","last_undegraded":"2023-10-09T06:42:05.523200+0000","last_fullsized":"2023-10-09T06:42:05.523200+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":493,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523146+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:42:05.523146+0000","last_peered":"2023-10-09T06:42:05.523146+0000","last_clean":"2023-10-09T06:42:05.523146+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:42:05.523146+0000","last_undegraded":"2023-10-09T06:42:05.523146+0000","last_fullsized":"2023-10-09T06:42:05.523146+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":797,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515686+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:42:05.515686+0000","last_peered":"2023-10-09T06:42:05.515686+0000","last_clean":"2023-10-09T06:42:05.515686+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:42:05.515686+0000","last_undegraded":"2023-10-09T06:42:05.515686+0000","last_fullsized":"2023-10-09T06:42:05.515686+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":372,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525051+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:42:05.525051+0000","last_peered":"2023-10-09T06:42:05.525051+0000","last_clean":"2023-10-09T06:42:05.525051+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:42:05.525051+0000","last_undegraded":"2023-10-09T06:42:05.525051+0000","last_fullsized":"2023-10-09T06:42:05.525051+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.752576+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:42:13.752576+0000","last_peered":"2023-10-09T06:42:13.752576+0000","last_clean":"2023-10-09T06:42:13.752576+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:42:13.752576+0000","last_undegraded":"2023-10-09T06:42:13.752576+0000","last_fullsized":"2023-10-09T06:42:13.752576+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":795,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524617+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:42:05.524617+0000","last_peered":"2023-10-09T06:42:05.524617+0000","last_clean":"2023-10-09T06:42:05.524617+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:42:05.524617+0000","last_undegraded":"2023-10-09T06:42:05.524617+0000","last_fullsized":"2023-10-09T06:42:05.524617+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":522,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516196+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:42:05.516196+0000","last_peered":"2023-10-09T06:42:05.516196+0000","last_clean":"2023-10-09T06:42:05.516196+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:42:05.516196+0000","last_undegraded":"2023-10-09T06:42:05.516196+0000","last_fullsized":"2023-10-09T06:42:05.516196+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":343,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525371+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:42:05.525371+0000","last_peered":"2023-10-09T06:42:05.525371+0000","last_clean":"2023-10-09T06:42:05.525371+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:42:05.525371+0000","last_undegraded":"2023-10-09T06:42:05.525371+0000","last_fullsized":"2023-10-09T06:42:05.525371+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":852,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737071+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:42:13.737071+0000","last_peered":"2023-10-09T06:42:13.737071+0000","last_clean":"2023-10-09T06:42:13.737071+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:42:13.737071+0000","last_undegraded":"2023-10-09T06:42:13.737071+0000","last_fullsized":"2023-10-09T06:42:13.737071+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":366,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524723+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:42:05.524723+0000","last_peered":"2023-10-09T06:42:05.524723+0000","last_clean":"2023-10-09T06:42:05.524723+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:42:05.524723+0000","last_undegraded":"2023-10-09T06:42:05.524723+0000","last_fullsized":"2023-10-09T06:42:05.524723+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":199,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516187+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:42:05.516187+0000","last_peered":"2023-10-09T06:42:05.516187+0000","last_clean":"2023-10-09T06:42:05.516187+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:42:05.516187+0000","last_undegraded":"2023-10-09T06:42:05.516187+0000","last_fullsized":"2023-10-09T06:42:05.516187+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":637,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.544565+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:42:05.544565+0000","last_peered":"2023-10-09T06:42:05.544565+0000","last_clean":"2023-10-09T06:42:05.544565+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:42:05.544565+0000","last_undegraded":"2023-10-09T06:42:05.544565+0000","last_fullsized":"2023-10-09T06:42:05.544565+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":602,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.758510+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:42:13.758510+0000","last_peered":"2023-10-09T06:42:13.758510+0000","last_clean":"2023-10-09T06:42:13.758510+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:42:13.758510+0000","last_undegraded":"2023-10-09T06:42:13.758510+0000","last_fullsized":"2023-10-09T06:42:13.758510+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.752175+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:42:13.752175+0000","last_peered":"2023-10-09T06:42:13.752175+0000","last_clean":"2023-10-09T06:42:13.752175+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:42:13.752175+0000","last_undegraded":"2023-10-09T06:42:13.752175+0000","last_fullsized":"2023-10-09T06:42:13.752175+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"523'7266","reported_seq":7344,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.788343+0000","last_change":"2023-10-09T06:41:45.688515+0000","last_active":"2023-10-09T06:42:27.788343+0000","last_peered":"2023-10-09T06:42:27.788343+0000","last_clean":"2023-10-09T06:42:27.788343+0000","last_became_active":"2023-10-09T06:40:43.094441+0000","last_became_peered":"2023-10-09T06:40:43.094441+0000","last_unstale":"2023-10-09T06:42:27.788343+0000","last_undegraded":"2023-10-09T06:42:27.788343+0000","last_fullsized":"2023-10-09T06:42:27.788343+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7266,"log_dups_size":0,"ondisk_log_size":7266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82870272,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4954,"num_write_kb":199008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":650,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525266+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:42:05.525266+0000","last_peered":"2023-10-09T06:42:05.525266+0000","last_clean":"2023-10-09T06:42:05.525266+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:42:05.525266+0000","last_undegraded":"2023-10-09T06:42:05.525266+0000","last_fullsized":"2023-10-09T06:42:05.525266+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":408,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525639+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:42:05.525639+0000","last_peered":"2023-10-09T06:42:05.525639+0000","last_clean":"2023-10-09T06:42:05.525639+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:42:05.525639+0000","last_undegraded":"2023-10-09T06:42:05.525639+0000","last_fullsized":"2023-10-09T06:42:05.525639+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":306,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524805+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:42:05.524805+0000","last_peered":"2023-10-09T06:42:05.524805+0000","last_clean":"2023-10-09T06:42:05.524805+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:42:05.524805+0000","last_undegraded":"2023-10-09T06:42:05.524805+0000","last_fullsized":"2023-10-09T06:42:05.524805+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"523'7272","reported_seq":7327,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.698831+0000","last_change":"2023-10-09T06:42:15.187041+0000","last_active":"2023-10-09T06:42:24.698831+0000","last_peered":"2023-10-09T06:42:24.698831+0000","last_clean":"2023-10-09T06:42:24.698831+0000","last_became_active":"2023-10-09T06:40:43.165023+0000","last_became_peered":"2023-10-09T06:40:43.165023+0000","last_unstale":"2023-10-09T06:42:24.698831+0000","last_undegraded":"2023-10-09T06:42:24.698831+0000","last_fullsized":"2023-10-09T06:42:24.698831+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7272,"log_dups_size":0,"ondisk_log_size":7272,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4952,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":911,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749614+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:42:13.749614+0000","last_peered":"2023-10-09T06:42:13.749614+0000","last_clean":"2023-10-09T06:42:13.749614+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:42:13.749614+0000","last_undegraded":"2023-10-09T06:42:13.749614+0000","last_fullsized":"2023-10-09T06:42:13.749614+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":370,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525602+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:42:05.525602+0000","last_peered":"2023-10-09T06:42:05.525602+0000","last_clean":"2023-10-09T06:42:05.525602+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:42:05.525602+0000","last_undegraded":"2023-10-09T06:42:05.525602+0000","last_fullsized":"2023-10-09T06:42:05.525602+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":520,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522780+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:42:05.522780+0000","last_peered":"2023-10-09T06:42:05.522780+0000","last_clean":"2023-10-09T06:42:05.522780+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:42:05.522780+0000","last_undegraded":"2023-10-09T06:42:05.522780+0000","last_fullsized":"2023-10-09T06:42:05.522780+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"523'7185","reported_seq":7239,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.787414+0000","last_change":"2023-10-09T06:40:43.082575+0000","last_active":"2023-10-09T06:42:24.787414+0000","last_peered":"2023-10-09T06:42:24.787414+0000","last_clean":"2023-10-09T06:42:24.787414+0000","last_became_active":"2023-10-09T06:40:43.080150+0000","last_became_peered":"2023-10-09T06:40:43.080150+0000","last_unstale":"2023-10-09T06:42:24.787414+0000","last_undegraded":"2023-10-09T06:42:24.787414+0000","last_fullsized":"2023-10-09T06:42:24.787414+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7185,"log_dups_size":0,"ondisk_log_size":7185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90603520,"num_objects":1382,"num_object_clones":0,"num_object_copies":4146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4894,"num_write_kb":200864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":825,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.759106+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:42:13.759106+0000","last_peered":"2023-10-09T06:42:13.759106+0000","last_clean":"2023-10-09T06:42:13.759106+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:42:13.759106+0000","last_undegraded":"2023-10-09T06:42:13.759106+0000","last_fullsized":"2023-10-09T06:42:13.759106+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":426,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525352+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:42:05.525352+0000","last_peered":"2023-10-09T06:42:05.525352+0000","last_clean":"2023-10-09T06:42:05.525352+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:42:05.525352+0000","last_undegraded":"2023-10-09T06:42:05.525352+0000","last_fullsized":"2023-10-09T06:42:05.525352+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":500,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.758879+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:42:13.758879+0000","last_peered":"2023-10-09T06:42:13.758879+0000","last_clean":"2023-10-09T06:42:13.758879+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:42:13.758879+0000","last_undegraded":"2023-10-09T06:42:13.758879+0000","last_fullsized":"2023-10-09T06:42:13.758879+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"524'7355","reported_seq":7474,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T06:42:29.043824+0000","last_change":"2023-10-09T06:40:47.766598+0000","last_active":"2023-10-09T06:42:29.043824+0000","last_peered":"2023-10-09T06:42:29.043824+0000","last_clean":"2023-10-09T06:42:29.043824+0000","last_became_active":"2023-10-09T06:40:43.101954+0000","last_became_peered":"2023-10-09T06:40:43.101954+0000","last_unstale":"2023-10-09T06:42:29.043824+0000","last_undegraded":"2023-10-09T06:42:29.043824+0000","last_fullsized":"2023-10-09T06:42:29.043824+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4698","last_scrub_stamp":"2023-10-09T06:40:47.766356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.766356+0000","objects_scrubbed":2441,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:35.858607+0000","scrub_duration":1.024576309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2667,"num_write_kb":48768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":647,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523336+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:42:05.523336+0000","last_peered":"2023-10-09T06:42:05.523336+0000","last_clean":"2023-10-09T06:42:05.523336+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:42:05.523336+0000","last_undegraded":"2023-10-09T06:42:05.523336+0000","last_fullsized":"2023-10-09T06:42:05.523336+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.752657+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:42:13.752657+0000","last_peered":"2023-10-09T06:42:13.752657+0000","last_clean":"2023-10-09T06:42:13.752657+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:42:13.752657+0000","last_undegraded":"2023-10-09T06:42:13.752657+0000","last_fullsized":"2023-10-09T06:42:13.752657+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":487,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516704+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:42:05.516704+0000","last_peered":"2023-10-09T06:42:05.516704+0000","last_clean":"2023-10-09T06:42:05.516704+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:42:05.516704+0000","last_undegraded":"2023-10-09T06:42:05.516704+0000","last_fullsized":"2023-10-09T06:42:05.516704+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"523'7137","reported_seq":7191,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.826621+0000","last_change":"2023-10-09T06:42:18.037795+0000","last_active":"2023-10-09T06:42:24.826621+0000","last_peered":"2023-10-09T06:42:24.826621+0000","last_clean":"2023-10-09T06:42:24.826621+0000","last_became_active":"2023-10-09T06:40:43.159937+0000","last_became_peered":"2023-10-09T06:40:43.159937+0000","last_unstale":"2023-10-09T06:42:24.826621+0000","last_undegraded":"2023-10-09T06:42:24.826621+0000","last_fullsized":"2023-10-09T06:42:24.826621+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7137,"log_dups_size":0,"ondisk_log_size":7137,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91160576,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":875,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524641+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:42:05.524641+0000","last_peered":"2023-10-09T06:42:05.524641+0000","last_clean":"2023-10-09T06:42:05.524641+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:42:05.524641+0000","last_undegraded":"2023-10-09T06:42:05.524641+0000","last_fullsized":"2023-10-09T06:42:05.524641+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":253,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.748937+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:42:13.748937+0000","last_peered":"2023-10-09T06:42:13.748937+0000","last_clean":"2023-10-09T06:42:13.748937+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:42:13.748937+0000","last_undegraded":"2023-10-09T06:42:13.748937+0000","last_fullsized":"2023-10-09T06:42:13.748937+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":503,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523154+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:42:05.523154+0000","last_peered":"2023-10-09T06:42:05.523154+0000","last_clean":"2023-10-09T06:42:05.523154+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:42:05.523154+0000","last_undegraded":"2023-10-09T06:42:05.523154+0000","last_fullsized":"2023-10-09T06:42:05.523154+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"523'7313","reported_seq":7471,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.540852+0000","last_change":"2023-10-09T06:40:45.736296+0000","last_active":"2023-10-09T06:42:27.540852+0000","last_peered":"2023-10-09T06:42:27.540852+0000","last_clean":"2023-10-09T06:42:27.540852+0000","last_became_active":"2023-10-09T06:40:43.168245+0000","last_became_peered":"2023-10-09T06:40:43.168245+0000","last_unstale":"2023-10-09T06:42:27.540852+0000","last_undegraded":"2023-10-09T06:42:27.540852+0000","last_fullsized":"2023-10-09T06:42:27.540852+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4658","last_scrub_stamp":"2023-10-09T06:40:45.736230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:45.736230+0000","objects_scrubbed":2357,"log_size":7313,"log_dups_size":0,"ondisk_log_size":7313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:31.145769+0000","scrub_duration":0.58060363000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86441984,"num_objects":1319,"num_object_clones":0,"num_object_copies":3957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2664,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":559,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749123+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:42:13.749123+0000","last_peered":"2023-10-09T06:42:13.749123+0000","last_clean":"2023-10-09T06:42:13.749123+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:42:13.749123+0000","last_undegraded":"2023-10-09T06:42:13.749123+0000","last_fullsized":"2023-10-09T06:42:13.749123+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":481,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.758805+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:42:13.758805+0000","last_peered":"2023-10-09T06:42:13.758805+0000","last_clean":"2023-10-09T06:42:13.758805+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:42:13.758805+0000","last_undegraded":"2023-10-09T06:42:13.758805+0000","last_fullsized":"2023-10-09T06:42:13.758805+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":500,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523440+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:42:05.523440+0000","last_peered":"2023-10-09T06:42:05.523440+0000","last_clean":"2023-10-09T06:42:05.523440+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:42:05.523440+0000","last_undegraded":"2023-10-09T06:42:05.523440+0000","last_fullsized":"2023-10-09T06:42:05.523440+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"523'7193","reported_seq":7357,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.757621+0000","last_change":"2023-10-09T06:40:51.573557+0000","last_active":"2023-10-09T06:42:27.757621+0000","last_peered":"2023-10-09T06:42:27.757621+0000","last_clean":"2023-10-09T06:42:27.757621+0000","last_became_active":"2023-10-09T06:40:43.081959+0000","last_became_peered":"2023-10-09T06:40:43.081959+0000","last_unstale":"2023-10-09T06:42:27.757621+0000","last_undegraded":"2023-10-09T06:42:27.757621+0000","last_fullsized":"2023-10-09T06:42:27.757621+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'4739","last_scrub_stamp":"2023-10-09T06:40:51.573471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:51.573471+0000","objects_scrubbed":2521,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:30.924217+0000","scrub_duration":0.34142884899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2454,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":861,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737148+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:42:13.737148+0000","last_peered":"2023-10-09T06:42:13.737148+0000","last_clean":"2023-10-09T06:42:13.737148+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:42:13.737148+0000","last_undegraded":"2023-10-09T06:42:13.737148+0000","last_fullsized":"2023-10-09T06:42:13.737148+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":504,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522689+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:42:05.522689+0000","last_peered":"2023-10-09T06:42:05.522689+0000","last_clean":"2023-10-09T06:42:05.522689+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:42:05.522689+0000","last_undegraded":"2023-10-09T06:42:05.522689+0000","last_fullsized":"2023-10-09T06:42:05.522689+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":483,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523489+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:42:05.523489+0000","last_peered":"2023-10-09T06:42:05.523489+0000","last_clean":"2023-10-09T06:42:05.523489+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:42:05.523489+0000","last_undegraded":"2023-10-09T06:42:05.523489+0000","last_fullsized":"2023-10-09T06:42:05.523489+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"523'7067","reported_seq":7181,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.806591+0000","last_change":"2023-10-09T06:40:47.192403+0000","last_active":"2023-10-09T06:42:24.806591+0000","last_peered":"2023-10-09T06:42:24.806591+0000","last_clean":"2023-10-09T06:42:24.806591+0000","last_became_active":"2023-10-09T06:40:43.074962+0000","last_became_peered":"2023-10-09T06:40:43.074962+0000","last_unstale":"2023-10-09T06:42:24.806591+0000","last_undegraded":"2023-10-09T06:42:24.806591+0000","last_fullsized":"2023-10-09T06:42:24.806591+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4664","last_scrub_stamp":"2023-10-09T06:40:47.192137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.192137+0000","objects_scrubbed":2418,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:24.365070+0000","scrub_duration":0.455376642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2406,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":486,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.758711+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:42:13.758711+0000","last_peered":"2023-10-09T06:42:13.758711+0000","last_clean":"2023-10-09T06:42:13.758711+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:42:13.758711+0000","last_undegraded":"2023-10-09T06:42:13.758711+0000","last_fullsized":"2023-10-09T06:42:13.758711+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.748978+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:42:13.748978+0000","last_peered":"2023-10-09T06:42:13.748978+0000","last_clean":"2023-10-09T06:42:13.748978+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:42:13.748978+0000","last_undegraded":"2023-10-09T06:42:13.748978+0000","last_fullsized":"2023-10-09T06:42:13.748978+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":481,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:18.204333+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:42:18.204333+0000","last_peered":"2023-10-09T06:42:18.204333+0000","last_clean":"2023-10-09T06:42:18.204333+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:42:18.204333+0000","last_undegraded":"2023-10-09T06:42:18.204333+0000","last_fullsized":"2023-10-09T06:42:18.204333+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"523'7524","reported_seq":7576,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.775564+0000","last_change":"2023-10-09T06:40:43.157665+0000","last_active":"2023-10-09T06:42:24.775564+0000","last_peered":"2023-10-09T06:42:24.775564+0000","last_clean":"2023-10-09T06:42:24.775564+0000","last_became_active":"2023-10-09T06:40:43.080919+0000","last_became_peered":"2023-10-09T06:40:43.080919+0000","last_unstale":"2023-10-09T06:42:24.775564+0000","last_undegraded":"2023-10-09T06:42:24.775564+0000","last_fullsized":"2023-10-09T06:42:24.775564+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90800128,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5133,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":816,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.759201+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:42:13.759201+0000","last_peered":"2023-10-09T06:42:13.759201+0000","last_clean":"2023-10-09T06:42:13.759201+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:42:13.759201+0000","last_undegraded":"2023-10-09T06:42:13.759201+0000","last_fullsized":"2023-10-09T06:42:13.759201+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":533,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.736909+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:42:13.736909+0000","last_peered":"2023-10-09T06:42:13.736909+0000","last_clean":"2023-10-09T06:42:13.736909+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:42:13.736909+0000","last_undegraded":"2023-10-09T06:42:13.736909+0000","last_fullsized":"2023-10-09T06:42:13.736909+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.749383+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:42:13.749383+0000","last_peered":"2023-10-09T06:42:13.749383+0000","last_clean":"2023-10-09T06:42:13.749383+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:42:13.749383+0000","last_undegraded":"2023-10-09T06:42:13.749383+0000","last_fullsized":"2023-10-09T06:42:13.749383+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":467,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737719+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:42:13.737719+0000","last_peered":"2023-10-09T06:42:13.737719+0000","last_clean":"2023-10-09T06:42:13.737719+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:42:13.737719+0000","last_undegraded":"2023-10-09T06:42:13.737719+0000","last_fullsized":"2023-10-09T06:42:13.737719+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"523'9930","reported_seq":9972,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.672937+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:42:27.672937+0000","last_peered":"2023-10-09T06:42:27.672937+0000","last_clean":"2023-10-09T06:42:27.672937+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:42:27.672937+0000","last_undegraded":"2023-10-09T06:42:27.672937+0000","last_fullsized":"2023-10-09T06:42:27.672937+0000","mapping_epoch":489,"log_start":"497'4300","ondisk_log_start":"497'4300","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170917888,"num_objects":2608,"num_object_clones":0,"num_object_copies":7824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9930,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":703,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515942+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:42:05.515942+0000","last_peered":"2023-10-09T06:42:05.515942+0000","last_clean":"2023-10-09T06:42:05.515942+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:42:05.515942+0000","last_undegraded":"2023-10-09T06:42:05.515942+0000","last_fullsized":"2023-10-09T06:42:05.515942+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":256,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.741616+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:42:13.741616+0000","last_peered":"2023-10-09T06:42:13.741616+0000","last_clean":"2023-10-09T06:42:13.741616+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:42:13.741616+0000","last_undegraded":"2023-10-09T06:42:13.741616+0000","last_fullsized":"2023-10-09T06:42:13.741616+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":504,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523067+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:42:05.523067+0000","last_peered":"2023-10-09T06:42:05.523067+0000","last_clean":"2023-10-09T06:42:05.523067+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:42:05.523067+0000","last_undegraded":"2023-10-09T06:42:05.523067+0000","last_fullsized":"2023-10-09T06:42:05.523067+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":497,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.736835+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:42:13.736835+0000","last_peered":"2023-10-09T06:42:13.736835+0000","last_clean":"2023-10-09T06:42:13.736835+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:42:13.736835+0000","last_undegraded":"2023-10-09T06:42:13.736835+0000","last_fullsized":"2023-10-09T06:42:13.736835+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"523'9879","reported_seq":9923,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.787835+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:42:27.787835+0000","last_peered":"2023-10-09T06:42:27.787835+0000","last_clean":"2023-10-09T06:42:27.787835+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:42:27.787835+0000","last_undegraded":"2023-10-09T06:42:27.787835+0000","last_fullsized":"2023-10-09T06:42:27.787835+0000","mapping_epoch":489,"log_start":"496'3400","ondisk_log_start":"496'3400","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937176,"num_objects":2533,"num_object_clones":0,"num_object_copies":7599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2533,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9879,"num_write_kb":397121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":865,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524860+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:42:05.524860+0000","last_peered":"2023-10-09T06:42:05.524860+0000","last_clean":"2023-10-09T06:42:05.524860+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:42:05.524860+0000","last_undegraded":"2023-10-09T06:42:05.524860+0000","last_fullsized":"2023-10-09T06:42:05.524860+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":518,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.753432+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:42:13.753432+0000","last_peered":"2023-10-09T06:42:13.753432+0000","last_clean":"2023-10-09T06:42:13.753432+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:42:13.753432+0000","last_undegraded":"2023-10-09T06:42:13.753432+0000","last_fullsized":"2023-10-09T06:42:13.753432+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":505,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.740919+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:42:13.740919+0000","last_peered":"2023-10-09T06:42:13.740919+0000","last_clean":"2023-10-09T06:42:13.740919+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:42:13.740919+0000","last_undegraded":"2023-10-09T06:42:13.740919+0000","last_fullsized":"2023-10-09T06:42:13.740919+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":527,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.741147+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:42:13.741147+0000","last_peered":"2023-10-09T06:42:13.741147+0000","last_clean":"2023-10-09T06:42:13.741147+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:42:13.741147+0000","last_undegraded":"2023-10-09T06:42:13.741147+0000","last_fullsized":"2023-10-09T06:42:13.741147+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"523'10127","reported_seq":10169,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.686861+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:42:27.686861+0000","last_peered":"2023-10-09T06:42:27.686861+0000","last_clean":"2023-10-09T06:42:27.686861+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:42:27.686861+0000","last_undegraded":"2023-10-09T06:42:27.686861+0000","last_fullsized":"2023-10-09T06:42:27.686861+0000","mapping_epoch":489,"log_start":"497'4000","ondisk_log_start":"497'4000","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176095232,"num_objects":2687,"num_object_clones":0,"num_object_copies":8061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10127,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":332,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525005+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:42:05.525005+0000","last_peered":"2023-10-09T06:42:05.525005+0000","last_clean":"2023-10-09T06:42:05.525005+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:42:05.525005+0000","last_undegraded":"2023-10-09T06:42:05.525005+0000","last_fullsized":"2023-10-09T06:42:05.525005+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":846,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524830+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:42:05.524830+0000","last_peered":"2023-10-09T06:42:05.524830+0000","last_clean":"2023-10-09T06:42:05.524830+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:42:05.524830+0000","last_undegraded":"2023-10-09T06:42:05.524830+0000","last_fullsized":"2023-10-09T06:42:05.524830+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":237,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.106620+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:42:14.106620+0000","last_peered":"2023-10-09T06:42:14.106620+0000","last_clean":"2023-10-09T06:42:14.106620+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:42:14.106620+0000","last_undegraded":"2023-10-09T06:42:14.106620+0000","last_fullsized":"2023-10-09T06:42:14.106620+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":486,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.758991+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:42:13.758991+0000","last_peered":"2023-10-09T06:42:13.758991+0000","last_clean":"2023-10-09T06:42:13.758991+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:42:13.758991+0000","last_undegraded":"2023-10-09T06:42:13.758991+0000","last_fullsized":"2023-10-09T06:42:13.758991+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"523'9967","reported_seq":10011,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.747366+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:42:24.747366+0000","last_peered":"2023-10-09T06:42:24.747366+0000","last_clean":"2023-10-09T06:42:24.747366+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:42:24.747366+0000","last_undegraded":"2023-10-09T06:42:24.747366+0000","last_fullsized":"2023-10-09T06:42:24.747366+0000","mapping_epoch":489,"log_start":"497'3900","ondisk_log_start":"497'3900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183959552,"num_objects":2807,"num_object_clones":0,"num_object_copies":8421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9967,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":836,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737223+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:42:13.737223+0000","last_peered":"2023-10-09T06:42:13.737223+0000","last_clean":"2023-10-09T06:42:13.737223+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:42:13.737223+0000","last_undegraded":"2023-10-09T06:42:13.737223+0000","last_fullsized":"2023-10-09T06:42:13.737223+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":386,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.524707+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:42:05.524707+0000","last_peered":"2023-10-09T06:42:05.524707+0000","last_clean":"2023-10-09T06:42:05.524707+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:42:05.524707+0000","last_undegraded":"2023-10-09T06:42:05.524707+0000","last_fullsized":"2023-10-09T06:42:05.524707+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":200,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.515865+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:42:05.515865+0000","last_peered":"2023-10-09T06:42:05.515865+0000","last_clean":"2023-10-09T06:42:05.515865+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:42:05.515865+0000","last_undegraded":"2023-10-09T06:42:05.515865+0000","last_fullsized":"2023-10-09T06:42:05.515865+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516605+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:42:05.516605+0000","last_peered":"2023-10-09T06:42:05.516605+0000","last_clean":"2023-10-09T06:42:05.516605+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:42:05.516605+0000","last_undegraded":"2023-10-09T06:42:05.516605+0000","last_fullsized":"2023-10-09T06:42:05.516605+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"524'7392","reported_seq":7507,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T06:42:29.024054+0000","last_change":"2023-10-09T06:40:48.261698+0000","last_active":"2023-10-09T06:42:29.024054+0000","last_peered":"2023-10-09T06:42:29.024054+0000","last_clean":"2023-10-09T06:42:29.024054+0000","last_became_active":"2023-10-09T06:40:43.090550+0000","last_became_peered":"2023-10-09T06:40:43.090550+0000","last_unstale":"2023-10-09T06:42:29.024054+0000","last_undegraded":"2023-10-09T06:42:29.024054+0000","last_fullsized":"2023-10-09T06:42:29.024054+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4775","last_scrub_stamp":"2023-10-09T06:40:48.261467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:48.261467+0000","objects_scrubbed":2451,"log_size":7392,"log_dups_size":0,"ondisk_log_size":7392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:45.927807+0000","scrub_duration":0.48199046000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84344832,"num_objects":1287,"num_object_clones":0,"num_object_copies":3861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2622,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":846,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.759033+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:42:13.759033+0000","last_peered":"2023-10-09T06:42:13.759033+0000","last_clean":"2023-10-09T06:42:13.759033+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:42:13.759033+0000","last_undegraded":"2023-10-09T06:42:13.759033+0000","last_fullsized":"2023-10-09T06:42:13.759033+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":386,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525426+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:42:05.525426+0000","last_peered":"2023-10-09T06:42:05.525426+0000","last_clean":"2023-10-09T06:42:05.525426+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:42:05.525426+0000","last_undegraded":"2023-10-09T06:42:05.525426+0000","last_fullsized":"2023-10-09T06:42:05.525426+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":523,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.736755+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:42:13.736755+0000","last_peered":"2023-10-09T06:42:13.736755+0000","last_clean":"2023-10-09T06:42:13.736755+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:42:13.736755+0000","last_undegraded":"2023-10-09T06:42:13.736755+0000","last_fullsized":"2023-10-09T06:42:13.736755+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":509,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516819+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:42:05.516819+0000","last_peered":"2023-10-09T06:42:05.516819+0000","last_clean":"2023-10-09T06:42:05.516819+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:42:05.516819+0000","last_undegraded":"2023-10-09T06:42:05.516819+0000","last_fullsized":"2023-10-09T06:42:05.516819+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"524'10122","reported_seq":10167,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T06:42:28.164420+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:42:28.164420+0000","last_peered":"2023-10-09T06:42:28.164420+0000","last_clean":"2023-10-09T06:42:28.164420+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:42:28.164420+0000","last_undegraded":"2023-10-09T06:42:28.164420+0000","last_fullsized":"2023-10-09T06:42:28.164420+0000","mapping_epoch":489,"log_start":"501'4700","ondisk_log_start":"501'4700","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5422,"log_dups_size":0,"ondisk_log_size":5422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168951808,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10122,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":846,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523355+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:42:05.523355+0000","last_peered":"2023-10-09T06:42:05.523355+0000","last_clean":"2023-10-09T06:42:05.523355+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:42:05.523355+0000","last_undegraded":"2023-10-09T06:42:05.523355+0000","last_fullsized":"2023-10-09T06:42:05.523355+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":257,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.752342+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:42:13.752342+0000","last_peered":"2023-10-09T06:42:13.752342+0000","last_clean":"2023-10-09T06:42:13.752342+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:42:13.752342+0000","last_undegraded":"2023-10-09T06:42:13.752342+0000","last_fullsized":"2023-10-09T06:42:13.752342+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":490,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.737748+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:42:13.737748+0000","last_peered":"2023-10-09T06:42:13.737748+0000","last_clean":"2023-10-09T06:42:13.737748+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:42:13.737748+0000","last_undegraded":"2023-10-09T06:42:13.737748+0000","last_fullsized":"2023-10-09T06:42:13.737748+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":246,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.748906+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:42:13.748906+0000","last_peered":"2023-10-09T06:42:13.748906+0000","last_clean":"2023-10-09T06:42:13.748906+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:42:13.748906+0000","last_undegraded":"2023-10-09T06:42:13.748906+0000","last_fullsized":"2023-10-09T06:42:13.748906+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"523'9964","reported_seq":10008,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.776263+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:42:24.776263+0000","last_peered":"2023-10-09T06:42:24.776263+0000","last_clean":"2023-10-09T06:42:24.776263+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:42:24.776263+0000","last_undegraded":"2023-10-09T06:42:24.776263+0000","last_fullsized":"2023-10-09T06:42:24.776263+0000","mapping_epoch":489,"log_start":"497'3900","ondisk_log_start":"497'3900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179044352,"num_objects":2732,"num_object_clones":0,"num_object_copies":8196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9964,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":880,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:19.107377+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:42:19.107377+0000","last_peered":"2023-10-09T06:42:19.107377+0000","last_clean":"2023-10-09T06:42:19.107377+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:42:19.107377+0000","last_undegraded":"2023-10-09T06:42:19.107377+0000","last_fullsized":"2023-10-09T06:42:19.107377+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":508,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.759271+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:42:13.759271+0000","last_peered":"2023-10-09T06:42:13.759271+0000","last_clean":"2023-10-09T06:42:13.759271+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:42:13.759271+0000","last_undegraded":"2023-10-09T06:42:13.759271+0000","last_fullsized":"2023-10-09T06:42:13.759271+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":491,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516430+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:42:05.516430+0000","last_peered":"2023-10-09T06:42:05.516430+0000","last_clean":"2023-10-09T06:42:05.516430+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:42:05.516430+0000","last_undegraded":"2023-10-09T06:42:05.516430+0000","last_fullsized":"2023-10-09T06:42:05.516430+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":486,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522735+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:42:05.522735+0000","last_peered":"2023-10-09T06:42:05.522735+0000","last_clean":"2023-10-09T06:42:05.522735+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:42:05.522735+0000","last_undegraded":"2023-10-09T06:42:05.522735+0000","last_fullsized":"2023-10-09T06:42:05.522735+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"523'7182","reported_seq":7234,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.795872+0000","last_change":"2023-10-09T06:40:43.188245+0000","last_active":"2023-10-09T06:42:24.795872+0000","last_peered":"2023-10-09T06:42:24.795872+0000","last_clean":"2023-10-09T06:42:24.795872+0000","last_became_active":"2023-10-09T06:40:43.186495+0000","last_became_peered":"2023-10-09T06:40:43.186495+0000","last_unstale":"2023-10-09T06:42:24.795872+0000","last_undegraded":"2023-10-09T06:42:24.795872+0000","last_fullsized":"2023-10-09T06:42:24.795872+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85590016,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4902,"num_write_kb":198656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":806,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.753221+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:42:13.753221+0000","last_peered":"2023-10-09T06:42:13.753221+0000","last_clean":"2023-10-09T06:42:13.753221+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:42:13.753221+0000","last_undegraded":"2023-10-09T06:42:13.753221+0000","last_fullsized":"2023-10-09T06:42:13.753221+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":492,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516841+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:42:05.516841+0000","last_peered":"2023-10-09T06:42:05.516841+0000","last_clean":"2023-10-09T06:42:05.516841+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:42:05.516841+0000","last_undegraded":"2023-10-09T06:42:05.516841+0000","last_fullsized":"2023-10-09T06:42:05.516841+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":515,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.736667+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:42:13.736667+0000","last_peered":"2023-10-09T06:42:13.736667+0000","last_clean":"2023-10-09T06:42:13.736667+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:42:13.736667+0000","last_undegraded":"2023-10-09T06:42:13.736667+0000","last_fullsized":"2023-10-09T06:42:13.736667+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"524'7242","reported_seq":12947,"reported_epoch":524,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:28.212576+0000","last_change":"2023-10-09T06:41:56.489849+0000","last_active":"2023-10-09T06:42:28.212576+0000","last_peered":"2023-10-09T06:42:28.212576+0000","last_clean":"2023-10-09T06:40:50.553480+0000","last_became_active":"2023-10-09T06:40:53.875917+0000","last_became_peered":"2023-10-09T06:40:53.875917+0000","last_unstale":"2023-10-09T06:42:28.212576+0000","last_undegraded":"2023-10-09T06:40:53.725278+0000","last_fullsized":"2023-10-09T06:40:53.643650+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":615,"num_objects_misplaced":615,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4962,"num_write_kb":200448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1882,"num_bytes_recovered":93782016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":687}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":737,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:19.107411+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:42:19.107411+0000","last_peered":"2023-10-09T06:42:19.107411+0000","last_clean":"2023-10-09T06:42:19.107411+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:42:19.107411+0000","last_undegraded":"2023-10-09T06:42:19.107411+0000","last_fullsized":"2023-10-09T06:42:19.107411+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":201,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523099+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:42:05.523099+0000","last_peered":"2023-10-09T06:42:05.523099+0000","last_clean":"2023-10-09T06:42:05.523099+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:42:05.523099+0000","last_undegraded":"2023-10-09T06:42:05.523099+0000","last_fullsized":"2023-10-09T06:42:05.523099+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":530,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523024+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:42:05.523024+0000","last_peered":"2023-10-09T06:42:05.523024+0000","last_clean":"2023-10-09T06:42:05.523024+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:42:05.523024+0000","last_undegraded":"2023-10-09T06:42:05.523024+0000","last_fullsized":"2023-10-09T06:42:05.523024+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"524'7333","reported_seq":17518,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-09T06:42:28.165871+0000","last_change":"2023-10-09T06:41:58.466796+0000","last_active":"2023-10-09T06:42:28.165871+0000","last_peered":"2023-10-09T06:42:28.165871+0000","last_clean":"2023-10-09T06:42:28.165871+0000","last_became_active":"2023-10-09T06:41:58.457422+0000","last_became_peered":"2023-10-09T06:41:58.457422+0000","last_unstale":"2023-10-09T06:42:28.165871+0000","last_undegraded":"2023-10-09T06:42:28.165871+0000","last_fullsized":"2023-10-09T06:42:28.165871+0000","mapping_epoch":519,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82214912,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5043,"num_write_kb":201504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2529,"num_bytes_recovered":165740544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":201,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.523479+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:42:05.523479+0000","last_peered":"2023-10-09T06:42:05.523479+0000","last_clean":"2023-10-09T06:42:05.523479+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:42:05.523479+0000","last_undegraded":"2023-10-09T06:42:05.523479+0000","last_fullsized":"2023-10-09T06:42:05.523479+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":540,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522644+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:42:05.522644+0000","last_peered":"2023-10-09T06:42:05.522644+0000","last_clean":"2023-10-09T06:42:05.522644+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:42:05.522644+0000","last_undegraded":"2023-10-09T06:42:05.522644+0000","last_fullsized":"2023-10-09T06:42:05.522644+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"523'7310","reported_seq":7366,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:24.819182+0000","last_change":"2023-10-09T06:40:43.203964+0000","last_active":"2023-10-09T06:42:24.819182+0000","last_peered":"2023-10-09T06:42:24.819182+0000","last_clean":"2023-10-09T06:42:24.819182+0000","last_became_active":"2023-10-09T06:40:43.185473+0000","last_became_peered":"2023-10-09T06:40:43.185473+0000","last_unstale":"2023-10-09T06:42:24.819182+0000","last_undegraded":"2023-10-09T06:42:24.819182+0000","last_fullsized":"2023-10-09T06:42:24.819182+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7310,"log_dups_size":0,"ondisk_log_size":7310,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89391104,"num_objects":1364,"num_object_clones":0,"num_object_copies":4092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4990,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":879,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.525530+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:42:05.525530+0000","last_peered":"2023-10-09T06:42:05.525530+0000","last_clean":"2023-10-09T06:42:05.525530+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:42:05.525530+0000","last_undegraded":"2023-10-09T06:42:05.525530+0000","last_fullsized":"2023-10-09T06:42:05.525530+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":228,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:13.752788+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:42:13.752788+0000","last_peered":"2023-10-09T06:42:13.752788+0000","last_clean":"2023-10-09T06:42:13.752788+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:42:13.752788+0000","last_undegraded":"2023-10-09T06:42:13.752788+0000","last_fullsized":"2023-10-09T06:42:13.752788+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":245,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:14.106589+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:42:14.106589+0000","last_peered":"2023-10-09T06:42:14.106589+0000","last_clean":"2023-10-09T06:42:14.106589+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:42:14.106589+0000","last_undegraded":"2023-10-09T06:42:14.106589+0000","last_fullsized":"2023-10-09T06:42:14.106589+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"523'7390","reported_seq":7454,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:27.760052+0000","last_change":"2023-10-09T06:41:43.517660+0000","last_active":"2023-10-09T06:42:27.760052+0000","last_peered":"2023-10-09T06:42:27.760052+0000","last_clean":"2023-10-09T06:42:27.760052+0000","last_became_active":"2023-10-09T06:40:43.111680+0000","last_became_peered":"2023-10-09T06:40:43.111680+0000","last_unstale":"2023-10-09T06:42:27.760052+0000","last_undegraded":"2023-10-09T06:42:27.760052+0000","last_fullsized":"2023-10-09T06:42:27.760052+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84312064,"num_objects":1287,"num_object_clones":0,"num_object_copies":3861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5079,"num_write_kb":203680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":648,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-09T06:42:18.204294+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:42:18.204294+0000","last_peered":"2023-10-09T06:42:18.204294+0000","last_clean":"2023-10-09T06:42:18.204294+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:42:18.204294+0000","last_undegraded":"2023-10-09T06:42:18.204294+0000","last_fullsized":"2023-10-09T06:42:18.204294+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":522,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.516002+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:42:05.516002+0000","last_peered":"2023-10-09T06:42:05.516002+0000","last_clean":"2023-10-09T06:42:05.516002+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:42:05.516002+0000","last_undegraded":"2023-10-09T06:42:05.516002+0000","last_fullsized":"2023-10-09T06:42:05.516002+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":483,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-09T06:42:05.522483+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:42:05.522483+0000","last_peered":"2023-10-09T06:42:05.522483+0000","last_clean":"2023-10-09T06:42:05.522483+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:42:05.522483+0000","last_undegraded":"2023-10-09T06:42:05.522483+0000","last_fullsized":"2023-10-09T06:42:05.522483+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":2766077976,"num_objects":42208,"num_object_clones":0,"num_object_copies":126624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":615,"num_objects_misplaced":615,"num_objects_unfound":0,"num_objects_dirty":42208,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142289,"num_write_kb":5352961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4413,"num_bytes_recovered":259653632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8735305728,"data_stored":8735293512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181180,"ondisk_log_size":181180,"up":78,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234542,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089324,"kb_used_data":584228,"kb_used_omap":0,"kb_used_meta":1505088,"kb_avail":91655828,"statfs":{"total":95995035648,"available":93855567872,"internally_reserved":0,"allocated":598249472,"data_stored":597183280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541210112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:18 2023","interfaces":[{"interface":"back","average":{"1min":4.5549999999999997,"5min":2.169,"15min":3.9420000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":55.783999999999999,"5min":55.783999999999999,"15min":215.97800000000001},"last":10.317},{"interface":"front","average":{"1min":4.5439999999999996,"5min":2.129,"15min":3.996},"min":{"1min":0.245,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":55.771999999999998,"5min":55.771999999999998,"15min":215.577},"last":10.339}]},{"osd":1,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":12.045999999999999,"5min":4.665,"15min":5.8719999999999999},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":169.51900000000001,"5min":169.51900000000001,"15min":215.61699999999999},"last":6.7720000000000002},{"interface":"front","average":{"1min":11.603999999999999,"5min":4.6269999999999998,"15min":4.867},"min":{"1min":0.28899999999999998,"5min":0.28100000000000003,"15min":0.23300000000000001},"max":{"1min":169.101,"5min":169.101,"15min":216.22200000000001},"last":6.5730000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":6.4640000000000004,"15min":5.8150000000000004},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":30.117000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.308},{"interface":"front","average":{"1min":2.5870000000000002,"5min":6.5140000000000002,"15min":5.7960000000000003},"min":{"1min":0.35499999999999998,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":30.245999999999999,"5min":228.291,"15min":228.291},"last":4.6449999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":5.6219999999999999,"5min":6.7199999999999998,"15min":5.774},"min":{"1min":0.38400000000000001,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":43.341000000000001,"5min":175.40600000000001,"15min":215.922},"last":0.373},{"interface":"front","average":{"1min":5.242,"5min":6.5490000000000004,"15min":5.7400000000000002},"min":{"1min":0.373,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":41.433999999999997,"5min":175.30099999999999,"15min":215.94900000000001},"last":6.4509999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":4.3609999999999998,"15min":3.1480000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":2.8199999999999998,"5min":225.041,"15min":225.041},"last":0.499},{"interface":"front","average":{"1min":0.65100000000000002,"5min":4.1680000000000001,"15min":3.0139999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.6059999999999999,"5min":225.244,"15min":225.244},"last":0.34200000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":4.9210000000000003,"15min":3.3799999999999999},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.879,"5min":219.923,"15min":219.923},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.628,"5min":4.6020000000000003,"15min":3.3199999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.29999999999999999},"max":{"1min":1.8600000000000001,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":4.5449999999999999,"15min":3.4620000000000002},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.317},"max":{"1min":2.23,"5min":222.179,"15min":222.179},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.73799999999999999,"5min":4.4829999999999997,"15min":3.294},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.4989999999999997,"5min":223.261,"15min":223.261},"last":0.51800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496178,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1953560,"kb_used_data":753872,"kb_used_omap":0,"kb_used_meta":1199680,"kb_avail":91791592,"statfs":{"total":95995035648,"available":93994590208,"internally_reserved":0,"allocated":771964928,"data_stored":770917962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1228472320},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.5740000000000001,"15min":2.4359999999999999},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20999999999999999},"max":{"1min":4.2199999999999998,"5min":23.125,"15min":75.061999999999998},"last":1.0549999999999999},{"interface":"front","average":{"1min":1.341,"5min":4.056,"15min":3.5150000000000001},"min":{"1min":0.33500000000000002,"5min":0.30099999999999999,"15min":0.23599999999999999},"max":{"1min":4.2329999999999997,"5min":212.096,"15min":212.096},"last":1.077}]},{"osd":1,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":4.9169999999999998,"15min":5.0380000000000003},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.20899999999999999},"max":{"1min":3.8919999999999999,"5min":219.93199999999999,"15min":292.92200000000003},"last":4.2489999999999997},{"interface":"front","average":{"1min":1.548,"5min":5.0259999999999998,"15min":5.0919999999999996},"min":{"1min":0.39300000000000002,"5min":0.222,"15min":0.222},"max":{"1min":8.3840000000000003,"5min":219.94800000000001,"15min":292.904},"last":4.2119999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.8970000000000002,"15min":3.79},"min":{"1min":0.34899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":6.6959999999999997,"5min":185.654,"15min":185.654},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.8460000000000001,"15min":3.593},"min":{"1min":0.38900000000000001,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":6.7130000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:41:30 2023","interfaces":[{"interface":"back","average":{"1min":8.7029999999999994,"5min":4.9080000000000004,"15min":4.0819999999999999},"min":{"1min":0.33400000000000002,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":161.60300000000001,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":9.0820000000000007,"5min":4.899,"15min":4.0170000000000003},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.318},"max":{"1min":161.56299999999999,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.41299999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.198,"15min":1.53},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":5.6890000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.27,"5min":3.254,"15min":2.177},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.223},"max":{"1min":8.2989999999999995,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.34000000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.3140000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.435,"5min":10.435,"15min":12.747999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.4259999999999999,"15min":1.5329999999999999},"min":{"1min":0.312,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3369999999999997,"5min":9.9139999999999997,"15min":16.536999999999999},"last":0.376}]},{"osd":7,"last update":"Mon Oct 9 06:41:48 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":2.8340000000000001,"15min":2.6749999999999998},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5220000000000002,"5min":165.37,"15min":165.37},"last":0.505},{"interface":"front","average":{"1min":1.1479999999999999,"5min":3.2170000000000001,"15min":2.9929999999999999},"min":{"1min":0.36899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.6369999999999996,"5min":165.40199999999999,"15min":165.40199999999999},"last":0.42799999999999999}]}]},{"osd":1,"up_from":110,"seq":472446402915,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2549544,"kb_used_data":1422688,"kb_used_omap":4,"kb_used_meta":1126651,"kb_avail":91195608,"statfs":{"total":95995035648,"available":93384302592,"internally_reserved":0,"allocated":1456832512,"data_stored":1454609052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1153691534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.5410000000000004,"5min":3.9950000000000001,"15min":4.9109999999999996},"min":{"1min":0.21299999999999999,"5min":0.182,"15min":0.182},"max":{"1min":102.941,"5min":110.479,"15min":269.79199999999997},"last":2.6240000000000001},{"interface":"front","average":{"1min":10.712999999999999,"5min":4.6639999999999997,"15min":5.2030000000000003},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":100.16800000000001,"5min":110.59999999999999,"15min":271.20999999999998},"last":1.9630000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.3420000000000001,"15min":2.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.184},"max":{"1min":8.0129999999999999,"5min":8.0129999999999999,"15min":138.452},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.5449999999999999,"15min":2.8530000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":17.555,"5min":17.555,"15min":138.47499999999999},"last":2.5779999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.7000000000000002,"5min":2.5379999999999998,"15min":2.7080000000000002},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":13.523999999999999,"5min":13.523999999999999,"15min":31.062000000000001},"last":1.9019999999999999},{"interface":"front","average":{"1min":2.0430000000000001,"5min":1.5129999999999999,"15min":1.8799999999999999},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.22700000000000001},"max":{"1min":8.4220000000000006,"5min":8.4220000000000006,"15min":27.657},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.013999999999999,"5min":3.927,"15min":3.2400000000000002},"min":{"1min":0.316,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":180.59700000000001,"5min":180.59700000000001,"15min":180.59700000000001},"last":1.944},{"interface":"front","average":{"1min":9.3659999999999997,"5min":3.6040000000000001,"15min":2.919},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":180.70599999999999,"5min":180.70599999999999,"15min":180.70599999999999},"last":1.9379999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.6459999999999999,"15min":2.5630000000000002},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":7.9420000000000002,"5min":15.83,"15min":119.408},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.988,"15min":2.7850000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":8.3819999999999997,"15min":119.483},"last":1.8720000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.7230000000000008,"5min":3.883,"15min":3.4409999999999998},"min":{"1min":0.33700000000000002,"5min":0.317,"15min":0.317},"max":{"1min":183.12100000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":1.9199999999999999},{"interface":"front","average":{"1min":10.446,"5min":4.3179999999999996,"15min":3.8639999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":183.13900000000001,"5min":183.13900000000001,"15min":183.13900000000001},"last":1.972}]},{"osd":7,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.129,"5min":4.5419999999999998,"15min":4.3570000000000002},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":178.74000000000001,"5min":178.74000000000001,"15min":178.74000000000001},"last":1.9930000000000001},{"interface":"front","average":{"1min":8.9540000000000006,"5min":3.8029999999999999,"15min":3.7109999999999999},"min":{"1min":0.379,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":178.75299999999999,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.40400000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024348,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527616,"kb_used_data":1420728,"kb_used_omap":5,"kb_used_meta":1106874,"kb_avail":91217536,"statfs":{"total":95995035648,"available":93406756864,"internally_reserved":0,"allocated":1454825472,"data_stored":1452226274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1133439610},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:41:51 2023","interfaces":[{"interface":"back","average":{"1min":13.958,"5min":8.4000000000000004,"15min":4.8179999999999996},"min":{"1min":0.192,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":172.762,"5min":201.381,"15min":201.381},"last":2.3069999999999999},{"interface":"front","average":{"1min":13.670999999999999,"5min":8.4559999999999995,"15min":5.0940000000000003},"min":{"1min":0.27300000000000002,"5min":0.193,"15min":0.187},"max":{"1min":172.773,"5min":201.44200000000001,"15min":201.44200000000001},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":3.3239999999999998,"15min":3.5099999999999998},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":4.5289999999999999,"5min":97.650000000000006,"15min":100.425},"last":1.405},{"interface":"front","average":{"1min":2.698,"5min":3.7210000000000001,"15min":3.9260000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":21.096,"5min":96.082999999999998,"15min":99.769999999999996},"last":2.8010000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":3.5019999999999998,"15min":4.9450000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.484999999999999,"5min":95.555000000000007,"15min":208.11199999999999},"last":2.867},{"interface":"front","average":{"1min":2.2799999999999998,"5min":3.6139999999999999,"15min":4.8719999999999999},"min":{"1min":0.33400000000000002,"5min":0.22900000000000001,"15min":0.19800000000000001},"max":{"1min":10.417,"5min":97.605000000000004,"15min":208.25899999999999},"last":1.0980000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.9610000000000001,"15min":3.1360000000000001},"min":{"1min":0.32400000000000001,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":7.9649999999999999,"5min":14.183,"15min":200.654},"last":1.9039999999999999},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.0960000000000001,"15min":3.4750000000000001},"min":{"1min":0.376,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":8.2850000000000001,"5min":11.738,"15min":200.86600000000001},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":2.1539999999999999,"15min":2.1619999999999999},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":8.0120000000000005,"5min":14.263,"15min":80.885999999999996},"last":1.845},{"interface":"front","average":{"1min":1.712,"5min":1.7809999999999999,"15min":2.2469999999999999},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":8.4019999999999992,"15min":80.438999999999993},"last":2.714}]},{"osd":6,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9610000000000001,"15min":3.2469999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":120.572},"last":2.7480000000000002},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.0600000000000001,"15min":3.0129999999999999},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":8.0449999999999999,"5min":14.308999999999999,"15min":120.64400000000001},"last":1.8759999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.9379999999999999,"15min":2.5019999999999998},"min":{"1min":0.42199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.2490000000000006,"5min":16.911000000000001,"15min":122.018},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.931,"5min":2.3889999999999998,"15min":2.9049999999999998},"min":{"1min":0.36199999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.4570000000000007,"5min":9.9190000000000005,"15min":122.32299999999999},"last":2.774}]}]},{"osd":2,"up_from":128,"seq":549755814217,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2271648,"kb_used_data":1244504,"kb_used_omap":7,"kb_used_meta":1027128,"kb_avail":91473504,"statfs":{"total":95995035648,"available":93668868096,"internally_reserved":0,"allocated":1274372096,"data_stored":1272206368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1051779153},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":6.5650000000000004,"15min":6.5469999999999997},"min":{"1min":0.254,"5min":0.189,"15min":0.189},"max":{"1min":14.374000000000001,"5min":214.595,"15min":214.595},"last":3.7309999999999999},{"interface":"front","average":{"1min":3.786,"5min":6.1200000000000001,"15min":6.226},"min":{"1min":0.221,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":17.053000000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":3.1549999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.6360000000000001,"15min":4.3070000000000004},"min":{"1min":0.217,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.773999999999999,"5min":80.141000000000005,"15min":140.452},"last":1.6619999999999999},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.665,"15min":4.5149999999999997},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":11.930999999999999,"5min":79.474000000000004,"15min":139.75299999999999},"last":2.383}]},{"osd":3,"last update":"Mon Oct 9 06:41:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":2.7650000000000001,"15min":3.464},"min":{"1min":0.27400000000000002,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":29.093,"5min":79.566000000000003,"15min":91.644000000000005},"last":2.4740000000000002},{"interface":"front","average":{"1min":3.089,"5min":2.7120000000000002,"15min":3.7669999999999999},"min":{"1min":0.37,"5min":0.27700000000000002,"15min":0.25700000000000001},"max":{"1min":29.126999999999999,"5min":79.614000000000004,"15min":91.685000000000002},"last":2.3540000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":2.036,"15min":2.3069999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.5679999999999996,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.441},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2120000000000002,"15min":2.54},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":4.4539999999999997,"5min":82.872,"15min":82.872},"last":0.309}]},{"osd":5,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.994,"15min":2.3570000000000002},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7040000000000006,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1720000000000002,"15min":2.4950000000000001},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":4.4109999999999996,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3039999999999998,"15min":3.2010000000000001},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.312},"max":{"1min":4.4880000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.5209999999999999,"15min":3.1850000000000001},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":6.3689999999999998,"5min":80.364999999999995,"15min":122.517},"last":1.734}]},{"osd":7,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":8.9900000000000002,"5min":3.8100000000000001,"15min":3.298},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":132.06399999999999,"5min":132.06399999999999,"15min":132.06399999999999},"last":1.7050000000000001},{"interface":"front","average":{"1min":8.7509999999999994,"5min":3.4420000000000002,"15min":2.9460000000000002},"min":{"1min":0.35899999999999999,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":132.196,"5min":132.196,"15min":132.196},"last":0.51900000000000002}]}]},{"osd":3,"up_from":149,"seq":639950127415,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2140004,"kb_used_data":1235036,"kb_used_omap":2,"kb_used_meta":904957,"kb_avail":91605148,"statfs":{"total":95995035648,"available":93803671552,"internally_reserved":0,"allocated":1264676864,"data_stored":1262632449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":926676021},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.7939999999999996,"5min":2.8290000000000002,"15min":4.0720000000000001},"min":{"1min":0.217,"5min":0.216,"15min":0.19},"max":{"1min":84.817999999999998,"5min":84.817999999999998,"15min":161.821},"last":0.19500000000000001},{"interface":"front","average":{"1min":6.6589999999999998,"5min":2.964,"15min":3.9300000000000002},"min":{"1min":0.246,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":84.635000000000005,"5min":84.635000000000005,"15min":163.52000000000001},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.5890000000000004,"5min":3.1419999999999999,"15min":3.9790000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21199999999999999},"max":{"1min":83.412000000000006,"5min":83.412000000000006,"15min":184.012},"last":0.443},{"interface":"front","average":{"1min":5.8739999999999997,"5min":3.1219999999999999,"15min":4.734},"min":{"1min":0.32100000000000001,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":83.394999999999996,"5min":83.394999999999996,"15min":184.703},"last":1.4059999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":2.9430000000000001,"15min":3.9510000000000001},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.0199999999999996,"5min":118.93899999999999,"15min":182.44900000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.081,"5min":3.0259999999999998,"15min":4.2759999999999998},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":6.7329999999999997,"5min":118.97199999999999,"15min":182.56399999999999},"last":1.1830000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":3.3820000000000001,"15min":3.3319999999999999},"min":{"1min":0.29099999999999998,"5min":0.28899999999999998,"15min":0.26900000000000002},"max":{"1min":5.2679999999999998,"5min":133.80099999999999,"15min":178.22200000000001},"last":6.0069999999999997},{"interface":"front","average":{"1min":2.2050000000000001,"5min":3.4340000000000002,"15min":3.3959999999999999},"min":{"1min":0.36899999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":7.2690000000000001,"5min":133.648,"15min":180.43600000000001},"last":3.1179999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":3.5329999999999999,"15min":3.6320000000000001},"min":{"1min":0.374,"5min":0.35699999999999998,"15min":0.34000000000000002},"max":{"1min":7.3600000000000003,"5min":128.083,"15min":177.56299999999999},"last":3.7879999999999998},{"interface":"front","average":{"1min":1.6220000000000001,"5min":3.359,"15min":3.0099999999999998},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":7.0060000000000002,"5min":128.11799999999999,"15min":177.59399999999999},"last":3.7759999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.9810000000000001,"15min":2.7400000000000002},"min":{"1min":0.40100000000000002,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":7.3159999999999998,"5min":10.505000000000001,"15min":130.39599999999999},"last":3.8250000000000002},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.8029999999999999,"15min":2.3809999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.9859999999999998,"5min":9.8179999999999996,"15min":130.67699999999999},"last":3.4910000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":5.657,"5min":2.641,"15min":3.04},"min":{"1min":0.45800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":99.248000000000005,"5min":99.248000000000005,"15min":121.613},"last":0.55200000000000005},{"interface":"front","average":{"1min":6.7199999999999998,"5min":2.8660000000000001,"15min":3.4860000000000002},"min":{"1min":0.41299999999999998,"5min":0.372,"15min":0.33400000000000002},"max":{"1min":99.457999999999998,"5min":99.457999999999998,"15min":121.559},"last":3.2690000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019451,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2658592,"kb_used_data":1179032,"kb_used_omap":0,"kb_used_meta":1479552,"kb_avail":91086560,"statfs":{"total":95995035648,"available":93272637440,"internally_reserved":0,"allocated":1207328768,"data_stored":1206273987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1515061248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.823,"15min":6.4610000000000003},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":69.165000000000006,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":5.2089999999999996,"5min":3.1459999999999999,"15min":6.6760000000000002},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":69.147999999999996,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.311}]},{"osd":1,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":5.7619999999999996,"5min":3.9820000000000002,"15min":4.1699999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":69.644000000000005,"5min":163.392,"15min":163.392},"last":1.1160000000000001},{"interface":"front","average":{"1min":6.1920000000000002,"5min":4.085,"15min":4.2439999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":69.572000000000003,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.58799999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":5.1680000000000001,"15min":4.3579999999999997},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3180000000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":1.046},{"interface":"front","average":{"1min":1.25,"5min":5.4630000000000001,"15min":4.3079999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.26400000000000001},"max":{"1min":3.3140000000000001,"5min":175.875,"15min":175.875},"last":0.71899999999999997}]},{"osd":3,"last update":"Mon Oct 9 06:41:29 2023","interfaces":[{"interface":"back","average":{"1min":7.0590000000000002,"5min":4.6420000000000003,"15min":5.9500000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":108.19799999999999,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":7.5979999999999999,"5min":4.3659999999999997,"15min":5.7699999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":112.21899999999999,"5min":112.21899999999999,"15min":161.934},"last":0.52600000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.006,"15min":0.96299999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":3.1070000000000002,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.253},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.198,"15min":1.087},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.7599999999999998,"5min":8.4600000000000009,"15min":9.5009999999999994},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":5.1189999999999998,"5min":1.788,"15min":2.2549999999999999},"min":{"1min":0.317,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":80.549000000000007,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":5.258,"5min":1.8049999999999999,"15min":2.4750000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":83.218000000000004,"5min":83.218000000000004,"15min":163.381},"last":0.38}]},{"osd":7,"last update":"Mon Oct 9 06:41:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.101,"15min":1.651},"min":{"1min":0.34899999999999998,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":0.94999999999999996,"5min":5.2539999999999996,"15min":106.801},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.63800000000000001,"5min":1.1559999999999999,"15min":1.694},"min":{"1min":0.34399999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.403,"5min":9.4749999999999996,"15min":106.827},"last":0.49399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757816,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2338136,"kb_used_data":858576,"kb_used_omap":0,"kb_used_meta":1479488,"kb_avail":91407016,"statfs":{"total":95995035648,"available":93600784384,"internally_reserved":0,"allocated":879181824,"data_stored":878131487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514995712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.4609999999999999,"15min":6.0819999999999999},"min":{"1min":0.23200000000000001,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":4.5460000000000003,"5min":78.171999999999997,"15min":294.89699999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":2.097,"5min":3.7440000000000002,"15min":6.2889999999999997},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":7.4630000000000001,"5min":78.230000000000004,"15min":294.30399999999997},"last":0.35799999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":10.257999999999999,"15min":7.2409999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.22},"max":{"1min":5.8120000000000003,"5min":153.91300000000001,"15min":188.34100000000001},"last":5.8630000000000004},{"interface":"front","average":{"1min":2.2349999999999999,"5min":10.214,"15min":7.2910000000000004},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1369999999999996,"5min":154.023,"15min":188.839},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":5.0970000000000004,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.49,"5min":100.46599999999999,"15min":120.613},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.2919999999999998,"15min":3.9540000000000002},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.4610000000000003,"5min":100.56999999999999,"15min":120.962},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":5.8540000000000001,"15min":6.5890000000000004},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":5.6340000000000003,"5min":220.41800000000001,"15min":220.41800000000001},"last":2.5099999999999998},{"interface":"front","average":{"1min":2.4390000000000001,"5min":5.7930000000000001,"15min":6.4100000000000001},"min":{"1min":0.442,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.4210000000000003,"5min":220.45500000000001,"15min":220.45500000000001},"last":2.6360000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.091,"15min":1.3799999999999999},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.247},"max":{"1min":2.7320000000000002,"5min":9.9290000000000003,"15min":79.043000000000006},"last":0.30099999999999999},{"interface":"front","average":{"1min":1,"5min":1.298,"15min":1.4039999999999999},"min":{"1min":0.378,"5min":0.251,"15min":0.251},"max":{"1min":3.3180000000000001,"5min":8.3970000000000002,"15min":79.165999999999997},"last":0.41499999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":6.3520000000000003,"5min":2.0299999999999998,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.251},"max":{"1min":81.254000000000005,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.38400000000000001},{"interface":"front","average":{"1min":6.0030000000000001,"5min":1.891,"15min":1.9159999999999999},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":79.216999999999999,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.32100000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.3700000000000001,"15min":1.5449999999999999},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.29099999999999998},"max":{"1min":4.3280000000000003,"5min":11.146000000000001,"15min":78.890000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.3049999999999999,"15min":1.395},"min":{"1min":0.35299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.359,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.47199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1421606912,"data_stored":1421606912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1444413440,"data_stored":1444413440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1249574912,"data_stored":1249574912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1242497024,"data_stored":1242497024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1178603520,"data_stored":1178599448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":860033024,"data_stored":860028952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":749539328,"data_stored":749535256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":589037568,"data_stored":589037568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:42:33.964 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:42:34.044 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:34.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:34.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:34.162 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:34.193 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:34.253 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:34.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:34.286 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:34.359 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:34.466 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:34.820 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:42:34.820 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104909 12.13 12:cd32261f:::benchmark_data_smithi107_479504_object5298:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.39827198099999, 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.143602357, 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.821 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104909 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.822 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413795+0000", 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413794+0000", 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.823 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413797+0000", 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.965e-06 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413803+0000", 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1959999999999996e-06 2023-10-09T06:42:34.824 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413809+0000", 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3020000000000002e-06 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413951+0000", 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014192100000000001 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.825 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413960+0000", 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5750000000000007e-06 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.536884+0000", 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.122924037 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.826 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.537024+0000", 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000139682 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.537140+0000", 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116098 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.827 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542524+0000", 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053841330000000001 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546061+0000", 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003536951 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.828 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555262+0000", 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0092012450000000003 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555328+0000", 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5580999999999994e-05 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.829 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557397+0000", 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020690410000000002 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104929 12.13 12:c993a819:::benchmark_data_smithi107_479504_object5318:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:42:34.830 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.39824787800001, 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1455242480000001, 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104929 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.831 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.832 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:42:34.832 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.832 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.832 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.832 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413819+0000", 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.833 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413818+0000", 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413821+0000", 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2630000000000002e-06 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.834 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413827+0000", 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6440000000000003e-06 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413832+0000", 2023-10-09T06:42:34.835 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0320000000000003e-06 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413980+0000", 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000148188 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.836 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413987+0000", 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3110000000000002e-06 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540392+0000", 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1264056950000001 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540559+0000", 2023-10-09T06:42:34.837 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000166832 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.540720+0000", 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016054 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.838 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542567+0000", 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018473140000000001 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546204+0000", 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036369829999999999 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.839 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557461+0000", 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.01125724 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.557519+0000", 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7157000000000002e-05 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.840 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559343+0000", 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018243999999999999 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:104981 12.13 12:ca7186c7:::benchmark_data_smithi107_479504_object5370:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.841 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.39823065200005, 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1475939610000001, 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 104981 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.842 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.843 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.844 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.844 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413836+0000", 2023-10-09T06:42:34.844 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.844 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413838+0000", 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9589999999999998e-06 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.845 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413843+0000", 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6010000000000002e-06 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.413848+0000", 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5379999999999996e-06 2023-10-09T06:42:34.846 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.414006+0000", 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015859700000000001 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.414012+0000", 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0499999999999997e-06 2023-10-09T06:42:34.847 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542596+0000", 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.128583541 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542754+0000", 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015784600000000001 2023-10-09T06:42:34.848 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.542904+0000", 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015026899999999999 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546127+0000", 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032231130000000001 2023-10-09T06:42:34.849 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546213+0000", 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6285000000000005e-05 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559478+0000", 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013264664000000001 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.850 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559548+0000", 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9995999999999999e-05 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561430+0000", 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018817879999999999 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.851 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105015 12.13 12:cc2c9607:::benchmark_data_smithi107_479504_object5404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.38556479900001, 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1370335789999999, 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.852 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105015 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.853 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426502+0000", 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426501+0000", 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.854 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426504+0000", 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4200000000000001e-06 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426510+0000", 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7710000000000003e-06 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.855 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.426515+0000", 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1329999999999998e-06 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.428629+0000", 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021145769999999999 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.856 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.428638+0000", 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3359999999999995e-06 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.857 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544481+0000", 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.115843734 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544606+0000", 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000124144 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.544711+0000", 2023-10-09T06:42:34.858 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105208 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546191+0000", 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001480669 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548067+0000", 2023-10-09T06:42:34.859 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018758469999999999 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561516+0000", 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013448422999999999 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561553+0000", 2023-10-09T06:42:34.860 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7020999999999997e-05 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563535+0000", 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001982667 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.861 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105018 12.13 12:cbda2284:::benchmark_data_smithi107_479504_object5407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.38237829800005, 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1380620459999999, 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.862 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105018 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.863 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429688+0000", 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429687+0000", 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.864 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429693+0000", 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3940000000000004e-06 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429702+0000", 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8470000000000008e-06 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.865 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429710+0000", 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9019999999999999e-06 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429756+0000", 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6026999999999998e-05 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.866 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.429767+0000", 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0538e-05 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546258+0000", 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.116491232 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.867 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546428+0000", 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016985500000000001 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.546592+0000", 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016419699999999999 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.868 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548119+0000", 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015267690000000001 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551585+0000", 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034658010000000001 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.869 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.565667+0000", 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014082479 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.565724+0000", 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6938000000000003e-05 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.870 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567750+0000", 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020260959999999998 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105034 12.13 12:c9f14c5b:::benchmark_data_smithi107_479504_object5423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.871 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.36271570600002, 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.126998041, 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105034 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.872 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449351+0000", 2023-10-09T06:42:34.873 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449350+0000", 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449354+0000", 2023-10-09T06:42:34.874 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4649999999999999e-06 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449367+0000", 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2629000000000001e-05 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449373+0000", 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4389999999999997e-06 2023-10-09T06:42:34.875 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449896+0000", 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052338799999999996 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.449907+0000", 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1019000000000001e-05 2023-10-09T06:42:34.876 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548166+0000", 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0982583720000001 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548285+0000", 2023-10-09T06:42:34.877 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011966 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.548386+0000", 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100751 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.878 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551503+0000", 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031171810000000001 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551596+0000", 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2949999999999996e-05 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.879 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574084+0000", 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.022488121999999999 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574136+0000", 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1932999999999999e-05 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.880 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576349+0000", 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002212533 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105111 12.13 12:c8794572:::benchmark_data_smithi107_479504_object5500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:42:34.881 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.277898897, 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.044615676, 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105111 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.882 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534168+0000", 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.883 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534167+0000", 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534170+0000", 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7980000000000001e-06 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.884 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534176+0000", 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8270000000000003e-06 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534181+0000", 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0250000000000002e-06 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.885 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534253+0000", 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2373999999999995e-05 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.534263+0000", 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3230000000000004e-06 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.886 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.549708+0000", 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0154453960000001 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.549877+0000", 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016942699999999999 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.887 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.550047+0000", 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000169836 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551570+0000", 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015229989999999999 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.888 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.555183+0000", 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036130289999999998 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576445+0000", 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021261571999999999 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.889 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576506+0000", 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1013e-05 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578783+0000", 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002277477 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.890 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105179 12.13 12:cdc11d2f:::benchmark_data_smithi107_479504_object5568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.19163859900004, 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96059906500000003, 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.891 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105179 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.892 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620428+0000", 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620426+0000", 2023-10-09T06:42:34.893 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620434+0000", 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1610000000000003e-06 2023-10-09T06:42:34.894 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620451+0000", 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6575999999999999e-05 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620460+0000", 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4739999999999994e-06 2023-10-09T06:42:34.895 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620504+0000", 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3748000000000002e-05 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.620512+0000", 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6279999999999996e-06 2023-10-09T06:42:34.896 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551645+0000", 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.93113328500000003 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551821+0000", 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017628899999999999 2023-10-09T06:42:34.897 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.551998+0000", 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00017652 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559364+0000", 2023-10-09T06:42:34.898 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0073662279999999998 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559423+0000", 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9576e-05 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.899 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578913+0000", 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019489689000000001 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578975+0000", 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1656999999999996e-05 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.900 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581027+0000", 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020522909999999999 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105184 12.13 12:cef60faf:::benchmark_data_smithi107_479504_object5573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.901 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.18635283900005, 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.95766810999999996, 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105184 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.902 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625714+0000", 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.903 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625712+0000", 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625720+0000", 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0760000000000003e-06 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.904 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625742+0000", 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2001000000000002e-05 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625752+0000", 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0223e-05 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.905 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625803+0000", 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0695000000000002e-05 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.625814+0000", 2023-10-09T06:42:34.906 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.095e-05 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553587+0000", 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.92777310000000002 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.907 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553744+0000", 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015638300000000001 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.553844+0000", 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9994000000000005e-05 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.908 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.559433+0000", 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055894350000000002 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.561489+0000", 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002055996 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.909 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581100+0000", 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.019610909999999999 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.581155+0000", 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4753000000000002e-05 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.910 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583382+0000", 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022271740000000002 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.911 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105256 12.13 12:cfe2b70d:::benchmark_data_smithi107_479504_object5645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:42:34.913 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.08303973900001, 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.86023841499999998, 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105256 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.914 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729027+0000", 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729025+0000", 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.915 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729034+0000", 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5469999999999997e-06 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729051+0000", 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7402999999999999e-05 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.916 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.729061+0000", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8989999999999996e-06 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.731930+0000", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002869099 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.731948+0000", 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7538e-05 2023-10-09T06:42:34.917 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563570+0000", 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83162247 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.918 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563710+0000", 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013946400000000001 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.563844+0000", 2023-10-09T06:42:34.919 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013422999999999999 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.569803+0000", 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059588599999999999 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.920 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.573966+0000", 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0041634949999999997 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.921 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.586857+0000", 2023-10-09T06:42:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012890301999999999 2023-10-09T06:42:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.922 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.586907+0000", 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0142999999999997e-05 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589265+0000", 2023-10-09T06:42:34.923 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023584819999999999 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105289 12.13 12:cbcbb33f:::benchmark_data_smithi107_479504_object5678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.924 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 582.03754264600002, 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.81706288400000004, 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.925 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105289 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:42:34.926 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774524+0000", 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.927 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774522+0000", 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774532+0000", 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0095000000000001e-05 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.928 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774560+0000", 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8200000000000001e-05 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774572+0000", 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1697e-05 2023-10-09T06:42:34.929 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774625+0000", 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3347000000000003e-05 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.774635+0000", 2023-10-09T06:42:34.930 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.815e-06 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567795+0000", 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.79315939099999999 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.567947+0000", 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015192999999999999 2023-10-09T06:42:34.931 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.568055+0000", 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108917 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.932 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574011+0000", 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059552269999999996 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574020+0000", 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6970000000000005e-06 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.933 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589336+0000", 2023-10-09T06:42:34.934 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015315702 2023-10-09T06:42:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.589393+0000", 2023-10-09T06:42:34.935 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6461e-05 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.591587+0000", 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002194227 2023-10-09T06:42:34.936 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105323 12.13 12:cc0ff923:::benchmark_data_smithi107_479504_object5712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:42:34.937 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 581.98233859200002, 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.76640025000000001, 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.938 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105323 2023-10-09T06:42:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.942 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829728+0000", 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.943 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829726+0000", 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829734+0000", 2023-10-09T06:42:34.944 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9030000000000004e-06 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829753+0000", 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9004999999999998e-05 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829763+0000", 2023-10-09T06:42:34.945 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3130000000000004e-06 2023-10-09T06:42:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.946 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829812+0000", 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9391000000000002e-05 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.829824+0000", 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1806e-05 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.947 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.569832+0000", 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.74000779400000005 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.570011+0000", 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000179548 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.948 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.570169+0000", 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000157448 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.574027+0000", 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038583659999999998 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.949 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.576408+0000", 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002380868 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593714+0000", 2023-10-09T06:42:34.950 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017305803000000002 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593760+0000", 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6144999999999999e-05 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596128+0000", 2023-10-09T06:42:34.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023683630000000001 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105350 12.13 12:ca4b3448:::benchmark_data_smithi107_479504_object5739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 581.94495578399994, 2023-10-09T06:42:34.952 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.73142369299999999, 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105350 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.953 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867111+0000", 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867109+0000", 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867116+0000", 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8619999999999997e-06 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867132+0000", 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6353000000000001e-05 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867141+0000", 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3659999999999992e-06 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867199+0000", 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.817e-05 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.867210+0000", 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1029999999999999e-05 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.571921+0000", 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70471069399999997 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.572088+0000", 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016699999999999999 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.572281+0000", 2023-10-09T06:42:34.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00019307300000000001 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.578846+0000", 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0065657179999999999 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.960 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583402+0000", 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045557030000000004 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596203+0000", 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012800638 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.596270+0000", 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6743999999999997e-05 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598534+0000", 2023-10-09T06:42:34.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022649459999999999 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:105356 12.13 12:c80cc14f:::benchmark_data_smithi107_479504_object5745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 581.93493601900002, 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.72982949600000002, 2023-10-09T06:42:34.963 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 105356 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877131+0000", 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877129+0000", 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877137+0000", 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.622e-06 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877149+0000", 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1925e-05 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.966 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877157+0000", 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2349999999999999e-06 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877198+0000", 2023-10-09T06:42:34.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0849000000000001e-05 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:52.877205+0000", 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0530000000000001e-06 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583440+0000", 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.70623574200000006 2023-10-09T06:42:34.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583607+0000", 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000166047 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.583764+0000", 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015757600000000001 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593599+0000", 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.009835313 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.593646+0000", 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6295999999999999e-05 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598596+0000", 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0049505399999999998 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.598649+0000", 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2311000000000001e-05 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:32:53.606960+0000", 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0083114710000000008 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:180931 12.13 12:ca7369d7:::benchmark_data_smithi107_479504_object81320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:42:34.972 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 476.05491024000003, 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34981270800000003, 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 180931 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757156+0000", 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.974 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757155+0000", 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757160+0000", 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4e-06 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757172+0000", 2023-10-09T06:42:34.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1929999999999999e-05 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.757178+0000", 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0190000000000003e-06 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758011+0000", 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00083210300000000005 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758143+0000", 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000132414 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.758646+0000", 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00050300799999999999 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.764111+0000", 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054646440000000003 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:38.768008+0000", 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038976800000000002 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106146+0000", 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33813771100000001 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106198+0000", 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1839999999999998e-05 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:39.106969+0000", 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00077123899999999995 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194026 12.e 12:7a8b5efc:::benchmark_data_smithi107_479504_object94415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 456.880812257, 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35583712499999998, 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.981 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194026 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.982 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931257+0000", 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7099999999999999e-06 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931263+0000", 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2750000000000001e-06 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931268+0000", 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931301+0000", 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3293999999999997e-05 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931411+0000", 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109904 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931511+0000", 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9732e-05 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940660+0000", 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091492589999999999 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940805+0000", 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014499900000000001 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946883+0000", 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006077753 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946930+0000", 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6730000000000002e-05 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287091+0000", 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016162900000002 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194028 12.e 12:78e231c3:::benchmark_data_smithi107_479504_object94417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:42:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:42:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 456.880786908, 2023-10-09T06:42:34.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35590891000000002, 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194028 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:34.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931279+0000", 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931282+0000", 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6410000000000002e-06 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:34.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931288+0000", 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9329999999999998e-06 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931293+0000", 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.147e-06 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936701+0000", 2023-10-09T06:42:34.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054079769999999996 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936818+0000", 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011694 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:42:34.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936931+0000", 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011294999999999999 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946850+0000", 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099191780000000007 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946867+0000", 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6532e-05 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287104+0000", 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34023740699999999 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287151+0000", 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7079000000000001e-05 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287189+0000", 2023-10-09T06:42:34.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7576000000000001e-05 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194030 12.a 12:5bb1da5c:::benchmark_data_smithi107_479504_object94419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 456.88076954799999, 2023-10-09T06:42:34.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35582703900000001, 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194030 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:42:35.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931299+0000", 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0710000000000002e-06 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931304+0000", 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9439999999999996e-06 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931309+0000", 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4859999999999997e-06 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938266+0000", 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069569489999999996 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:35.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938372+0000", 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105999 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938476+0000", 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103669 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.942149+0000", 2023-10-09T06:42:35.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003673277 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.945613+0000", 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034637270000000002 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946916+0000", 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013029070000000001 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287077+0000", 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016102300000001 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287124+0000", 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7277999999999997e-05 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:35.006 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98792 13.13 13:cfe6566e:::benchmark_data_smithi107_535588_object98791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 338.55843924599998, 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97467514200000005, 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:35.007 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98792 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:42:35.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253625+0000", 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253660+0000", 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5043000000000003e-05 2023-10-09T06:42:35.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253679+0000", 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9123999999999999e-05 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253690+0000", 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0679000000000001e-05 2023-10-09T06:42:35.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259065+0000", 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053750710000000004 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259078+0000", 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3473e-05 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228190+0000", 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96911134899999996 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228302+0000", 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000112927 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98797 13.3 13:c2aca6be:::benchmark_data_smithi107_535588_object98796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 338.458944444, 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.85765422099999999, 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:42:35.013 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98797 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:42:35.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353120+0000", 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353150+0000", 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0685000000000002e-05 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353166+0000", 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5914e-05 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353175+0000", 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.935e-06 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:35.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353222+0000", 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6635999999999999e-05 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353232+0000", 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0478e-05 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:42:35.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188558+0000", 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83532556599999996 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188732+0000", 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173661 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188879+0000", 2023-10-09T06:42:35.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014706499999999999 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.192739+0000", 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038604580000000002 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.201831+0000", 2023-10-09T06:42:35.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090918390000000009 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210658+0000", 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088270789999999998 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210725+0000", 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7260000000000003e-05 2023-10-09T06:42:35.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210776+0000", 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1085000000000003e-05 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.021 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:42:35.022 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:42:35.022 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:42:35.096 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:35.097 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:34 smithi107 ceph-mon[220921]: pgmap v1628: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 87/116931 objects degraded (0.074%); 87/116931 objects misplaced (0.074%); 0 B/s, 95 objects/s recovering 2023-10-09T06:42:35.098 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:34 smithi107 ceph-mon[212750]: pgmap v1628: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 87/116931 objects degraded (0.074%); 87/116931 objects misplaced (0.074%); 0 B/s, 95 objects/s recovering 2023-10-09T06:42:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:35.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:34 smithi150 ceph-mon[194178]: pgmap v1628: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 87/116931 objects degraded (0.074%); 87/116931 objects misplaced (0.074%); 0 B/s, 95 objects/s recovering 2023-10-09T06:42:35.199 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:35.258 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:36.044 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[212750]: 14.2 scrub starts 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:35 smithi150 ceph-mon[194178]: 14.2 scrub starts 2023-10-09T06:42:36.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: osdmap e525: 8 total, 8 up, 8 in 2023-10-09T06:42:36.158 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: failed to encode map e525 with expected crc 2023-10-09T06:42:36.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:35 smithi107 ceph-mon[220921]: 14.2 scrub starts 2023-10-09T06:42:36.540 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:36.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:36.646 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:36.696 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:36.766 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:36.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:36.876 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:36.896 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: 14.2 scrub ok 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: pgmap v1630: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 119 objects/s recovering 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[212750]: 14.10 scrub starts 2023-10-09T06:42:36.897 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: 14.2 scrub ok 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: pgmap v1630: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 119 objects/s recovering 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: failed to encode map e526 with expected crc 2023-10-09T06:42:36.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:36 smithi107 ceph-mon[220921]: 14.10 scrub starts 2023-10-09T06:42:36.925 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:42:36.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:37.131 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:37.143 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:37.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:37.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: 14.2 scrub ok 2023-10-09T06:42:37.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: pgmap v1630: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 119 objects/s recovering 2023-10-09T06:42:37.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: failed to encode map e526 with expected crc 2023-10-09T06:42:37.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: failed to encode map e526 with expected crc 2023-10-09T06:42:37.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: osdmap e526: 8 total, 8 up, 8 in 2023-10-09T06:42:37.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: failed to encode map e526 with expected crc 2023-10-09T06:42:37.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: failed to encode map e526 with expected crc 2023-10-09T06:42:37.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: failed to encode map e526 with expected crc 2023-10-09T06:42:37.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:36 smithi150 ceph-mon[194178]: 14.10 scrub starts 2023-10-09T06:42:37.252 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:37.365 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:37.553 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:37.977 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:37.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:38.093 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[220921]: 14.10 scrub ok 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[220921]: 14.14 scrub starts 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[220921]: 14.14 scrub ok 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[212750]: 14.10 scrub ok 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[212750]: 14.14 scrub starts 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[212750]: 14.14 scrub ok 2023-10-09T06:42:38.119 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:37 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:38.201 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:37 smithi150 ceph-mon[194178]: 14.10 scrub ok 2023-10-09T06:42:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:37 smithi150 ceph-mon[194178]: 14.14 scrub starts 2023-10-09T06:42:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:37 smithi150 ceph-mon[194178]: 14.14 scrub ok 2023-10-09T06:42:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:37 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T06:42:38.957 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:42:39.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[212750]: pgmap v1632: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 65 objects/s recovering 2023-10-09T06:42:39.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[212750]: 14.11 scrub starts 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[212750]: 14.11 scrub ok 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/921509246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[220921]: pgmap v1632: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 65 objects/s recovering 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[220921]: 14.11 scrub starts 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[220921]: 14.11 scrub ok 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/921509246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:38 smithi150 ceph-mon[194178]: pgmap v1632: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 128 active+clean; 2.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 87/111273 objects degraded (0.078%); 87/111273 objects misplaced (0.078%); 0 B/s, 65 objects/s recovering 2023-10-09T06:42:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:38 smithi150 ceph-mon[194178]: 14.11 scrub starts 2023-10-09T06:42:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:38 smithi150 ceph-mon[194178]: 14.11 scrub ok 2023-10-09T06:42:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/921509246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:42:39.414 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:39.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:39.521 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:39.550 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:39.626 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:39.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:39.737 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:39.764 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:39.814 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:39.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:39.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:40.023 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:42:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:39 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:42:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:40.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:42:40.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: osdmap e527: 8 total, 8 up, 8 in 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: failed to encode map e527 with expected crc 2023-10-09T06:42:40.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:39 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T06:42:40.749 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:40.811 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:40.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:40.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:40.951 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: pgmap v1634: 129 pgs: 129 active+clean; 2.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[212750]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: pgmap v1634: 129 pgs: 129 active+clean; 2.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:40 smithi107 ceph-mon[220921]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.389 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:41.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: pgmap v1634: 129 pgs: 129 active+clean; 2.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/111273 objects degraded (0.078%), 1 pg degraded, 1 pg undersized) 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: osdmap e528: 8 total, 8 up, 8 in 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:40 smithi150 ceph-mon[194178]: failed to encode map e528 with expected crc 2023-10-09T06:42:41.494 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.523 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:41.567 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:41.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:41.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:41.672 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:41.699 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:41.741 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:41.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:41.825 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:42:41.826 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2174,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":529,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":8,"osd_in_since":1696832954,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":129}],"num_pgs":129,"num_pools":5,"num_objects":34927,"data_bytes":2287933368,"bytes_used":17734189056,"bytes_avail":750226096128,"bytes_total":767960285184},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":29,"modified":"2023-10-09T06:39:15.531037+0000","services":{}},"progress_events":{"0340c5d7-1441-466f-980c-d4021dcb5528":{"message":"Global Recovery Event (100s)\n [===========================.] ","progress":0.9922480583190918,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:42:41.846 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:41.872 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:41.914 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:41.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:42.022 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:42.047 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:42.093 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:42.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:42.150 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:42] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3793403231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:42:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3793403231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: osdmap e529: 8 total, 8 up, 8 in 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: failed to encode map e529 with expected crc 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T06:42:42.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3793403231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:42:42.200 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:42.228 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:42.244 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:42:42.278 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:42.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:42.289 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:42:42.328 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:42:42.364 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:42:42.389 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:42.398 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:42:42.452 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:42:42.464 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:42.498 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:42:42.543 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:42:42.564 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:42:42.579 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.579 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.579 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.580 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.580 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.580 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.580 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.580 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:42:42.851 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:42.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:42.955 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:43.039 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: failed to encode map e529 with expected crc 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: pgmap v1637: 129 pgs: 129 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/742820457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[220921]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: failed to encode map e529 with expected crc 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: pgmap v1637: 129 pgs: 129 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/742820457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:43.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:43.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:43.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:43.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:42 smithi107 ceph-mon[212750]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:42:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: failed to encode map e529 with expected crc 2023-10-09T06:42:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: pgmap v1637: 129 pgs: 129 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail 2023-10-09T06:42:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: osdmap e530: 8 total, 8 up, 8 in 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: failed to encode map e530 with expected crc 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/742820457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:43.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:42 smithi150 ceph-mon[194178]: osdmap e531: 8 total, 8 up, 8 in 2023-10-09T06:42:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[220921]: failed to encode map e531 with expected crc 2023-10-09T06:42:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:42:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:42:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[220921]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:42:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[212750]: failed to encode map e531 with expected crc 2023-10-09T06:42:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:42:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:42:44.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:44 smithi107 ceph-mon[212750]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:42:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:44 smithi150 ceph-mon[194178]: failed to encode map e531 with expected crc 2023-10-09T06:42:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T06:42:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:44 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T06:42:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:44 smithi150 ceph-mon[194178]: osdmap e532: 8 total, 8 up, 8 in 2023-10-09T06:42:44.550 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:44.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:44.573 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T06:42:44.574 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:42:44.662 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:44.745 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:44.852 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:44.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:44.958 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:44.973 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:44.989 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:45.005 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[212750]: pgmap v1640: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1922/96267 objects misplaced (1.997%) 2023-10-09T06:42:45.005 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[212750]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.005 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[212750]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.005 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[212750]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.005 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[212750]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:42:45.056 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:45.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:45.109 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:42:45.110 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:42:45.161 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:45.188 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:45.229 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:45.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:45.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[220921]: pgmap v1640: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1922/96267 objects misplaced (1.997%) 2023-10-09T06:42:45.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[220921]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[220921]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[220921]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.267 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:45 smithi107 ceph-mon[220921]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:42:45.334 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:45 smithi150 ceph-mon[194178]: pgmap v1640: 129 pgs: 1 active+recovering+undersized+remapped, 128 active+clean; 2.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1922/96267 objects misplaced (1.997%) 2023-10-09T06:42:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:45 smithi150 ceph-mon[194178]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:45 smithi150 ceph-mon[194178]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:45 smithi150 ceph-mon[194178]: failed to encode map e532 with expected crc 2023-10-09T06:42:45.358 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:45 smithi150 ceph-mon[194178]: osdmap e533: 8 total, 8 up, 8 in 2023-10-09T06:42:45.360 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:45.404 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:45.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:45.509 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:45.535 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:45.577 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:45.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:45.682 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:45.720 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:46.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[212750]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.019 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[212750]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:42:46.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[220921]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[220921]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[220921]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[220921]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:46 smithi107 ceph-mon[220921]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:42:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:46 smithi150 ceph-mon[194178]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:46 smithi150 ceph-mon[194178]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:46 smithi150 ceph-mon[194178]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:46 smithi150 ceph-mon[194178]: failed to encode map e533 with expected crc 2023-10-09T06:42:46.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:46 smithi150 ceph-mon[194178]: osdmap e534: 8 total, 8 up, 8 in 2023-10-09T06:42:46.648 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:42:47.032 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: pgmap v1643: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: pgmap v1643: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: Health check failed: Degraded data redundancy: 6298/93546 objects degraded (6.733%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: pgmap v1643: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: Health check failed: Degraded data redundancy: 6298/93546 objects degraded (6.733%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:42:47.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:47.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: failed to encode map e534 with expected crc 2023-10-09T06:42:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: Health check failed: Degraded data redundancy: 6298/93546 objects degraded (6.733%), 1 pg degraded (PG_DEGRADED) 2023-10-09T06:42:47.407 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:47.408 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:42:48.157 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:48.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:48.264 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:48.295 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:48.322 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:42:48.389 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:48.523 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:48.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:48.555 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:48.633 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:48.765 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:49.039 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:48 smithi107 ceph-mon[220921]: pgmap v1645: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:49.040 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:49.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:48 smithi107 ceph-mon[212750]: pgmap v1645: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:49.048 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:49.108 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:49.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:48 smithi150 ceph-mon[194178]: pgmap v1645: 129 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 127 active+clean; 1.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6298/93546 objects degraded (6.733%); 5071/93546 objects misplaced (5.421%) 2023-10-09T06:42:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:49.220 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:49.297 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:49.767 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:42:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2162474737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:49 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2162474737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:49 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2162474737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:49 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:42:50.775 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:50.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:50.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:50.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:50.931 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:51.005 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:51.010 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: pgmap v1646: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 9304/88995 objects degraded (10.455%); 4501/88995 objects misplaced (5.058%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:42:51.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:51.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:51.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:42:51.011 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:50 smithi150 ceph-mon[194178]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:51.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:51.113 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:51.139 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: pgmap v1646: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 9304/88995 objects degraded (10.455%); 4501/88995 objects misplaced (5.058%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:42:51.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[212750]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: pgmap v1646: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 9304/88995 objects degraded (10.455%); 4501/88995 objects misplaced (5.058%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: osdmap e535: 8 total, 8 up, 8 in 2023-10-09T06:42:51.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:50 smithi107 ceph-mon[220921]: failed to encode map e535 with expected crc 2023-10-09T06:42:51.180 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:51.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:51.288 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:51.348 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:52.015 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:51 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 9144/85506 objects degraded (10.694%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:52.015 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:51 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 9144/85506 objects degraded (10.694%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:52.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:51 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 9144/85506 objects degraded (10.694%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:52.404 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:42:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:42:52] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2023-10-09T06:42:52.651 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:42:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:52.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:42:52.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:52.758 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:52.794 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:52.853 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:52.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:52.961 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:53.057 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:53.073 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:52 smithi107 ceph-mon[212750]: pgmap v1648: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 9144/85506 objects degraded (10.694%); 4421/85506 objects misplaced (5.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:42:53.073 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:52 smithi107 ceph-mon[220921]: pgmap v1648: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 9144/85506 objects degraded (10.694%); 4421/85506 objects misplaced (5.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:42:53.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:52 smithi150 ceph-mon[194178]: pgmap v1648: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 9144/85506 objects degraded (10.694%); 4421/85506 objects misplaced (5.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T06:42:53.246 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:53.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:53.351 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:53.385 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:53.434 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:53.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:53.504 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:42:53.531 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:42:53.545 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:53.570 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:42:53.583 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:53.629 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:42:53.641 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:53.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:53.676 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:42:53.729 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:42:53.750 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:53.787 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:42:53.823 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.843 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:42:53.867 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:53.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/770858909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:53.966 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:53.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:53.977 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/770858909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:53.977 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/770858909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:42:54.078 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:42:54.144 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:42:54.157 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:42:54.163 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:54.527 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:42:54.853 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:54 smithi107 ceph-mon[212750]: pgmap v1649: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 9144/80595 objects degraded (11.346%); 4061/80595 objects misplaced (5.039%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:42:54.853 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:54 smithi107 ceph-mon[212750]: from='client.47273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:54.853 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:54 smithi107 ceph-mon[220921]: pgmap v1649: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 9144/80595 objects degraded (11.346%); 4061/80595 objects misplaced (5.039%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:42:54.854 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:54 smithi107 ceph-mon[220921]: from='client.47273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:54 smithi150 ceph-mon[194178]: pgmap v1649: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 9144/80595 objects degraded (11.346%); 4061/80595 objects misplaced (5.039%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T06:42:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:54 smithi150 ceph-mon[194178]: from='client.47273 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:42:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:55.694 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:42:55.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:42:55.796 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:55.870 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:56.672 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:56.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:56.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:42:56.853 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:56.967 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:56 smithi107 ceph-mon[212750]: pgmap v1650: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:42:56.974 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:56 smithi107 ceph-mon[220921]: pgmap v1650: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:42:57.093 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:42:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:57.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:42:57.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:56 smithi150 ceph-mon[194178]: pgmap v1650: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:42:57.203 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:42:57.247 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:57.251 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1649,"stamp":"2023-10-09T06:42:53.641324+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1759582136,"num_objects":26865,"num_object_clones":0,"num_object_copies":80595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9144,"num_objects_misplaced":4061,"num_objects_unfound":0,"num_objects_dirty":26865,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137120,"num_write_kb":4557470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":696,"num_bytes_recovered":42084300,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":193684,"ondisk_log_size":193684,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15815320,"kb_used_data":5621016,"kb_used_omap":20,"kb_used_meta":10193963,"kb_avail":734145896,"statfs":{"total":767960285184,"available":751765397504,"internally_reserved":0,"allocated":5755920384,"data_stored":5742467523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10438618766},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-282918912,"num_objects":-4317,"num_object_clones":0,"num_object_copies":-12951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2846,"num_objects_misplaced":-1010,"num_objects_unfound":0,"num_objects_dirty":-4317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4317,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":545,"num_bytes_recovered":35717120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.010991"},"pg_stats":[{"pgid":"14.17","version":"535'7666","reported_seq":7847,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.631914+0000","last_change":"2023-10-09T06:40:52.372943+0000","last_active":"2023-10-09T06:42:52.631914+0000","last_peered":"2023-10-09T06:42:52.631914+0000","last_clean":"2023-10-09T06:42:52.631914+0000","last_became_active":"2023-10-09T06:40:43.185643+0000","last_became_peered":"2023-10-09T06:40:43.185643+0000","last_unstale":"2023-10-09T06:42:52.631914+0000","last_undegraded":"2023-10-09T06:42:52.631914+0000","last_fullsized":"2023-10-09T06:42:52.631914+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4783","last_scrub_stamp":"2023-10-09T06:40:52.372858+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:52.372858+0000","objects_scrubbed":2511,"log_size":7666,"log_dups_size":0,"ondisk_log_size":7666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:22.222358+0000","scrub_duration":0.27153366200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48758784,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2883,"num_write_kb":35712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":829,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722082+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:42:45.722082+0000","last_peered":"2023-10-09T06:42:45.722082+0000","last_clean":"2023-10-09T06:42:45.722082+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:42:45.722082+0000","last_undegraded":"2023-10-09T06:42:45.722082+0000","last_fullsized":"2023-10-09T06:42:45.722082+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":514,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692829+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:42:45.692829+0000","last_peered":"2023-10-09T06:42:45.692829+0000","last_clean":"2023-10-09T06:42:45.692829+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:42:45.692829+0000","last_undegraded":"2023-10-09T06:42:45.692829+0000","last_fullsized":"2023-10-09T06:42:45.692829+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":237,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708262+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:42:44.708262+0000","last_peered":"2023-10-09T06:42:44.708262+0000","last_clean":"2023-10-09T06:42:44.708262+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:42:44.708262+0000","last_undegraded":"2023-10-09T06:42:44.708262+0000","last_fullsized":"2023-10-09T06:42:44.708262+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"534'7478","reported_seq":7611,"reported_epoch":534,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:49.111918+0000","last_change":"2023-10-09T06:42:45.839310+0000","last_active":"2023-10-09T06:42:49.111918+0000","last_peered":"2023-10-09T06:42:49.111918+0000","last_clean":"2023-10-09T06:42:43.293915+0000","last_became_active":"2023-10-09T06:42:45.820286+0000","last_became_peered":"2023-10-09T06:42:45.820286+0000","last_unstale":"2023-10-09T06:42:49.111918+0000","last_undegraded":"2023-10-09T06:42:45.719211+0000","last_fullsized":"2023-10-09T06:42:45.692966+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7291","last_scrub_stamp":"2023-10-09T06:42:37.253113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:37.253113+0000","objects_scrubbed":1009,"log_size":7478,"log_dups_size":0,"ondisk_log_size":7478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:53.601935+0000","scrub_duration":0.250396232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53739520,"num_objects":820,"num_object_clones":0,"num_object_copies":2460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3006,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":189,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6","objects":-77},{"shards":"3,6,7","objects":820}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":790,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692610+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:42:45.692610+0000","last_peered":"2023-10-09T06:42:45.692610+0000","last_clean":"2023-10-09T06:42:45.692610+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:42:45.692610+0000","last_undegraded":"2023-10-09T06:42:45.692610+0000","last_fullsized":"2023-10-09T06:42:45.692610+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":532,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.700830+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:42:45.700830+0000","last_peered":"2023-10-09T06:42:45.700830+0000","last_clean":"2023-10-09T06:42:45.700830+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:42:45.700830+0000","last_undegraded":"2023-10-09T06:42:45.700830+0000","last_fullsized":"2023-10-09T06:42:45.700830+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":537,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.691324+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:42:45.691324+0000","last_peered":"2023-10-09T06:42:45.691324+0000","last_clean":"2023-10-09T06:42:45.691324+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:42:45.691324+0000","last_undegraded":"2023-10-09T06:42:45.691324+0000","last_fullsized":"2023-10-09T06:42:45.691324+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"535'7570","reported_seq":7635,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.854326+0000","last_change":"2023-10-09T06:40:43.168550+0000","last_active":"2023-10-09T06:42:49.854326+0000","last_peered":"2023-10-09T06:42:49.854326+0000","last_clean":"2023-10-09T06:42:49.854326+0000","last_became_active":"2023-10-09T06:40:43.166693+0000","last_became_peered":"2023-10-09T06:40:43.166693+0000","last_unstale":"2023-10-09T06:42:49.854326+0000","last_undegraded":"2023-10-09T06:42:49.854326+0000","last_fullsized":"2023-10-09T06:42:49.854326+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5309,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":782,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692720+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:42:45.692720+0000","last_peered":"2023-10-09T06:42:45.692720+0000","last_clean":"2023-10-09T06:42:45.692720+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:42:45.692720+0000","last_undegraded":"2023-10-09T06:42:45.692720+0000","last_fullsized":"2023-10-09T06:42:45.692720+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":255,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.707633+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:42:44.707633+0000","last_peered":"2023-10-09T06:42:44.707633+0000","last_clean":"2023-10-09T06:42:44.707633+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:42:44.707633+0000","last_undegraded":"2023-10-09T06:42:44.707633+0000","last_fullsized":"2023-10-09T06:42:44.707633+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":524,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692850+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:42:45.692850+0000","last_peered":"2023-10-09T06:42:45.692850+0000","last_clean":"2023-10-09T06:42:45.692850+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:42:45.692850+0000","last_undegraded":"2023-10-09T06:42:45.692850+0000","last_fullsized":"2023-10-09T06:42:45.692850+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":643,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722418+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:42:45.722418+0000","last_peered":"2023-10-09T06:42:45.722418+0000","last_clean":"2023-10-09T06:42:45.722418+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:42:45.722418+0000","last_undegraded":"2023-10-09T06:42:45.722418+0000","last_fullsized":"2023-10-09T06:42:45.722418+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690744+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:42:45.690744+0000","last_peered":"2023-10-09T06:42:45.690744+0000","last_clean":"2023-10-09T06:42:45.690744+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:42:45.690744+0000","last_undegraded":"2023-10-09T06:42:45.690744+0000","last_fullsized":"2023-10-09T06:42:45.690744+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":237,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708767+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:42:44.708767+0000","last_peered":"2023-10-09T06:42:44.708767+0000","last_clean":"2023-10-09T06:42:44.708767+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:42:44.708767+0000","last_undegraded":"2023-10-09T06:42:44.708767+0000","last_fullsized":"2023-10-09T06:42:44.708767+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":680,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.729874+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:42:45.729874+0000","last_peered":"2023-10-09T06:42:45.729874+0000","last_clean":"2023-10-09T06:42:45.729874+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:42:45.729874+0000","last_undegraded":"2023-10-09T06:42:45.729874+0000","last_fullsized":"2023-10-09T06:42:45.729874+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.857264+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:42:45.857264+0000","last_peered":"2023-10-09T06:42:45.857264+0000","last_clean":"2023-10-09T06:42:45.857264+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:42:45.857264+0000","last_undegraded":"2023-10-09T06:42:45.857264+0000","last_fullsized":"2023-10-09T06:42:45.857264+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":237,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.709535+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:42:44.709535+0000","last_peered":"2023-10-09T06:42:44.709535+0000","last_clean":"2023-10-09T06:42:44.709535+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:42:44.709535+0000","last_undegraded":"2023-10-09T06:42:44.709535+0000","last_fullsized":"2023-10-09T06:42:44.709535+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"535'7995","reported_seq":8059,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.845135+0000","last_change":"2023-10-09T06:42:32.156372+0000","last_active":"2023-10-09T06:42:49.845135+0000","last_peered":"2023-10-09T06:42:49.845135+0000","last_clean":"2023-10-09T06:42:49.845135+0000","last_became_active":"2023-10-09T06:40:43.180666+0000","last_became_peered":"2023-10-09T06:40:43.180666+0000","last_unstale":"2023-10-09T06:42:49.845135+0000","last_undegraded":"2023-10-09T06:42:49.845135+0000","last_fullsized":"2023-10-09T06:42:49.845135+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7995,"log_dups_size":0,"ondisk_log_size":7995,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60325888,"num_objects":921,"num_object_clones":0,"num_object_copies":2763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":208800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":664,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690470+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:42:45.690470+0000","last_peered":"2023-10-09T06:42:45.690470+0000","last_clean":"2023-10-09T06:42:45.690470+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:42:45.690470+0000","last_undegraded":"2023-10-09T06:42:45.690470+0000","last_fullsized":"2023-10-09T06:42:45.690470+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":416,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722786+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:42:45.722786+0000","last_peered":"2023-10-09T06:42:45.722786+0000","last_clean":"2023-10-09T06:42:45.722786+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:42:45.722786+0000","last_undegraded":"2023-10-09T06:42:45.722786+0000","last_fullsized":"2023-10-09T06:42:45.722786+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":473,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701957+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:42:45.701957+0000","last_peered":"2023-10-09T06:42:45.701957+0000","last_clean":"2023-10-09T06:42:45.701957+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:42:45.701957+0000","last_undegraded":"2023-10-09T06:42:45.701957+0000","last_fullsized":"2023-10-09T06:42:45.701957+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"534'7707","reported_seq":7833,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.016521+0000","last_change":"2023-10-09T06:40:50.305746+0000","last_active":"2023-10-09T06:42:49.016521+0000","last_peered":"2023-10-09T06:42:49.016521+0000","last_clean":"2023-10-09T06:42:49.016521+0000","last_became_active":"2023-10-09T06:40:43.093584+0000","last_became_peered":"2023-10-09T06:40:43.093584+0000","last_unstale":"2023-10-09T06:42:49.016521+0000","last_undegraded":"2023-10-09T06:42:49.016521+0000","last_fullsized":"2023-10-09T06:42:49.016521+0000","mapping_epoch":500,"log_start":"493'84","ondisk_log_start":"493'84","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4832","last_scrub_stamp":"2023-10-09T06:40:50.305626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:50.305626+0000","objects_scrubbed":2516,"log_size":7623,"log_dups_size":0,"ondisk_log_size":7623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:57.896131+0000","scrub_duration":0.56208410600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2884,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":645,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701357+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:42:45.701357+0000","last_peered":"2023-10-09T06:42:45.701357+0000","last_clean":"2023-10-09T06:42:45.701357+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:42:45.701357+0000","last_undegraded":"2023-10-09T06:42:45.701357+0000","last_fullsized":"2023-10-09T06:42:45.701357+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":541,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693558+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:42:45.693558+0000","last_peered":"2023-10-09T06:42:45.693558+0000","last_clean":"2023-10-09T06:42:45.693558+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:42:45.693558+0000","last_undegraded":"2023-10-09T06:42:45.693558+0000","last_fullsized":"2023-10-09T06:42:45.693558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":505,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693079+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:42:45.693079+0000","last_peered":"2023-10-09T06:42:45.693079+0000","last_clean":"2023-10-09T06:42:45.693079+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:42:45.693079+0000","last_undegraded":"2023-10-09T06:42:45.693079+0000","last_fullsized":"2023-10-09T06:42:45.693079+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":809,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690110+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:42:45.690110+0000","last_peered":"2023-10-09T06:42:45.690110+0000","last_clean":"2023-10-09T06:42:45.690110+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:42:45.690110+0000","last_undegraded":"2023-10-09T06:42:45.690110+0000","last_fullsized":"2023-10-09T06:42:45.690110+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":384,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722567+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:42:45.722567+0000","last_peered":"2023-10-09T06:42:45.722567+0000","last_clean":"2023-10-09T06:42:45.722567+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:42:45.722567+0000","last_undegraded":"2023-10-09T06:42:45.722567+0000","last_fullsized":"2023-10-09T06:42:45.722567+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.714087+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:42:44.714087+0000","last_peered":"2023-10-09T06:42:44.714087+0000","last_clean":"2023-10-09T06:42:44.714087+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:42:44.714087+0000","last_undegraded":"2023-10-09T06:42:44.714087+0000","last_fullsized":"2023-10-09T06:42:44.714087+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":807,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722080+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:42:45.722080+0000","last_peered":"2023-10-09T06:42:45.722080+0000","last_clean":"2023-10-09T06:42:45.722080+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:42:45.722080+0000","last_undegraded":"2023-10-09T06:42:45.722080+0000","last_fullsized":"2023-10-09T06:42:45.722080+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":534,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690053+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:42:45.690053+0000","last_peered":"2023-10-09T06:42:45.690053+0000","last_clean":"2023-10-09T06:42:45.690053+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:42:45.690053+0000","last_undegraded":"2023-10-09T06:42:45.690053+0000","last_fullsized":"2023-10-09T06:42:45.690053+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":355,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722709+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:42:45.722709+0000","last_peered":"2023-10-09T06:42:45.722709+0000","last_clean":"2023-10-09T06:42:45.722709+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:42:45.722709+0000","last_undegraded":"2023-10-09T06:42:45.722709+0000","last_fullsized":"2023-10-09T06:42:45.722709+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":863,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701235+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:42:45.701235+0000","last_peered":"2023-10-09T06:42:45.701235+0000","last_clean":"2023-10-09T06:42:45.701235+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:42:45.701235+0000","last_undegraded":"2023-10-09T06:42:45.701235+0000","last_fullsized":"2023-10-09T06:42:45.701235+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":378,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722239+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:42:45.722239+0000","last_peered":"2023-10-09T06:42:45.722239+0000","last_clean":"2023-10-09T06:42:45.722239+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:42:45.722239+0000","last_undegraded":"2023-10-09T06:42:45.722239+0000","last_fullsized":"2023-10-09T06:42:45.722239+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":211,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.691013+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:42:45.691013+0000","last_peered":"2023-10-09T06:42:45.691013+0000","last_clean":"2023-10-09T06:42:45.691013+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:42:45.691013+0000","last_undegraded":"2023-10-09T06:42:45.691013+0000","last_fullsized":"2023-10-09T06:42:45.691013+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":649,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.858749+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:42:45.858749+0000","last_peered":"2023-10-09T06:42:45.858749+0000","last_clean":"2023-10-09T06:42:45.858749+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:42:45.858749+0000","last_undegraded":"2023-10-09T06:42:45.858749+0000","last_fullsized":"2023-10-09T06:42:45.858749+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":613,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692383+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:42:45.692383+0000","last_peered":"2023-10-09T06:42:45.692383+0000","last_clean":"2023-10-09T06:42:45.692383+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:42:45.692383+0000","last_undegraded":"2023-10-09T06:42:45.692383+0000","last_fullsized":"2023-10-09T06:42:45.692383+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.710024+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:42:44.710024+0000","last_peered":"2023-10-09T06:42:44.710024+0000","last_clean":"2023-10-09T06:42:44.710024+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:42:44.710024+0000","last_undegraded":"2023-10-09T06:42:44.710024+0000","last_fullsized":"2023-10-09T06:42:44.710024+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"535'7762","reported_seq":7904,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.756092+0000","last_change":"2023-10-09T06:42:35.324097+0000","last_active":"2023-10-09T06:42:52.756092+0000","last_peered":"2023-10-09T06:42:52.756092+0000","last_clean":"2023-10-09T06:42:52.756092+0000","last_became_active":"2023-10-09T06:40:43.094441+0000","last_became_peered":"2023-10-09T06:40:43.094441+0000","last_unstale":"2023-10-09T06:42:52.756092+0000","last_undegraded":"2023-10-09T06:42:52.756092+0000","last_fullsized":"2023-10-09T06:42:52.756092+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"525'7435","last_scrub_stamp":"2023-10-09T06:42:35.324027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:35.324027+0000","objects_scrubbed":1088,"log_size":7762,"log_dups_size":0,"ondisk_log_size":7762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:40.756290+0000","scrub_duration":0.32391870699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49676288,"num_objects":758,"num_object_clones":0,"num_object_copies":2274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":330,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":662,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722478+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:42:45.722478+0000","last_peered":"2023-10-09T06:42:45.722478+0000","last_clean":"2023-10-09T06:42:45.722478+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:42:45.722478+0000","last_undegraded":"2023-10-09T06:42:45.722478+0000","last_fullsized":"2023-10-09T06:42:45.722478+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":420,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722500+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:42:45.722500+0000","last_peered":"2023-10-09T06:42:45.722500+0000","last_clean":"2023-10-09T06:42:45.722500+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:42:45.722500+0000","last_undegraded":"2023-10-09T06:42:45.722500+0000","last_fullsized":"2023-10-09T06:42:45.722500+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":318,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722312+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:42:45.722312+0000","last_peered":"2023-10-09T06:42:45.722312+0000","last_clean":"2023-10-09T06:42:45.722312+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:42:45.722312+0000","last_undegraded":"2023-10-09T06:42:45.722312+0000","last_fullsized":"2023-10-09T06:42:45.722312+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"535'7788","reported_seq":7855,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754639+0000","last_change":"2023-10-09T06:42:15.187041+0000","last_active":"2023-10-09T06:42:49.754639+0000","last_peered":"2023-10-09T06:42:49.754639+0000","last_clean":"2023-10-09T06:42:49.754639+0000","last_became_active":"2023-10-09T06:40:43.165023+0000","last_became_peered":"2023-10-09T06:40:43.165023+0000","last_unstale":"2023-10-09T06:42:49.754639+0000","last_undegraded":"2023-10-09T06:42:49.754639+0000","last_fullsized":"2023-10-09T06:42:49.754639+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7788,"log_dups_size":0,"ondisk_log_size":7788,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5468,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":921,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.713957+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:42:44.713957+0000","last_peered":"2023-10-09T06:42:44.713957+0000","last_clean":"2023-10-09T06:42:44.713957+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:42:44.713957+0000","last_undegraded":"2023-10-09T06:42:44.713957+0000","last_fullsized":"2023-10-09T06:42:44.713957+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":382,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722627+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:42:45.722627+0000","last_peered":"2023-10-09T06:42:45.722627+0000","last_clean":"2023-10-09T06:42:45.722627+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:42:45.722627+0000","last_undegraded":"2023-10-09T06:42:45.722627+0000","last_fullsized":"2023-10-09T06:42:45.722627+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":532,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692788+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:42:45.692788+0000","last_peered":"2023-10-09T06:42:45.692788+0000","last_clean":"2023-10-09T06:42:45.692788+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:42:45.692788+0000","last_undegraded":"2023-10-09T06:42:45.692788+0000","last_fullsized":"2023-10-09T06:42:45.692788+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"534'7675","reported_seq":7742,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.714516+0000","last_change":"2023-10-09T06:42:26.019283+0000","last_active":"2023-10-09T06:42:49.714516+0000","last_peered":"2023-10-09T06:42:49.714516+0000","last_clean":"2023-10-09T06:42:49.714516+0000","last_became_active":"2023-10-09T06:40:43.080150+0000","last_became_peered":"2023-10-09T06:40:43.080150+0000","last_unstale":"2023-10-09T06:42:49.714516+0000","last_undegraded":"2023-10-09T06:42:49.714516+0000","last_fullsized":"2023-10-09T06:42:49.714516+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7675,"log_dups_size":0,"ondisk_log_size":7675,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58490880,"num_objects":892,"num_object_clones":0,"num_object_copies":2676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5384,"num_write_kb":200864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":836,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692808+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:42:45.692808+0000","last_peered":"2023-10-09T06:42:45.692808+0000","last_clean":"2023-10-09T06:42:45.692808+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:42:45.692808+0000","last_undegraded":"2023-10-09T06:42:45.692808+0000","last_fullsized":"2023-10-09T06:42:45.692808+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":438,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722624+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:42:45.722624+0000","last_peered":"2023-10-09T06:42:45.722624+0000","last_clean":"2023-10-09T06:42:45.722624+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:42:45.722624+0000","last_undegraded":"2023-10-09T06:42:45.722624+0000","last_fullsized":"2023-10-09T06:42:45.722624+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":511,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693429+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:42:45.693429+0000","last_peered":"2023-10-09T06:42:45.693429+0000","last_clean":"2023-10-09T06:42:45.693429+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:42:45.693429+0000","last_undegraded":"2023-10-09T06:42:45.693429+0000","last_fullsized":"2023-10-09T06:42:45.693429+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"534'7720","reported_seq":7849,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:48.986436+0000","last_change":"2023-10-09T06:40:47.766598+0000","last_active":"2023-10-09T06:42:48.986436+0000","last_peered":"2023-10-09T06:42:48.986436+0000","last_clean":"2023-10-09T06:42:48.986436+0000","last_became_active":"2023-10-09T06:40:43.101954+0000","last_became_peered":"2023-10-09T06:40:43.101954+0000","last_unstale":"2023-10-09T06:42:48.986436+0000","last_undegraded":"2023-10-09T06:42:48.986436+0000","last_fullsized":"2023-10-09T06:42:48.986436+0000","mapping_epoch":489,"log_start":"493'223","ondisk_log_start":"493'223","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4698","last_scrub_stamp":"2023-10-09T06:40:47.766356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.766356+0000","objects_scrubbed":2441,"log_size":7497,"log_dups_size":0,"ondisk_log_size":7497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:35.858607+0000","scrub_duration":1.024576309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61145088,"num_objects":933,"num_object_clones":0,"num_object_copies":2799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3032,"num_write_kb":48768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":659,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693780+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:42:45.693780+0000","last_peered":"2023-10-09T06:42:45.693780+0000","last_clean":"2023-10-09T06:42:45.693780+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:42:45.693780+0000","last_undegraded":"2023-10-09T06:42:45.693780+0000","last_fullsized":"2023-10-09T06:42:45.693780+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":105,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.714146+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:42:44.714146+0000","last_peered":"2023-10-09T06:42:44.714146+0000","last_clean":"2023-10-09T06:42:44.714146+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:42:44.714146+0000","last_undegraded":"2023-10-09T06:42:44.714146+0000","last_fullsized":"2023-10-09T06:42:44.714146+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":499,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690502+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:42:45.690502+0000","last_peered":"2023-10-09T06:42:45.690502+0000","last_clean":"2023-10-09T06:42:45.690502+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:42:45.690502+0000","last_undegraded":"2023-10-09T06:42:45.690502+0000","last_fullsized":"2023-10-09T06:42:45.690502+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"535'7650","reported_seq":7716,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.849821+0000","last_change":"2023-10-09T06:42:18.037795+0000","last_active":"2023-10-09T06:42:49.849821+0000","last_peered":"2023-10-09T06:42:49.849821+0000","last_clean":"2023-10-09T06:42:49.849821+0000","last_became_active":"2023-10-09T06:40:43.159937+0000","last_became_peered":"2023-10-09T06:40:43.159937+0000","last_unstale":"2023-10-09T06:42:49.849821+0000","last_undegraded":"2023-10-09T06:42:49.849821+0000","last_fullsized":"2023-10-09T06:42:49.849821+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7650,"log_dups_size":0,"ondisk_log_size":7650,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5370,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":887,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722143+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:42:45.722143+0000","last_peered":"2023-10-09T06:42:45.722143+0000","last_clean":"2023-10-09T06:42:45.722143+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:42:45.722143+0000","last_undegraded":"2023-10-09T06:42:45.722143+0000","last_fullsized":"2023-10-09T06:42:45.722143+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":263,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708932+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:42:44.708932+0000","last_peered":"2023-10-09T06:42:44.708932+0000","last_clean":"2023-10-09T06:42:44.708932+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:42:44.708932+0000","last_undegraded":"2023-10-09T06:42:44.708932+0000","last_fullsized":"2023-10-09T06:42:44.708932+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":515,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693380+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:42:45.693380+0000","last_peered":"2023-10-09T06:42:45.693380+0000","last_clean":"2023-10-09T06:42:45.693380+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:42:45.693380+0000","last_undegraded":"2023-10-09T06:42:45.693380+0000","last_fullsized":"2023-10-09T06:42:45.693380+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"535'7775","reported_seq":7945,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.612341+0000","last_change":"2023-10-09T06:40:45.736296+0000","last_active":"2023-10-09T06:42:52.612341+0000","last_peered":"2023-10-09T06:42:52.612341+0000","last_clean":"2023-10-09T06:42:52.612341+0000","last_became_active":"2023-10-09T06:40:43.168245+0000","last_became_peered":"2023-10-09T06:40:43.168245+0000","last_unstale":"2023-10-09T06:42:52.612341+0000","last_undegraded":"2023-10-09T06:42:52.612341+0000","last_fullsized":"2023-10-09T06:42:52.612341+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4658","last_scrub_stamp":"2023-10-09T06:40:45.736230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:45.736230+0000","objects_scrubbed":2357,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:31.145769+0000","scrub_duration":0.58060363000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3126,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":569,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708809+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:42:44.708809+0000","last_peered":"2023-10-09T06:42:44.708809+0000","last_clean":"2023-10-09T06:42:44.708809+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:42:44.708809+0000","last_undegraded":"2023-10-09T06:42:44.708809+0000","last_fullsized":"2023-10-09T06:42:44.708809+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":492,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693312+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:42:45.693312+0000","last_peered":"2023-10-09T06:42:45.693312+0000","last_clean":"2023-10-09T06:42:45.693312+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:42:45.693312+0000","last_undegraded":"2023-10-09T06:42:45.693312+0000","last_fullsized":"2023-10-09T06:42:45.693312+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":512,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693558+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:42:45.693558+0000","last_peered":"2023-10-09T06:42:45.693558+0000","last_clean":"2023-10-09T06:42:45.693558+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:42:45.693558+0000","last_undegraded":"2023-10-09T06:42:45.693558+0000","last_fullsized":"2023-10-09T06:42:45.693558+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"535'7680","reported_seq":7855,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.823621+0000","last_change":"2023-10-09T06:40:51.573557+0000","last_active":"2023-10-09T06:42:52.823621+0000","last_peered":"2023-10-09T06:42:52.823621+0000","last_clean":"2023-10-09T06:42:52.823621+0000","last_became_active":"2023-10-09T06:40:43.081959+0000","last_became_peered":"2023-10-09T06:40:43.081959+0000","last_unstale":"2023-10-09T06:42:52.823621+0000","last_undegraded":"2023-10-09T06:42:52.823621+0000","last_fullsized":"2023-10-09T06:42:52.823621+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'4739","last_scrub_stamp":"2023-10-09T06:40:51.573471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:51.573471+0000","objects_scrubbed":2521,"log_size":7680,"log_dups_size":0,"ondisk_log_size":7680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:30.924217+0000","scrub_duration":0.34142884899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52166656,"num_objects":796,"num_object_clones":0,"num_object_copies":2388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2941,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":872,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701321+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:42:45.701321+0000","last_peered":"2023-10-09T06:42:45.701321+0000","last_clean":"2023-10-09T06:42:45.701321+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:42:45.701321+0000","last_undegraded":"2023-10-09T06:42:45.701321+0000","last_fullsized":"2023-10-09T06:42:45.701321+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":516,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692666+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:42:45.692666+0000","last_peered":"2023-10-09T06:42:45.692666+0000","last_clean":"2023-10-09T06:42:45.692666+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:42:45.692666+0000","last_undegraded":"2023-10-09T06:42:45.692666+0000","last_fullsized":"2023-10-09T06:42:45.692666+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":495,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693919+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:42:45.693919+0000","last_peered":"2023-10-09T06:42:45.693919+0000","last_clean":"2023-10-09T06:42:45.693919+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:42:45.693919+0000","last_undegraded":"2023-10-09T06:42:45.693919+0000","last_fullsized":"2023-10-09T06:42:45.693919+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"535'7587","reported_seq":7713,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.781706+0000","last_change":"2023-10-09T06:40:47.192403+0000","last_active":"2023-10-09T06:42:49.781706+0000","last_peered":"2023-10-09T06:42:49.781706+0000","last_clean":"2023-10-09T06:42:49.781706+0000","last_became_active":"2023-10-09T06:40:43.074962+0000","last_became_peered":"2023-10-09T06:40:43.074962+0000","last_unstale":"2023-10-09T06:42:49.781706+0000","last_undegraded":"2023-10-09T06:42:49.781706+0000","last_fullsized":"2023-10-09T06:42:49.781706+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4664","last_scrub_stamp":"2023-10-09T06:40:47.192137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.192137+0000","objects_scrubbed":2418,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:24.365070+0000","scrub_duration":0.455376642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54132736,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":497,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693231+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:42:45.693231+0000","last_peered":"2023-10-09T06:42:45.693231+0000","last_clean":"2023-10-09T06:42:45.693231+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:42:45.693231+0000","last_undegraded":"2023-10-09T06:42:45.693231+0000","last_fullsized":"2023-10-09T06:42:45.693231+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":238,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692679+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:42:45.692679+0000","last_peered":"2023-10-09T06:42:45.692679+0000","last_clean":"2023-10-09T06:42:45.692679+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:42:45.692679+0000","last_undegraded":"2023-10-09T06:42:45.692679+0000","last_fullsized":"2023-10-09T06:42:45.692679+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":492,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.700993+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:42:45.700993+0000","last_peered":"2023-10-09T06:42:45.700993+0000","last_clean":"2023-10-09T06:42:45.700993+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:42:45.700993+0000","last_undegraded":"2023-10-09T06:42:45.700993+0000","last_fullsized":"2023-10-09T06:42:45.700993+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"535'8035","reported_seq":8100,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753132+0000","last_change":"2023-10-09T06:42:25.028645+0000","last_active":"2023-10-09T06:42:49.753132+0000","last_peered":"2023-10-09T06:42:49.753132+0000","last_clean":"2023-10-09T06:42:49.753132+0000","last_became_active":"2023-10-09T06:40:43.080919+0000","last_became_peered":"2023-10-09T06:40:43.080919+0000","last_unstale":"2023-10-09T06:42:49.753132+0000","last_undegraded":"2023-10-09T06:42:49.753132+0000","last_fullsized":"2023-10-09T06:42:49.753132+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8035,"log_dups_size":0,"ondisk_log_size":8035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57311232,"num_objects":874,"num_object_clones":0,"num_object_copies":2622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5644,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":827,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692899+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:42:45.692899+0000","last_peered":"2023-10-09T06:42:45.692899+0000","last_clean":"2023-10-09T06:42:45.692899+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:42:45.692899+0000","last_undegraded":"2023-10-09T06:42:45.692899+0000","last_fullsized":"2023-10-09T06:42:45.692899+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":544,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701175+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:42:45.701175+0000","last_peered":"2023-10-09T06:42:45.701175+0000","last_clean":"2023-10-09T06:42:45.701175+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:42:45.701175+0000","last_undegraded":"2023-10-09T06:42:45.701175+0000","last_fullsized":"2023-10-09T06:42:45.701175+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":105,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708171+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:42:44.708171+0000","last_peered":"2023-10-09T06:42:44.708171+0000","last_clean":"2023-10-09T06:42:44.708171+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:42:44.708171+0000","last_undegraded":"2023-10-09T06:42:44.708171+0000","last_fullsized":"2023-10-09T06:42:44.708171+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":478,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701619+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:42:45.701619+0000","last_peered":"2023-10-09T06:42:45.701619+0000","last_clean":"2023-10-09T06:42:45.701619+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:42:45.701619+0000","last_undegraded":"2023-10-09T06:42:45.701619+0000","last_fullsized":"2023-10-09T06:42:45.701619+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"535'10957","reported_seq":11011,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.719329+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:42:52.719329+0000","last_peered":"2023-10-09T06:42:52.719329+0000","last_clean":"2023-10-09T06:42:52.719329+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:42:52.719329+0000","last_undegraded":"2023-10-09T06:42:52.719329+0000","last_fullsized":"2023-10-09T06:42:52.719329+0000","mapping_epoch":489,"log_start":"508'5632","ondisk_log_start":"508'5632","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5325,"log_dups_size":0,"ondisk_log_size":5325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103612416,"num_objects":1581,"num_object_clones":0,"num_object_copies":4743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10957,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":715,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690104+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:42:45.690104+0000","last_peered":"2023-10-09T06:42:45.690104+0000","last_clean":"2023-10-09T06:42:45.690104+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:42:45.690104+0000","last_undegraded":"2023-10-09T06:42:45.690104+0000","last_fullsized":"2023-10-09T06:42:45.690104+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":267,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701754+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:42:45.701754+0000","last_peered":"2023-10-09T06:42:45.701754+0000","last_clean":"2023-10-09T06:42:45.701754+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:42:45.701754+0000","last_undegraded":"2023-10-09T06:42:45.701754+0000","last_fullsized":"2023-10-09T06:42:45.701754+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":516,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692937+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:42:45.692937+0000","last_peered":"2023-10-09T06:42:45.692937+0000","last_clean":"2023-10-09T06:42:45.692937+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:42:45.692937+0000","last_undegraded":"2023-10-09T06:42:45.692937+0000","last_fullsized":"2023-10-09T06:42:45.692937+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":508,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701085+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:42:45.701085+0000","last_peered":"2023-10-09T06:42:45.701085+0000","last_clean":"2023-10-09T06:42:45.701085+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:42:45.701085+0000","last_undegraded":"2023-10-09T06:42:45.701085+0000","last_fullsized":"2023-10-09T06:42:45.701085+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"535'10865","reported_seq":10920,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.840627+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:42:52.840627+0000","last_peered":"2023-10-09T06:42:52.840627+0000","last_clean":"2023-10-09T06:42:52.840627+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:42:52.840627+0000","last_undegraded":"2023-10-09T06:42:52.840627+0000","last_fullsized":"2023-10-09T06:42:52.840627+0000","mapping_epoch":489,"log_start":"498'4400","ondisk_log_start":"498'4400","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101318680,"num_objects":1547,"num_object_clones":0,"num_object_copies":4641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1547,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10865,"num_write_kb":397121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":877,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722286+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:42:45.722286+0000","last_peered":"2023-10-09T06:42:45.722286+0000","last_clean":"2023-10-09T06:42:45.722286+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:42:45.722286+0000","last_undegraded":"2023-10-09T06:42:45.722286+0000","last_fullsized":"2023-10-09T06:42:45.722286+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":528,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.709403+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:42:44.709403+0000","last_peered":"2023-10-09T06:42:44.709403+0000","last_clean":"2023-10-09T06:42:44.709403+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:42:44.709403+0000","last_undegraded":"2023-10-09T06:42:44.709403+0000","last_fullsized":"2023-10-09T06:42:44.709403+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":516,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.700714+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:42:45.700714+0000","last_peered":"2023-10-09T06:42:45.700714+0000","last_clean":"2023-10-09T06:42:45.700714+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:42:45.700714+0000","last_undegraded":"2023-10-09T06:42:45.700714+0000","last_fullsized":"2023-10-09T06:42:45.700714+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":538,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701064+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:42:45.701064+0000","last_peered":"2023-10-09T06:42:45.701064+0000","last_clean":"2023-10-09T06:42:45.701064+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:42:45.701064+0000","last_undegraded":"2023-10-09T06:42:45.701064+0000","last_fullsized":"2023-10-09T06:42:45.701064+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"535'11148","reported_seq":11202,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.623088+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:42:52.623088+0000","last_peered":"2023-10-09T06:42:52.623088+0000","last_clean":"2023-10-09T06:42:52.623088+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:42:52.623088+0000","last_undegraded":"2023-10-09T06:42:52.623088+0000","last_fullsized":"2023-10-09T06:42:52.623088+0000","mapping_epoch":489,"log_start":"502'5000","ondisk_log_start":"502'5000","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11148,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":344,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722676+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:42:45.722676+0000","last_peered":"2023-10-09T06:42:45.722676+0000","last_clean":"2023-10-09T06:42:45.722676+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:42:45.722676+0000","last_undegraded":"2023-10-09T06:42:45.722676+0000","last_fullsized":"2023-10-09T06:42:45.722676+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":858,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722371+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:42:45.722371+0000","last_peered":"2023-10-09T06:42:45.722371+0000","last_clean":"2023-10-09T06:42:45.722371+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:42:45.722371+0000","last_undegraded":"2023-10-09T06:42:45.722371+0000","last_fullsized":"2023-10-09T06:42:45.722371+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":247,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.709774+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:42:44.709774+0000","last_peered":"2023-10-09T06:42:44.709774+0000","last_clean":"2023-10-09T06:42:44.709774+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:42:44.709774+0000","last_undegraded":"2023-10-09T06:42:44.709774+0000","last_fullsized":"2023-10-09T06:42:44.709774+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":497,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693874+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:42:45.693874+0000","last_peered":"2023-10-09T06:42:45.693874+0000","last_clean":"2023-10-09T06:42:45.693874+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:42:45.693874+0000","last_undegraded":"2023-10-09T06:42:45.693874+0000","last_fullsized":"2023-10-09T06:42:45.693874+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"535'10971","reported_seq":11027,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.840849+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:42:49.840849+0000","last_peered":"2023-10-09T06:42:49.840849+0000","last_clean":"2023-10-09T06:42:49.840849+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:42:49.840849+0000","last_undegraded":"2023-10-09T06:42:49.840849+0000","last_fullsized":"2023-10-09T06:42:49.840849+0000","mapping_epoch":489,"log_start":"501'4900","ondisk_log_start":"501'4900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118161408,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10971,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":847,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701402+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:42:45.701402+0000","last_peered":"2023-10-09T06:42:45.701402+0000","last_clean":"2023-10-09T06:42:45.701402+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:42:45.701402+0000","last_undegraded":"2023-10-09T06:42:45.701402+0000","last_fullsized":"2023-10-09T06:42:45.701402+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":398,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722228+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:42:45.722228+0000","last_peered":"2023-10-09T06:42:45.722228+0000","last_clean":"2023-10-09T06:42:45.722228+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:42:45.722228+0000","last_undegraded":"2023-10-09T06:42:45.722228+0000","last_fullsized":"2023-10-09T06:42:45.722228+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690030+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:42:45.690030+0000","last_peered":"2023-10-09T06:42:45.690030+0000","last_clean":"2023-10-09T06:42:45.690030+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:42:45.690030+0000","last_undegraded":"2023-10-09T06:42:45.690030+0000","last_fullsized":"2023-10-09T06:42:45.690030+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":522,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.691401+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:42:45.691401+0000","last_peered":"2023-10-09T06:42:45.691401+0000","last_clean":"2023-10-09T06:42:45.691401+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:42:45.691401+0000","last_undegraded":"2023-10-09T06:42:45.691401+0000","last_fullsized":"2023-10-09T06:42:45.691401+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"534'7770","reported_seq":7895,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.098130+0000","last_change":"2023-10-09T06:40:48.261698+0000","last_active":"2023-10-09T06:42:49.098130+0000","last_peered":"2023-10-09T06:42:49.098130+0000","last_clean":"2023-10-09T06:42:49.098130+0000","last_became_active":"2023-10-09T06:40:43.090550+0000","last_became_peered":"2023-10-09T06:40:43.090550+0000","last_unstale":"2023-10-09T06:42:49.098130+0000","last_undegraded":"2023-10-09T06:42:49.098130+0000","last_fullsized":"2023-10-09T06:42:49.098130+0000","mapping_epoch":489,"log_start":"493'195","ondisk_log_start":"493'195","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4775","last_scrub_stamp":"2023-10-09T06:40:48.261467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:48.261467+0000","objects_scrubbed":2451,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:45.927807+0000","scrub_duration":0.48199046000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59572224,"num_objects":909,"num_object_clones":0,"num_object_copies":2727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3000,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":857,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693125+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:42:45.693125+0000","last_peered":"2023-10-09T06:42:45.693125+0000","last_clean":"2023-10-09T06:42:45.693125+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:42:45.693125+0000","last_undegraded":"2023-10-09T06:42:45.693125+0000","last_fullsized":"2023-10-09T06:42:45.693125+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":398,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722716+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:42:45.722716+0000","last_peered":"2023-10-09T06:42:45.722716+0000","last_clean":"2023-10-09T06:42:45.722716+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:42:45.722716+0000","last_undegraded":"2023-10-09T06:42:45.722716+0000","last_fullsized":"2023-10-09T06:42:45.722716+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":534,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701009+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:42:45.701009+0000","last_peered":"2023-10-09T06:42:45.701009+0000","last_clean":"2023-10-09T06:42:45.701009+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:42:45.701009+0000","last_undegraded":"2023-10-09T06:42:45.701009+0000","last_fullsized":"2023-10-09T06:42:45.701009+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":521,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690731+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:42:45.690731+0000","last_peered":"2023-10-09T06:42:45.690731+0000","last_clean":"2023-10-09T06:42:45.690731+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:42:45.690731+0000","last_undegraded":"2023-10-09T06:42:45.690731+0000","last_fullsized":"2023-10-09T06:42:45.690731+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"535'11117","reported_seq":11173,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:53.186200+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:42:53.186200+0000","last_peered":"2023-10-09T06:42:53.186200+0000","last_clean":"2023-10-09T06:42:53.186200+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:42:53.186200+0000","last_undegraded":"2023-10-09T06:42:53.186200+0000","last_fullsized":"2023-10-09T06:42:53.186200+0000","mapping_epoch":489,"log_start":"508'5700","ondisk_log_start":"508'5700","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103743488,"num_objects":1583,"num_object_clones":0,"num_object_copies":4749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11117,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":858,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693428+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:42:45.693428+0000","last_peered":"2023-10-09T06:42:45.693428+0000","last_clean":"2023-10-09T06:42:45.693428+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:42:45.693428+0000","last_undegraded":"2023-10-09T06:42:45.693428+0000","last_fullsized":"2023-10-09T06:42:45.693428+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":268,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.700139+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:42:45.700139+0000","last_peered":"2023-10-09T06:42:45.700139+0000","last_clean":"2023-10-09T06:42:45.700139+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:42:45.700139+0000","last_undegraded":"2023-10-09T06:42:45.700139+0000","last_fullsized":"2023-10-09T06:42:45.700139+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":501,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701568+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:42:45.701568+0000","last_peered":"2023-10-09T06:42:45.701568+0000","last_clean":"2023-10-09T06:42:45.701568+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:42:45.701568+0000","last_undegraded":"2023-10-09T06:42:45.701568+0000","last_fullsized":"2023-10-09T06:42:45.701568+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708687+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:42:44.708687+0000","last_peered":"2023-10-09T06:42:44.708687+0000","last_clean":"2023-10-09T06:42:44.708687+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:42:44.708687+0000","last_undegraded":"2023-10-09T06:42:44.708687+0000","last_fullsized":"2023-10-09T06:42:44.708687+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"535'10943","reported_seq":10999,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.855884+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:42:49.855884+0000","last_peered":"2023-10-09T06:42:49.855884+0000","last_clean":"2023-10-09T06:42:49.855884+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:42:49.855884+0000","last_undegraded":"2023-10-09T06:42:49.855884+0000","last_fullsized":"2023-10-09T06:42:49.855884+0000","mapping_epoch":489,"log_start":"501'4900","ondisk_log_start":"501'4900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6043,"log_dups_size":0,"ondisk_log_size":6043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114884608,"num_objects":1753,"num_object_clones":0,"num_object_copies":5259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10943,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":890,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.714191+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:42:44.714191+0000","last_peered":"2023-10-09T06:42:44.714191+0000","last_clean":"2023-10-09T06:42:44.714191+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:42:44.714191+0000","last_undegraded":"2023-10-09T06:42:44.714191+0000","last_fullsized":"2023-10-09T06:42:44.714191+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":519,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692984+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:42:45.692984+0000","last_peered":"2023-10-09T06:42:45.692984+0000","last_clean":"2023-10-09T06:42:45.692984+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:42:45.692984+0000","last_undegraded":"2023-10-09T06:42:45.692984+0000","last_fullsized":"2023-10-09T06:42:45.692984+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":503,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.691233+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:42:45.691233+0000","last_peered":"2023-10-09T06:42:45.691233+0000","last_clean":"2023-10-09T06:42:45.691233+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:42:45.691233+0000","last_undegraded":"2023-10-09T06:42:45.691233+0000","last_fullsized":"2023-10-09T06:42:45.691233+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":498,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693292+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:42:45.693292+0000","last_peered":"2023-10-09T06:42:45.693292+0000","last_clean":"2023-10-09T06:42:45.693292+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:42:45.693292+0000","last_undegraded":"2023-10-09T06:42:45.693292+0000","last_fullsized":"2023-10-09T06:42:45.693292+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"535'7647","reported_seq":7711,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.855261+0000","last_change":"2023-10-09T06:40:43.188245+0000","last_active":"2023-10-09T06:42:49.855261+0000","last_peered":"2023-10-09T06:42:49.855261+0000","last_clean":"2023-10-09T06:42:49.855261+0000","last_became_active":"2023-10-09T06:40:43.186495+0000","last_became_peered":"2023-10-09T06:40:43.186495+0000","last_unstale":"2023-10-09T06:42:49.855261+0000","last_undegraded":"2023-10-09T06:42:49.855261+0000","last_fullsized":"2023-10-09T06:42:49.855261+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7647,"log_dups_size":0,"ondisk_log_size":7647,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55115776,"num_objects":841,"num_object_clones":0,"num_object_copies":2523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5367,"num_write_kb":198656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":816,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.709300+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:42:44.709300+0000","last_peered":"2023-10-09T06:42:44.709300+0000","last_clean":"2023-10-09T06:42:44.709300+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:42:44.709300+0000","last_undegraded":"2023-10-09T06:42:44.709300+0000","last_fullsized":"2023-10-09T06:42:44.709300+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":504,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690649+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:42:45.690649+0000","last_peered":"2023-10-09T06:42:45.690649+0000","last_clean":"2023-10-09T06:42:45.690649+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:42:45.690649+0000","last_undegraded":"2023-10-09T06:42:45.690649+0000","last_fullsized":"2023-10-09T06:42:45.690649+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":526,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.700928+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:42:45.700928+0000","last_peered":"2023-10-09T06:42:45.700928+0000","last_clean":"2023-10-09T06:42:45.700928+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:42:45.700928+0000","last_undegraded":"2023-10-09T06:42:45.700928+0000","last_fullsized":"2023-10-09T06:42:45.700928+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"535'7705","reported_seq":15331,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:52.572632+0000","last_change":"2023-10-09T06:42:38.062683+0000","last_active":"2023-10-09T06:42:52.572632+0000","last_peered":"2023-10-09T06:42:52.572632+0000","last_clean":"2023-10-09T06:42:52.572632+0000","last_became_active":"2023-10-09T06:42:35.898459+0000","last_became_peered":"2023-10-09T06:42:35.898459+0000","last_unstale":"2023-10-09T06:42:52.572632+0000","last_undegraded":"2023-10-09T06:42:52.572632+0000","last_fullsized":"2023-10-09T06:42:52.572632+0000","mapping_epoch":525,"log_start":"493'260","ondisk_log_start":"493'260","created":500,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7448","last_scrub_stamp":"2023-10-09T06:42:38.062621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:38.062621+0000","objects_scrubbed":1070,"log_size":7445,"log_dups_size":0,"ondisk_log_size":7445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:34.705165+0000","scrub_duration":0.30123573999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":264,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":747,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708359+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:42:44.708359+0000","last_peered":"2023-10-09T06:42:44.708359+0000","last_clean":"2023-10-09T06:42:44.708359+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:42:44.708359+0000","last_undegraded":"2023-10-09T06:42:44.708359+0000","last_fullsized":"2023-10-09T06:42:44.708359+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":213,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693469+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:42:45.693469+0000","last_peered":"2023-10-09T06:42:45.693469+0000","last_clean":"2023-10-09T06:42:45.693469+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:42:45.693469+0000","last_undegraded":"2023-10-09T06:42:45.693469+0000","last_fullsized":"2023-10-09T06:42:45.693469+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":542,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692909+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:42:45.692909+0000","last_peered":"2023-10-09T06:42:45.692909+0000","last_clean":"2023-10-09T06:42:45.692909+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:42:45.692909+0000","last_undegraded":"2023-10-09T06:42:45.692909+0000","last_fullsized":"2023-10-09T06:42:45.692909+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"535'7805","reported_seq":18033,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:53.161447+0000","last_change":"2023-10-09T06:42:36.563651+0000","last_active":"2023-10-09T06:42:53.161447+0000","last_peered":"2023-10-09T06:42:53.161447+0000","last_clean":"2023-10-09T06:42:53.161447+0000","last_became_active":"2023-10-09T06:41:58.457422+0000","last_became_peered":"2023-10-09T06:41:58.457422+0000","last_unstale":"2023-10-09T06:42:53.161447+0000","last_undegraded":"2023-10-09T06:42:53.161447+0000","last_fullsized":"2023-10-09T06:42:53.161447+0000","mapping_epoch":519,"log_start":"493'358","ondisk_log_start":"493'358","created":500,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7520","last_scrub_stamp":"2023-10-09T06:42:36.563378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:36.563378+0000","objects_scrubbed":1096,"log_size":7447,"log_dups_size":0,"ondisk_log_size":7447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:24.101176+0000","scrub_duration":0.49466481600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":290,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":213,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693672+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:42:45.693672+0000","last_peered":"2023-10-09T06:42:45.693672+0000","last_clean":"2023-10-09T06:42:45.693672+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:42:45.693672+0000","last_undegraded":"2023-10-09T06:42:45.693672+0000","last_fullsized":"2023-10-09T06:42:45.693672+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":552,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692909+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:42:45.692909+0000","last_peered":"2023-10-09T06:42:45.692909+0000","last_clean":"2023-10-09T06:42:45.692909+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:42:45.692909+0000","last_undegraded":"2023-10-09T06:42:45.692909+0000","last_fullsized":"2023-10-09T06:42:45.692909+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"535'7803","reported_seq":8345,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:42:49.870646+0000","last_change":"2023-10-09T06:42:43.880894+0000","last_active":"2023-10-09T06:42:49.870646+0000","last_peered":"2023-10-09T06:42:49.870646+0000","last_clean":"2023-10-09T06:42:40.976884+0000","last_became_active":"2023-10-09T06:42:43.866227+0000","last_became_peered":"2023-10-09T06:42:43.866227+0000","last_unstale":"2023-10-09T06:42:49.870646+0000","last_undegraded":"2023-10-09T06:42:43.809722+0000","last_fullsized":"2023-10-09T06:42:43.686346+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7803,"log_dups_size":0,"ondisk_log_size":7803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57081856,"num_objects":871,"num_object_clones":0,"num_object_copies":2613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6138,"num_objects_misplaced":3069,"num_objects_unfound":0,"num_objects_dirty":871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5483,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":7733248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[0,2],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":880},{"shards":"0,2","objects":-9}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":891,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.729876+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:42:45.729876+0000","last_peered":"2023-10-09T06:42:45.729876+0000","last_clean":"2023-10-09T06:42:45.729876+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:42:45.729876+0000","last_undegraded":"2023-10-09T06:42:45.729876+0000","last_fullsized":"2023-10-09T06:42:45.729876+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":238,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.708533+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:42:44.708533+0000","last_peered":"2023-10-09T06:42:44.708533+0000","last_clean":"2023-10-09T06:42:44.708533+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:42:44.708533+0000","last_undegraded":"2023-10-09T06:42:44.708533+0000","last_fullsized":"2023-10-09T06:42:44.708533+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":255,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-09T06:42:44.709498+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:42:44.709498+0000","last_peered":"2023-10-09T06:42:44.709498+0000","last_clean":"2023-10-09T06:42:44.709498+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:42:44.709498+0000","last_undegraded":"2023-10-09T06:42:44.709498+0000","last_fullsized":"2023-10-09T06:42:44.709498+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"535'7874","reported_seq":9530,"reported_epoch":535,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:42:53.214218+0000","last_change":"2023-10-09T06:42:42.186055+0000","last_active":"2023-10-09T06:42:53.214218+0000","last_peered":"2023-10-09T06:42:53.214218+0000","last_clean":"2023-10-09T06:42:39.926512+0000","last_became_active":"2023-10-09T06:42:42.144898+0000","last_became_peered":"2023-10-09T06:42:42.144898+0000","last_unstale":"2023-10-09T06:42:53.214218+0000","last_undegraded":"2023-10-09T06:42:53.214218+0000","last_fullsized":"2023-10-09T06:42:41.983721+0000","mapping_epoch":529,"log_start":"493'403","ondisk_log_start":"493'403","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7471,"log_dups_size":0,"ondisk_log_size":7471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52592640,"num_objects":803,"num_object_clones":0,"num_object_copies":2409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":992,"num_objects_unfound":0,"num_objects_dirty":803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":203680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":524,"num_bytes_recovered":34340864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":307},{"shards":"1,3,6","objects":496}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":659,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.701748+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:42:45.701748+0000","last_peered":"2023-10-09T06:42:45.701748+0000","last_clean":"2023-10-09T06:42:45.701748+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:42:45.701748+0000","last_undegraded":"2023-10-09T06:42:45.701748+0000","last_fullsized":"2023-10-09T06:42:45.701748+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":534,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.690669+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:42:45.690669+0000","last_peered":"2023-10-09T06:42:45.690669+0000","last_clean":"2023-10-09T06:42:45.690669+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:42:45.690669+0000","last_undegraded":"2023-10-09T06:42:45.690669+0000","last_fullsized":"2023-10-09T06:42:45.690669+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":495,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692443+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:42:45.692443+0000","last_peered":"2023-10-09T06:42:45.692443+0000","last_clean":"2023-10-09T06:42:45.692443+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:42:45.692443+0000","last_undegraded":"2023-10-09T06:42:45.692443+0000","last_fullsized":"2023-10-09T06:42:45.692443+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":1758986264,"num_objects":26841,"num_object_clones":0,"num_object_copies":80523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9144,"num_objects_misplaced":4061,"num_objects_unfound":0,"num_objects_dirty":26841,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137059,"num_write_kb":4557441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":642,"num_bytes_recovered":42074112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5615071232,"data_stored":5615059016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":188638,"ondisk_log_size":188638,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234547,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1874520,"kb_used_data":373520,"kb_used_omap":0,"kb_used_meta":1500992,"kb_avail":91870632,"statfs":{"total":95995035648,"available":94075527168,"internally_reserved":0,"allocated":382484480,"data_stored":381405732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537015808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:18 2023","interfaces":[{"interface":"back","average":{"1min":4.5549999999999997,"5min":2.169,"15min":3.9420000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":55.783999999999999,"5min":55.783999999999999,"15min":215.97800000000001},"last":2.2930000000000001},{"interface":"front","average":{"1min":4.5439999999999996,"5min":2.129,"15min":3.996},"min":{"1min":0.245,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":55.771999999999998,"5min":55.771999999999998,"15min":215.577},"last":2.282}]},{"osd":1,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":12.045999999999999,"5min":4.665,"15min":5.8719999999999999},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.23200000000000001},"max":{"1min":169.51900000000001,"5min":169.51900000000001,"15min":215.61699999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":11.603999999999999,"5min":4.6269999999999998,"15min":4.867},"min":{"1min":0.28899999999999998,"5min":0.28100000000000003,"15min":0.23300000000000001},"max":{"1min":169.101,"5min":169.101,"15min":216.22200000000001},"last":1.0049999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:41:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":6.4640000000000004,"15min":5.8150000000000004},"min":{"1min":0.29699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":30.117000000000001,"5min":228.24000000000001,"15min":228.24000000000001},"last":1.8640000000000001},{"interface":"front","average":{"1min":2.5870000000000002,"5min":6.5140000000000002,"15min":5.7960000000000003},"min":{"1min":0.35499999999999998,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":30.245999999999999,"5min":228.291,"15min":228.291},"last":1.9399999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":4.4500000000000002,"15min":4.8659999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.8740000000000001,"5min":88.897999999999996,"15min":187.34200000000001},"last":1.8380000000000001},{"interface":"front","average":{"1min":1.952,"5min":4.4160000000000004,"15min":4.8529999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":6.4509999999999996,"5min":88.926000000000002,"15min":187.08199999999999},"last":0.90400000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.393000000000001,"5min":5.2069999999999999,"15min":2.9020000000000001},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":130.774,"5min":225.041,"15min":225.041},"last":0.29499999999999998},{"interface":"front","average":{"1min":11.032999999999999,"5min":5.0499999999999998,"15min":2.7450000000000001},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.72999999999999,"5min":225.244,"15min":225.244},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.8820000000000001,"15min":2.4060000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.60299999999999998,"5min":219.923,"15min":219.923},"last":1.22},{"interface":"front","average":{"1min":0.82999999999999996,"5min":2.9460000000000002,"15min":2.3610000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":219.86000000000001,"15min":219.86000000000001},"last":1.105}]},{"osd":6,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.494999999999999,"5min":5.1429999999999998,"15min":3.2050000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.317},"max":{"1min":138.477,"5min":222.179,"15min":222.179},"last":0.34599999999999997},{"interface":"front","average":{"1min":11.484999999999999,"5min":5.04,"15min":3.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":135.244,"5min":223.261,"15min":223.261},"last":0.38100000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496183,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1976516,"kb_used_data":480380,"kb_used_omap":0,"kb_used_meta":1496128,"kb_avail":91768636,"statfs":{"total":95995035648,"available":93971083264,"internally_reserved":0,"allocated":491909120,"data_stored":490849598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1532035072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8999999999999999,"15min":2.597},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":28.026,"5min":28.026,"15min":75.061999999999998},"last":4.8019999999999996},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.3979999999999997,"15min":3.6880000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":28.007000000000001,"5min":212.096,"15min":212.096},"last":4.6749999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":4.9169999999999998,"15min":5.0380000000000003},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.20899999999999999},"max":{"1min":3.8919999999999999,"5min":219.93199999999999,"15min":292.92200000000003},"last":4.7599999999999998},{"interface":"front","average":{"1min":1.548,"5min":5.0259999999999998,"15min":5.0919999999999996},"min":{"1min":0.39300000000000002,"5min":0.222,"15min":0.222},"max":{"1min":8.3840000000000003,"5min":219.94800000000001,"15min":292.904},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.8970000000000002,"15min":3.79},"min":{"1min":0.34899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":6.6959999999999997,"5min":185.654,"15min":185.654},"last":1.6279999999999999},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.8460000000000001,"15min":3.593},"min":{"1min":0.38900000000000001,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":6.7130000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":0.312}]},{"osd":3,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4859999999999998,"5min":5.2430000000000003,"15min":4.407},"min":{"1min":0.33200000000000002,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":80.168000000000006,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":5.4390000000000001,"5min":5.0700000000000003,"15min":4.3380000000000001},"min":{"1min":0.371,"5min":0.32100000000000001,"15min":0.318},"max":{"1min":80.242999999999995,"5min":161.56299999999999,"15min":161.56299999999999},"last":4.8360000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.198,"15min":1.53},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":5.6890000000000001,"5min":10.009,"15min":86.171999999999997},"last":1.585},{"interface":"front","average":{"1min":1.27,"5min":3.254,"15min":2.177},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.223},"max":{"1min":8.2989999999999995,"5min":212.03399999999999,"15min":212.03399999999999},"last":1.2470000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.3140000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.435,"5min":10.435,"15min":12.747999999999999},"last":1.899},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.4259999999999999,"15min":1.5329999999999999},"min":{"1min":0.312,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3369999999999997,"5min":9.9139999999999997,"15min":16.536999999999999},"last":0.436}]},{"osd":7,"last update":"Mon Oct 9 06:42:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.0329999999999999,"15min":1.7170000000000001},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8300000000000001,"5min":8.8059999999999992,"15min":165.37},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1779999999999999,"15min":2.056},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.3610000000000007,"5min":8.5980000000000008,"15min":165.40199999999999},"last":0.35099999999999998}]}]},{"osd":1,"up_from":110,"seq":472446402920,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002856,"kb_used_data":884320,"kb_used_omap":4,"kb_used_meta":1118459,"kb_avail":91742296,"statfs":{"total":95995035648,"available":93944111104,"internally_reserved":0,"allocated":905543680,"data_stored":903282040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1145302926},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.5410000000000004,"5min":3.9950000000000001,"15min":4.9109999999999996},"min":{"1min":0.21299999999999999,"5min":0.182,"15min":0.182},"max":{"1min":102.941,"5min":110.479,"15min":269.79199999999997},"last":54.895000000000003},{"interface":"front","average":{"1min":10.712999999999999,"5min":4.6639999999999997,"15min":5.2030000000000003},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.186},"max":{"1min":100.16800000000001,"5min":110.59999999999999,"15min":271.20999999999998},"last":58.039999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.3420000000000001,"15min":2.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.184},"max":{"1min":8.0129999999999999,"5min":8.0129999999999999,"15min":138.452},"last":0.246},{"interface":"front","average":{"1min":4.1040000000000001,"5min":2.5449999999999999,"15min":2.8530000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20599999999999999},"max":{"1min":17.555,"5min":17.555,"15min":138.47499999999999},"last":0.25900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:38 2023","interfaces":[{"interface":"back","average":{"1min":3.548,"5min":2.9089999999999998,"15min":2.8959999999999999},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":27.655999999999999,"5min":27.655999999999999,"15min":31.062000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.177,"5min":1.9930000000000001,"15min":2.0270000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":27.745999999999999,"5min":27.745999999999999,"15min":27.745999999999999},"last":1.5409999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.013999999999999,"5min":3.927,"15min":3.2400000000000002},"min":{"1min":0.316,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":180.59700000000001,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":9.3659999999999997,"5min":3.6040000000000001,"15min":2.919},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":180.70599999999999,"5min":180.70599999999999,"15min":180.70599999999999},"last":0.313}]},{"osd":5,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.6459999999999999,"15min":2.5630000000000002},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.253},"max":{"1min":7.9420000000000002,"5min":15.83,"15min":119.408},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.988,"15min":2.7850000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":7.5839999999999996,"5min":8.3819999999999997,"15min":119.483},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":9.7230000000000008,"5min":3.883,"15min":3.4409999999999998},"min":{"1min":0.33700000000000002,"5min":0.317,"15min":0.317},"max":{"1min":183.12100000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":10.446,"5min":4.3179999999999996,"15min":3.8639999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":183.13900000000001,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":10.129,"5min":4.5419999999999998,"15min":4.3570000000000002},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":178.74000000000001,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":8.9540000000000006,"5min":3.8029999999999999,"15min":3.7109999999999999},"min":{"1min":0.379,"5min":0.372,"15min":0.33100000000000002},"max":{"1min":178.75299999999999,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.46800000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024353,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2060236,"kb_used_data":891012,"kb_used_omap":5,"kb_used_meta":1169210,"kb_avail":91684916,"statfs":{"total":95995035648,"available":93885353984,"internally_reserved":0,"allocated":912396288,"data_stored":909748700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1197271674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:41:51 2023","interfaces":[{"interface":"back","average":{"1min":13.958,"5min":8.4000000000000004,"15min":4.8179999999999996},"min":{"1min":0.192,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":172.762,"5min":201.381,"15min":201.381},"last":1.3680000000000001},{"interface":"front","average":{"1min":13.670999999999999,"5min":8.4559999999999995,"15min":5.0940000000000003},"min":{"1min":0.27300000000000002,"5min":0.193,"15min":0.187},"max":{"1min":172.773,"5min":201.44200000000001,"15min":201.44200000000001},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:42:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":3.3239999999999998,"15min":3.5099999999999998},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":4.5289999999999999,"5min":97.650000000000006,"15min":100.425},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.698,"5min":3.7210000000000001,"15min":3.9260000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":21.096,"5min":96.082999999999998,"15min":99.769999999999996},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":3.5019999999999998,"15min":4.9450000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.484999999999999,"5min":95.555000000000007,"15min":208.11199999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.2799999999999998,"5min":3.6139999999999999,"15min":4.8719999999999999},"min":{"1min":0.33400000000000002,"5min":0.22900000000000001,"15min":0.19800000000000001},"max":{"1min":10.417,"5min":97.605000000000004,"15min":208.25899999999999},"last":0.497}]},{"osd":4,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.9610000000000001,"15min":3.1360000000000001},"min":{"1min":0.32400000000000001,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":7.9649999999999999,"5min":14.183,"15min":200.654},"last":1.4970000000000001},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.0960000000000001,"15min":3.4750000000000001},"min":{"1min":0.376,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":8.2850000000000001,"5min":11.738,"15min":200.86600000000001},"last":0.57399999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":2.1539999999999999,"15min":2.1619999999999999},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":8.0120000000000005,"5min":14.263,"15min":80.885999999999996},"last":1.4710000000000001},{"interface":"front","average":{"1min":1.712,"5min":1.7809999999999999,"15min":2.2469999999999999},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":8.4019999999999992,"15min":80.438999999999993},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9610000000000001,"15min":3.2469999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":120.572},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.0600000000000001,"15min":3.0129999999999999},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":8.0449999999999999,"5min":14.308999999999999,"15min":120.64400000000001},"last":1.4430000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.9379999999999999,"15min":2.5019999999999998},"min":{"1min":0.42199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.2490000000000006,"5min":16.911000000000001,"15min":122.018},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.931,"5min":2.3889999999999998,"15min":2.9049999999999998},"min":{"1min":0.36199999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.4570000000000007,"5min":9.9190000000000005,"15min":122.32299999999999},"last":0.629}]}]},{"osd":2,"up_from":128,"seq":549755814222,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818796,"kb_used_data":799844,"kb_used_omap":7,"kb_used_meta":1018936,"kb_avail":91926356,"statfs":{"total":95995035648,"available":94132588544,"internally_reserved":0,"allocated":819040256,"data_stored":816832543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1043390545},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":6.5650000000000004,"15min":6.5469999999999997},"min":{"1min":0.254,"5min":0.189,"15min":0.189},"max":{"1min":14.374000000000001,"5min":214.595,"15min":214.595},"last":89.287000000000006},{"interface":"front","average":{"1min":3.786,"5min":6.1200000000000001,"15min":6.226},"min":{"1min":0.221,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":17.053000000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":89.225999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.6360000000000001,"15min":4.3070000000000004},"min":{"1min":0.217,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.773999999999999,"5min":80.141000000000005,"15min":140.452},"last":137.749},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.665,"15min":4.5149999999999997},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":11.930999999999999,"5min":79.474000000000004,"15min":139.75299999999999},"last":138.24199999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":5.7050000000000001,"5min":3.5979999999999999,"15min":3.7909999999999999},"min":{"1min":0.30099999999999999,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":37.965000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":74.317999999999998},{"interface":"front","average":{"1min":5.5890000000000004,"5min":3.5419999999999998,"15min":4.0739999999999998},"min":{"1min":0.27800000000000002,"5min":0.27700000000000002,"15min":0.26700000000000002},"max":{"1min":37.990000000000002,"5min":79.614000000000004,"15min":91.685000000000002},"last":74.355999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":2.036,"15min":2.3069999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.5679999999999996,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2120000000000002,"15min":2.54},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":4.4539999999999997,"5min":82.872,"15min":82.872},"last":0.497}]},{"osd":5,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.994,"15min":2.3570000000000002},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7040000000000006,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1720000000000002,"15min":2.4950000000000001},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":4.4109999999999996,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3039999999999998,"15min":3.2010000000000001},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.312},"max":{"1min":4.4880000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.5209999999999999,"15min":3.1850000000000001},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":6.3689999999999998,"5min":80.364999999999995,"15min":122.517},"last":0.504}]},{"osd":7,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":8.9900000000000002,"5min":3.8100000000000001,"15min":3.298},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":132.06399999999999,"5min":132.06399999999999,"15min":132.06399999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":8.7509999999999994,"5min":3.4420000000000002,"15min":2.9460000000000002},"min":{"1min":0.35899999999999999,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":132.196,"5min":132.196,"15min":132.196},"last":0.45300000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127419,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827916,"kb_used_data":904836,"kb_used_omap":2,"kb_used_meta":923005,"kb_avail":91917236,"statfs":{"total":95995035648,"available":94123249664,"internally_reserved":0,"allocated":926552064,"data_stored":924484644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":945157173},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6050000000000004,"5min":4.0170000000000003,"15min":4.4989999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19},"max":{"1min":95.281999999999996,"5min":95.281999999999996,"15min":161.821},"last":4.9219999999999997},{"interface":"front","average":{"1min":9.0289999999999999,"5min":4.3010000000000002,"15min":4.4269999999999996},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":98.572000000000003,"5min":98.572000000000003,"15min":163.52000000000001},"last":12.006}]},{"osd":1,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.8460000000000001,"5min":4.0039999999999996,"15min":4.4359999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":102.468,"5min":102.468,"15min":184.012},"last":6.8860000000000001},{"interface":"front","average":{"1min":9.3949999999999996,"5min":4.399,"15min":5.2839999999999998},"min":{"1min":0.41399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":102.58,"5min":102.58,"15min":184.703},"last":4.9450000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":2.9449999999999998,"15min":3.9870000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.9900000000000002,"5min":118.93899999999999,"15min":182.44900000000001},"last":12.145},{"interface":"front","average":{"1min":1.6830000000000001,"5min":3.1070000000000002,"15min":4.3250000000000002},"min":{"1min":0.22,"5min":0.219,"15min":0.219},"max":{"1min":14.9,"5min":118.97199999999999,"15min":182.56399999999999},"last":8.7899999999999991}]},{"osd":4,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.0329999999999999,"15min":3.3879999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.7390000000000008,"5min":133.80099999999999,"15min":178.22200000000001},"last":4.8490000000000002},{"interface":"front","average":{"1min":1.444,"5min":3.2650000000000001,"15min":3.4060000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7999999999999998,"5min":133.648,"15min":180.43600000000001},"last":0.31900000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.181,"15min":3.6440000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34000000000000002},"max":{"1min":7.8380000000000001,"5min":128.083,"15min":177.56299999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.2170000000000001,"5min":3.0569999999999999,"15min":3.0920000000000001},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":8.0570000000000004,"5min":128.11799999999999,"15min":177.59399999999999},"last":2.4910000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.895,"15min":2.7410000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3710000000000004,"5min":10.505000000000001,"15min":130.39599999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6379999999999999,"15min":2.4279999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.5019999999999998,"5min":7.5949999999999998,"15min":130.67699999999999},"last":2.448}]},{"osd":7,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.4470000000000001,"15min":3.0619999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.6850000000000005,"5min":99.248000000000005,"15min":121.613},"last":4.8929999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.6299999999999999,"15min":3.4569999999999999},"min":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":3.2690000000000001,"5min":99.457999999999998,"15min":121.559},"last":0.38700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019456,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241996,"kb_used_data":758340,"kb_used_omap":0,"kb_used_meta":1483648,"kb_avail":91503156,"statfs":{"total":95995035648,"available":93699231744,"internally_reserved":0,"allocated":776540160,"data_stored":775472823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519255552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.823,"15min":6.4610000000000003},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":69.165000000000006,"5min":108.01300000000001,"15min":287.58800000000002},"last":11.52},{"interface":"front","average":{"1min":5.2089999999999996,"5min":3.1459999999999999,"15min":6.6760000000000002},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":69.147999999999996,"5min":107.96299999999999,"15min":287.57100000000003},"last":11.587}]},{"osd":1,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":5.7619999999999996,"5min":3.9820000000000002,"15min":4.1699999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":69.644000000000005,"5min":163.392,"15min":163.392},"last":0.41199999999999998},{"interface":"front","average":{"1min":6.1920000000000002,"5min":4.085,"15min":4.2439999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":69.572000000000003,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":5.1680000000000001,"15min":4.3579999999999997},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3180000000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.25,"5min":5.4630000000000001,"15min":4.3079999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.26400000000000001},"max":{"1min":3.3140000000000001,"5min":175.875,"15min":175.875},"last":0.32500000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":4.8929999999999998,"15min":6.1379999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":29.042999999999999,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.2509999999999999,"5min":4.7560000000000002,"15min":5.9329999999999998},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":29.192,"5min":112.21899999999999,"15min":161.934},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.006,"15min":0.96299999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":3.1070000000000002,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.198,"15min":1.087},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.7599999999999998,"5min":8.4600000000000009,"15min":9.5009999999999994},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":5.1189999999999998,"5min":1.788,"15min":2.2549999999999999},"min":{"1min":0.317,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":80.549000000000007,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":5.258,"5min":1.8049999999999999,"15min":2.4750000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":83.218000000000004,"5min":83.218000000000004,"15min":163.381},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":4.984,"5min":1.6930000000000001,"15min":1.9379999999999999},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":76.081000000000003,"5min":76.081000000000003,"15min":106.801},"last":0.48499999999999999},{"interface":"front","average":{"1min":4.8639999999999999,"5min":1.6879999999999999,"15min":1.9730000000000001},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":76.283000000000001,"5min":76.283000000000001,"15min":106.827},"last":0.504}]}]},{"osd":5,"up_from":38,"seq":163208757821,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2012484,"kb_used_data":528764,"kb_used_omap":0,"kb_used_meta":1483584,"kb_avail":91732668,"statfs":{"total":95995035648,"available":93934252032,"internally_reserved":0,"allocated":541454336,"data_stored":540391443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519190016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.4609999999999999,"15min":6.0819999999999999},"min":{"1min":0.23200000000000001,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":4.5460000000000003,"5min":78.171999999999997,"15min":294.89699999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.097,"5min":3.7440000000000002,"15min":6.2889999999999997},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":7.4630000000000001,"5min":78.230000000000004,"15min":294.30399999999997},"last":3.504}]},{"osd":1,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":10.257999999999999,"15min":7.2409999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.22},"max":{"1min":5.8120000000000003,"5min":153.91300000000001,"15min":188.34100000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.2349999999999999,"5min":10.214,"15min":7.2910000000000004},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1369999999999996,"5min":154.023,"15min":188.839},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":5.0970000000000004,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.49,"5min":100.46599999999999,"15min":120.613},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.2919999999999998,"15min":3.9540000000000002},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.4610000000000003,"5min":100.56999999999999,"15min":120.962},"last":0.51200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":5.8540000000000001,"15min":6.5890000000000004},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":5.6340000000000003,"5min":220.41800000000001,"15min":220.41800000000001},"last":6.8029999999999999},{"interface":"front","average":{"1min":2.4390000000000001,"5min":5.7930000000000001,"15min":6.4100000000000001},"min":{"1min":0.442,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.4210000000000003,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.60899999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.091,"15min":1.3799999999999999},"min":{"1min":0.28799999999999998,"5min":0.255,"15min":0.247},"max":{"1min":2.7320000000000002,"5min":9.9290000000000003,"15min":79.043000000000006},"last":0.26100000000000001},{"interface":"front","average":{"1min":1,"5min":1.298,"15min":1.4039999999999999},"min":{"1min":0.378,"5min":0.251,"15min":0.251},"max":{"1min":3.3180000000000001,"5min":8.3970000000000002,"15min":79.165999999999997},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":6.3520000000000003,"5min":2.0299999999999998,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.251},"max":{"1min":81.254000000000005,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.379},{"interface":"front","average":{"1min":6.0030000000000001,"5min":1.891,"15min":1.9159999999999999},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":79.216999999999999,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.42599999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1719999999999999,"15min":1.577},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":4.718,"5min":11.146000000000001,"15min":78.890000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.1180000000000001,"15min":1.4039999999999999},"min":{"1min":0.379,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":2.7069999999999999,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.48199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":889323520,"data_stored":889323520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":894304256,"data_stored":894304256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":797179904,"data_stored":797179904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":904593408,"data_stored":904593408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":752685056,"data_stored":752680984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":525930496,"data_stored":525926424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":475205632,"data_stored":475201560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":375848960,"data_stored":375848960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:42:57.310 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:42:57.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:57.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:42:57.419 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:57.451 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:57.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:57.616 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:57.650 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:42:57.712 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:57.807 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:42:57.808 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:42:57.808 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:42:57.808 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:42:57.808 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:42:57.860 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:57.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:57.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:57 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8915/74985 objects degraded (11.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:57.881 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:57.972 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:58.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:57 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8915/74985 objects degraded (11.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:58.043 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:58.045 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:57 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8915/74985 objects degraded (11.889%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:42:58.046 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:42:58.047 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:58.209 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:58.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:58.314 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:58.359 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:58.431 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:58.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:58.540 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:58.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:58.661 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:58.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:58.772 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:58.808 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:58.850 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:58.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:58.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:58.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:42:59.010 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:59.055 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:42:59.065 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:42:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:59.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:42:59.143 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:42:58 smithi107 ceph-mon[212750]: pgmap v1651: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:59.144 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:42:58 smithi107 ceph-mon[220921]: pgmap v1651: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:59.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:42:58 smithi150 ceph-mon[194178]: pgmap v1651: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 8915/74985 objects degraded (11.889%); 3947/74985 objects misplaced (5.264%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T06:42:59.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:42:59.211 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:59.269 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:42:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:59.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:42:59.381 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:42:59.423 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:59.489 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:42:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:59.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:42:59.596 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:42:59.640 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:42:59.654 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:42:59.701 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:42:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:42:59.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:42:59.810 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:42:59.923 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:00.042 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:00.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:00.149 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:00.208 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:00.262 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:43:00.627 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:00.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:00.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:00.739 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:00.841 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:00.898 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:00 smithi107 ceph-mon[220921]: pgmap v1652: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8915/68778 objects degraded (12.962%); 3555/68778 objects misplaced (5.169%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:43:00.906 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:00 smithi107 ceph-mon[212750]: pgmap v1652: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8915/68778 objects degraded (12.962%); 3555/68778 objects misplaced (5.169%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:43:00.918 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:00.947 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:00.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:01.052 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:01.053 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:01.126 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:01.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:00 smithi150 ceph-mon[194178]: pgmap v1652: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 8915/68778 objects degraded (12.962%); 3555/68778 objects misplaced (5.169%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T06:43:01.394 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:01.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:01.505 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:01.612 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:01.658 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:01.755 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:01.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:01.871 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:01.988 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:02.022 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:02] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:43:02.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:02.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:02.976 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:43:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:02 smithi150 ceph-mon[194178]: pgmap v1653: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8692/63828 objects degraded (13.618%); 3231/63828 objects misplaced (5.062%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:43:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1916140736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:02 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:02 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8692/63828 objects degraded (13.618%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:03.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[212750]: pgmap v1653: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8692/63828 objects degraded (13.618%); 3231/63828 objects misplaced (5.062%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:43:03.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1916140736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.409 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8692/63828 objects degraded (13.618%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:03.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[220921]: pgmap v1653: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 8692/63828 objects degraded (13.618%); 3231/63828 objects misplaced (5.062%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T06:43:03.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1916140736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:43:03.410 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:02 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8692/63828 objects degraded (13.618%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:03.773 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:03.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:03.885 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:03.924 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:03.995 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:04.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:04.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:04.102 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[212750]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:04.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:04.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:43:04.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.156 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:03 smithi107 ceph-mon[220921]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.172 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:04.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:04.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:43:04.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: osdmap e536: 8 total, 8 up, 8 in 2023-10-09T06:43:04.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: failed to encode map e536 with expected crc 2023-10-09T06:43:04.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:03 smithi150 ceph-mon[194178]: failed to encode map e536 with expected crc 2023-10-09T06:43:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:04 smithi107 ceph-mon[212750]: pgmap v1655: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8692/60192 objects degraded (14.440%); 3231/60192 objects misplaced (5.368%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:04 smithi107 ceph-mon[220921]: pgmap v1655: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8692/60192 objects degraded (14.440%); 3231/60192 objects misplaced (5.368%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:04 smithi150 ceph-mon[194178]: pgmap v1655: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8692/60192 objects degraded (14.440%); 3231/60192 objects misplaced (5.368%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T06:43:05.466 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:05.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:05.577 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:05.610 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:05.685 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:05.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:05.792 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:05.830 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:05.902 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:05.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:06.011 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:06.047 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:06.108 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:06.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:06.220 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:06.260 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:06.330 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:06.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:06.435 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:06.483 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:06.542 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:06.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:06.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:06.650 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:06.690 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:06.763 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:06.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:06.874 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:06.934 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:07.008 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:07.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:07.118 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:07.194 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:07.256 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:06 smithi107 ceph-mon[212750]: pgmap v1656: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 8457/54456 objects degraded (15.530%); 3114/54456 objects misplaced (5.718%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:43:07.256 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:06 smithi107 ceph-mon[220921]: pgmap v1656: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 8457/54456 objects degraded (15.530%); 3114/54456 objects misplaced (5.718%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:43:07.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:06 smithi150 ceph-mon[194178]: pgmap v1656: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 8457/54456 objects degraded (15.530%); 3114/54456 objects misplaced (5.718%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T06:43:07.471 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:07.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:07.583 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:07.621 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:07.660 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:43:07.687 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:07.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:07.699 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:43:07.734 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:43:07.777 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:43:07.789 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:43:07.789 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:43:07.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:07.845 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:07.846 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:43:07.897 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:43:07.903 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:07.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:07.940 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:43:07.977 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:43:08.008 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:08.017 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.017 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.017 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.018 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.018 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.018 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.018 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.018 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:43:08.093 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/637278211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:08.094 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:08 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8457/54456 objects degraded (15.530%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:08.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/637278211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:08.094 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:08 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8457/54456 objects degraded (15.530%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:08.104 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:08.186 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:43:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/637278211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:08.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:08 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8457/54456 objects degraded (15.530%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:08.558 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:43:08.579 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:43:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:09 smithi107 ceph-mon[212750]: pgmap v1657: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8457/53145 objects degraded (15.913%); 2726/53145 objects misplaced (5.129%); 4.5 MiB/s, 75 objects/s recovering 2023-10-09T06:43:09.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:09 smithi107 ceph-mon[220921]: pgmap v1657: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8457/53145 objects degraded (15.913%); 2726/53145 objects misplaced (5.129%); 4.5 MiB/s, 75 objects/s recovering 2023-10-09T06:43:09.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:09 smithi150 ceph-mon[194178]: pgmap v1657: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8457/53145 objects degraded (15.913%); 2726/53145 objects misplaced (5.129%); 4.5 MiB/s, 75 objects/s recovering 2023-10-09T06:43:09.898 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:09.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:10.007 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:10.027 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:10 smithi107 ceph-mon[212750]: from='client.57037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:10.053 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:10.125 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:10.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:10.231 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:10.260 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:10.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:10 smithi107 ceph-mon[220921]: from='client.57037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:10.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:10 smithi150 ceph-mon[194178]: from='client.57037 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:10.545 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:43:11.315 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:11.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:11.341 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:11 smithi107 ceph-mon[212750]: pgmap v1658: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 996 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 8457/47868 objects degraded (17.667%); 2726/47868 objects misplaced (5.695%); 3.3 MiB/s, 56 objects/s recovering 2023-10-09T06:43:11.341 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:11 smithi107 ceph-mon[220921]: pgmap v1658: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 996 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 8457/47868 objects degraded (17.667%); 2726/47868 objects misplaced (5.695%); 3.3 MiB/s, 56 objects/s recovering 2023-10-09T06:43:11.353 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1657,"stamp":"2023-10-09T06:43:07.652790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1159927736,"num_objects":17715,"num_object_clones":0,"num_object_copies":53145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8457,"num_objects_misplaced":2726,"num_objects_unfound":0,"num_objects_dirty":17715,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146270,"num_write_kb":4557470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1567,"num_bytes_recovered":97069004,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195769,"ondisk_log_size":195769,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14129968,"kb_used_data":3899120,"kb_used_omap":20,"kb_used_meta":10228395,"kb_avail":735831248,"statfs":{"total":767960285184,"available":753491197952,"internally_reserved":0,"allocated":3992698880,"data_stored":3979206655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10473877134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,7,8,3,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-477102080,"num_objects":-7280,"num_object_clones":0,"num_object_copies":-21840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-458,"num_objects_misplaced":-1221,"num_objects_unfound":0,"num_objects_dirty":-7280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7280,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":757,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008445"},"pg_stats":[{"pgid":"14.17","version":"535'7875","reported_seq":8056,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.667938+0000","last_change":"2023-10-09T06:40:52.372943+0000","last_active":"2023-10-09T06:43:02.667938+0000","last_peered":"2023-10-09T06:43:02.667938+0000","last_clean":"2023-10-09T06:43:02.667938+0000","last_became_active":"2023-10-09T06:40:43.185643+0000","last_became_peered":"2023-10-09T06:40:43.185643+0000","last_unstale":"2023-10-09T06:43:02.667938+0000","last_undegraded":"2023-10-09T06:43:02.667938+0000","last_fullsized":"2023-10-09T06:43:02.667938+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4783","last_scrub_stamp":"2023-10-09T06:40:52.372858+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:52.372858+0000","objects_scrubbed":2511,"log_size":7875,"log_dups_size":0,"ondisk_log_size":7875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:22.222358+0000","scrub_duration":0.27153366200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35061760,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3092,"num_write_kb":35712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":829,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722082+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:42:45.722082+0000","last_peered":"2023-10-09T06:42:45.722082+0000","last_clean":"2023-10-09T06:42:45.722082+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:42:45.722082+0000","last_undegraded":"2023-10-09T06:42:45.722082+0000","last_fullsized":"2023-10-09T06:42:45.722082+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":514,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692829+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:42:45.692829+0000","last_peered":"2023-10-09T06:42:45.692829+0000","last_clean":"2023-10-09T06:42:45.692829+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:42:45.692829+0000","last_undegraded":"2023-10-09T06:42:45.692829+0000","last_fullsized":"2023-10-09T06:42:45.692829+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":239,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756932+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:42:49.756932+0000","last_peered":"2023-10-09T06:42:49.756932+0000","last_clean":"2023-10-09T06:42:49.756932+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:42:49.756932+0000","last_undegraded":"2023-10-09T06:42:49.756932+0000","last_fullsized":"2023-10-09T06:42:49.756932+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"536'7774","reported_seq":7909,"reported_epoch":536,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:43:04.121491+0000","last_change":"2023-10-09T06:42:45.839310+0000","last_active":"2023-10-09T06:43:04.121491+0000","last_peered":"2023-10-09T06:43:04.121491+0000","last_clean":"2023-10-09T06:42:43.293915+0000","last_became_active":"2023-10-09T06:42:45.820286+0000","last_became_peered":"2023-10-09T06:42:45.820286+0000","last_unstale":"2023-10-09T06:43:04.121491+0000","last_undegraded":"2023-10-09T06:42:45.719211+0000","last_fullsized":"2023-10-09T06:42:45.692966+0000","mapping_epoch":533,"log_start":"493'245","ondisk_log_start":"493'245","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7291","last_scrub_stamp":"2023-10-09T06:42:37.253113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:37.253113+0000","objects_scrubbed":1009,"log_size":7529,"log_dups_size":0,"ondisk_log_size":7529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:53.601935+0000","scrub_duration":0.250396232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3006,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":485,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6","objects":-373},{"shards":"3,6,7","objects":524}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":790,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692610+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:42:45.692610+0000","last_peered":"2023-10-09T06:42:45.692610+0000","last_clean":"2023-10-09T06:42:45.692610+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:42:45.692610+0000","last_undegraded":"2023-10-09T06:42:45.692610+0000","last_fullsized":"2023-10-09T06:42:45.692610+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":533,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754948+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:42:49.754948+0000","last_peered":"2023-10-09T06:42:49.754948+0000","last_clean":"2023-10-09T06:42:49.754948+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:42:49.754948+0000","last_undegraded":"2023-10-09T06:42:49.754948+0000","last_fullsized":"2023-10-09T06:42:49.754948+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":538,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753952+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:42:49.753952+0000","last_peered":"2023-10-09T06:42:49.753952+0000","last_clean":"2023-10-09T06:42:49.753952+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:42:49.753952+0000","last_undegraded":"2023-10-09T06:42:49.753952+0000","last_fullsized":"2023-10-09T06:42:49.753952+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"536'7896","reported_seq":7962,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.808377+0000","last_change":"2023-10-09T06:40:43.168550+0000","last_active":"2023-10-09T06:43:04.808377+0000","last_peered":"2023-10-09T06:43:04.808377+0000","last_clean":"2023-10-09T06:43:04.808377+0000","last_became_active":"2023-10-09T06:40:43.166693+0000","last_became_peered":"2023-10-09T06:40:43.166693+0000","last_unstale":"2023-10-09T06:43:04.808377+0000","last_undegraded":"2023-10-09T06:43:04.808377+0000","last_fullsized":"2023-10-09T06:43:04.808377+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7896,"log_dups_size":0,"ondisk_log_size":7896,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5635,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":782,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692720+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:42:45.692720+0000","last_peered":"2023-10-09T06:42:45.692720+0000","last_clean":"2023-10-09T06:42:45.692720+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:42:45.692720+0000","last_undegraded":"2023-10-09T06:42:45.692720+0000","last_fullsized":"2023-10-09T06:42:45.692720+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":257,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756404+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:42:49.756404+0000","last_peered":"2023-10-09T06:42:49.756404+0000","last_clean":"2023-10-09T06:42:49.756404+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:42:49.756404+0000","last_undegraded":"2023-10-09T06:42:49.756404+0000","last_fullsized":"2023-10-09T06:42:49.756404+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":524,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692850+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:42:45.692850+0000","last_peered":"2023-10-09T06:42:45.692850+0000","last_clean":"2023-10-09T06:42:45.692850+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:42:45.692850+0000","last_undegraded":"2023-10-09T06:42:45.692850+0000","last_fullsized":"2023-10-09T06:42:45.692850+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":643,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722418+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:42:45.722418+0000","last_peered":"2023-10-09T06:42:45.722418+0000","last_clean":"2023-10-09T06:42:45.722418+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:42:45.722418+0000","last_undegraded":"2023-10-09T06:42:45.722418+0000","last_fullsized":"2023-10-09T06:42:45.722418+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":212,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754100+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:42:49.754100+0000","last_peered":"2023-10-09T06:42:49.754100+0000","last_clean":"2023-10-09T06:42:49.754100+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:42:49.754100+0000","last_undegraded":"2023-10-09T06:42:49.754100+0000","last_fullsized":"2023-10-09T06:42:49.754100+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":239,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756622+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:42:49.756622+0000","last_peered":"2023-10-09T06:42:49.756622+0000","last_clean":"2023-10-09T06:42:49.756622+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:42:49.756622+0000","last_undegraded":"2023-10-09T06:42:49.756622+0000","last_fullsized":"2023-10-09T06:42:49.756622+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":680,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.729874+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:42:45.729874+0000","last_peered":"2023-10-09T06:42:45.729874+0000","last_clean":"2023-10-09T06:42:45.729874+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:42:45.729874+0000","last_undegraded":"2023-10-09T06:42:45.729874+0000","last_fullsized":"2023-10-09T06:42:45.729874+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.857264+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:42:45.857264+0000","last_peered":"2023-10-09T06:42:45.857264+0000","last_clean":"2023-10-09T06:42:45.857264+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:42:45.857264+0000","last_undegraded":"2023-10-09T06:42:45.857264+0000","last_fullsized":"2023-10-09T06:42:45.857264+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":239,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758053+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:42:49.758053+0000","last_peered":"2023-10-09T06:42:49.758053+0000","last_clean":"2023-10-09T06:42:49.758053+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:42:49.758053+0000","last_undegraded":"2023-10-09T06:42:49.758053+0000","last_fullsized":"2023-10-09T06:42:49.758053+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"536'8331","reported_seq":8396,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.742191+0000","last_change":"2023-10-09T06:42:32.156372+0000","last_active":"2023-10-09T06:43:04.742191+0000","last_peered":"2023-10-09T06:43:04.742191+0000","last_clean":"2023-10-09T06:43:04.742191+0000","last_became_active":"2023-10-09T06:40:43.180666+0000","last_became_peered":"2023-10-09T06:40:43.180666+0000","last_unstale":"2023-10-09T06:43:04.742191+0000","last_undegraded":"2023-10-09T06:43:04.742191+0000","last_fullsized":"2023-10-09T06:43:04.742191+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8331,"log_dups_size":0,"ondisk_log_size":8331,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38305792,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":208800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":665,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753819+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:42:49.753819+0000","last_peered":"2023-10-09T06:42:49.753819+0000","last_clean":"2023-10-09T06:42:49.753819+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:42:49.753819+0000","last_undegraded":"2023-10-09T06:42:49.753819+0000","last_fullsized":"2023-10-09T06:42:49.753819+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":416,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722786+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:42:45.722786+0000","last_peered":"2023-10-09T06:42:45.722786+0000","last_clean":"2023-10-09T06:42:45.722786+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:42:45.722786+0000","last_undegraded":"2023-10-09T06:42:45.722786+0000","last_fullsized":"2023-10-09T06:42:45.722786+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":474,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.759226+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:42:49.759226+0000","last_peered":"2023-10-09T06:42:49.759226+0000","last_clean":"2023-10-09T06:42:49.759226+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:42:49.759226+0000","last_undegraded":"2023-10-09T06:42:49.759226+0000","last_fullsized":"2023-10-09T06:42:49.759226+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"536'8005","reported_seq":8133,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.079947+0000","last_change":"2023-10-09T06:40:50.305746+0000","last_active":"2023-10-09T06:43:04.079947+0000","last_peered":"2023-10-09T06:43:04.079947+0000","last_clean":"2023-10-09T06:43:04.079947+0000","last_became_active":"2023-10-09T06:40:43.093584+0000","last_became_peered":"2023-10-09T06:40:43.093584+0000","last_unstale":"2023-10-09T06:43:04.079947+0000","last_undegraded":"2023-10-09T06:43:04.079947+0000","last_fullsized":"2023-10-09T06:43:04.079947+0000","mapping_epoch":500,"log_start":"493'647","ondisk_log_start":"493'647","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4832","last_scrub_stamp":"2023-10-09T06:40:50.305626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:50.305626+0000","objects_scrubbed":2516,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:57.896131+0000","scrub_duration":0.56208410600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34471936,"num_objects":526,"num_object_clones":0,"num_object_copies":1578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3182,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":646,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758150+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:42:49.758150+0000","last_peered":"2023-10-09T06:42:49.758150+0000","last_clean":"2023-10-09T06:42:49.758150+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:42:49.758150+0000","last_undegraded":"2023-10-09T06:42:49.758150+0000","last_fullsized":"2023-10-09T06:42:49.758150+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":541,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693558+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:42:45.693558+0000","last_peered":"2023-10-09T06:42:45.693558+0000","last_clean":"2023-10-09T06:42:45.693558+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:42:45.693558+0000","last_undegraded":"2023-10-09T06:42:45.693558+0000","last_fullsized":"2023-10-09T06:42:45.693558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":505,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693079+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:42:45.693079+0000","last_peered":"2023-10-09T06:42:45.693079+0000","last_clean":"2023-10-09T06:42:45.693079+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:42:45.693079+0000","last_undegraded":"2023-10-09T06:42:45.693079+0000","last_fullsized":"2023-10-09T06:42:45.693079+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":810,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753071+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:42:49.753071+0000","last_peered":"2023-10-09T06:42:49.753071+0000","last_clean":"2023-10-09T06:42:49.753071+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:42:49.753071+0000","last_undegraded":"2023-10-09T06:42:49.753071+0000","last_fullsized":"2023-10-09T06:42:49.753071+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":384,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722567+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:42:45.722567+0000","last_peered":"2023-10-09T06:42:45.722567+0000","last_clean":"2023-10-09T06:42:45.722567+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:42:45.722567+0000","last_undegraded":"2023-10-09T06:42:45.722567+0000","last_fullsized":"2023-10-09T06:42:45.722567+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761844+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:42:49.761844+0000","last_peered":"2023-10-09T06:42:49.761844+0000","last_clean":"2023-10-09T06:42:49.761844+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:42:49.761844+0000","last_undegraded":"2023-10-09T06:42:49.761844+0000","last_fullsized":"2023-10-09T06:42:49.761844+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":807,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722080+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:42:45.722080+0000","last_peered":"2023-10-09T06:42:45.722080+0000","last_clean":"2023-10-09T06:42:45.722080+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:42:45.722080+0000","last_undegraded":"2023-10-09T06:42:45.722080+0000","last_fullsized":"2023-10-09T06:42:45.722080+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":535,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.752981+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:42:49.752981+0000","last_peered":"2023-10-09T06:42:49.752981+0000","last_clean":"2023-10-09T06:42:49.752981+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:42:49.752981+0000","last_undegraded":"2023-10-09T06:42:49.752981+0000","last_fullsized":"2023-10-09T06:42:49.752981+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":355,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722709+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:42:45.722709+0000","last_peered":"2023-10-09T06:42:45.722709+0000","last_clean":"2023-10-09T06:42:45.722709+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:42:45.722709+0000","last_undegraded":"2023-10-09T06:42:45.722709+0000","last_fullsized":"2023-10-09T06:42:45.722709+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":864,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758792+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:42:49.758792+0000","last_peered":"2023-10-09T06:42:49.758792+0000","last_clean":"2023-10-09T06:42:49.758792+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:42:49.758792+0000","last_undegraded":"2023-10-09T06:42:49.758792+0000","last_fullsized":"2023-10-09T06:42:49.758792+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":378,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722239+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:42:45.722239+0000","last_peered":"2023-10-09T06:42:45.722239+0000","last_clean":"2023-10-09T06:42:45.722239+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:42:45.722239+0000","last_undegraded":"2023-10-09T06:42:45.722239+0000","last_fullsized":"2023-10-09T06:42:45.722239+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":212,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754080+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:42:49.754080+0000","last_peered":"2023-10-09T06:42:49.754080+0000","last_clean":"2023-10-09T06:42:49.754080+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:42:49.754080+0000","last_undegraded":"2023-10-09T06:42:49.754080+0000","last_fullsized":"2023-10-09T06:42:49.754080+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":649,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.858749+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:42:45.858749+0000","last_peered":"2023-10-09T06:42:45.858749+0000","last_clean":"2023-10-09T06:42:45.858749+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:42:45.858749+0000","last_undegraded":"2023-10-09T06:42:45.858749+0000","last_fullsized":"2023-10-09T06:42:45.858749+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":613,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692383+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:42:45.692383+0000","last_peered":"2023-10-09T06:42:45.692383+0000","last_clean":"2023-10-09T06:42:45.692383+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:42:45.692383+0000","last_undegraded":"2023-10-09T06:42:45.692383+0000","last_fullsized":"2023-10-09T06:42:45.692383+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":239,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758369+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:42:49.758369+0000","last_peered":"2023-10-09T06:42:49.758369+0000","last_clean":"2023-10-09T06:42:49.758369+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:42:49.758369+0000","last_undegraded":"2023-10-09T06:42:49.758369+0000","last_fullsized":"2023-10-09T06:42:49.758369+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"535'7980","reported_seq":8122,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.819094+0000","last_change":"2023-10-09T06:42:35.324097+0000","last_active":"2023-10-09T06:43:02.819094+0000","last_peered":"2023-10-09T06:43:02.819094+0000","last_clean":"2023-10-09T06:43:02.819094+0000","last_became_active":"2023-10-09T06:40:43.094441+0000","last_became_peered":"2023-10-09T06:40:43.094441+0000","last_unstale":"2023-10-09T06:43:02.819094+0000","last_undegraded":"2023-10-09T06:43:02.819094+0000","last_fullsized":"2023-10-09T06:43:02.819094+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"525'7435","last_scrub_stamp":"2023-10-09T06:42:35.324027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:35.324027+0000","objects_scrubbed":1088,"log_size":7980,"log_dups_size":0,"ondisk_log_size":7980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:40.756290+0000","scrub_duration":0.32391870699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35389440,"num_objects":540,"num_object_clones":0,"num_object_copies":1620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":548,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":662,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722478+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:42:45.722478+0000","last_peered":"2023-10-09T06:42:45.722478+0000","last_clean":"2023-10-09T06:42:45.722478+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:42:45.722478+0000","last_undegraded":"2023-10-09T06:42:45.722478+0000","last_fullsized":"2023-10-09T06:42:45.722478+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":420,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722500+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:42:45.722500+0000","last_peered":"2023-10-09T06:42:45.722500+0000","last_clean":"2023-10-09T06:42:45.722500+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:42:45.722500+0000","last_undegraded":"2023-10-09T06:42:45.722500+0000","last_fullsized":"2023-10-09T06:42:45.722500+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":318,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722312+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:42:45.722312+0000","last_peered":"2023-10-09T06:42:45.722312+0000","last_clean":"2023-10-09T06:42:45.722312+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:42:45.722312+0000","last_undegraded":"2023-10-09T06:42:45.722312+0000","last_fullsized":"2023-10-09T06:42:45.722312+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"536'8130","reported_seq":8198,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.866338+0000","last_change":"2023-10-09T06:42:15.187041+0000","last_active":"2023-10-09T06:43:04.866338+0000","last_peered":"2023-10-09T06:43:04.866338+0000","last_clean":"2023-10-09T06:43:04.866338+0000","last_became_active":"2023-10-09T06:40:43.165023+0000","last_became_peered":"2023-10-09T06:40:43.165023+0000","last_unstale":"2023-10-09T06:43:04.866338+0000","last_undegraded":"2023-10-09T06:43:04.866338+0000","last_fullsized":"2023-10-09T06:43:04.866338+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35651584,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5810,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":923,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761635+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:42:49.761635+0000","last_peered":"2023-10-09T06:42:49.761635+0000","last_clean":"2023-10-09T06:42:49.761635+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:42:49.761635+0000","last_undegraded":"2023-10-09T06:42:49.761635+0000","last_fullsized":"2023-10-09T06:42:49.761635+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":382,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722627+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:42:45.722627+0000","last_peered":"2023-10-09T06:42:45.722627+0000","last_clean":"2023-10-09T06:42:45.722627+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:42:45.722627+0000","last_undegraded":"2023-10-09T06:42:45.722627+0000","last_fullsized":"2023-10-09T06:42:45.722627+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":532,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692788+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:42:45.692788+0000","last_peered":"2023-10-09T06:42:45.692788+0000","last_clean":"2023-10-09T06:42:45.692788+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:42:45.692788+0000","last_undegraded":"2023-10-09T06:42:45.692788+0000","last_fullsized":"2023-10-09T06:42:45.692788+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"536'7993","reported_seq":8061,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.890441+0000","last_change":"2023-10-09T06:42:26.019283+0000","last_active":"2023-10-09T06:43:04.890441+0000","last_peered":"2023-10-09T06:43:04.890441+0000","last_clean":"2023-10-09T06:43:04.890441+0000","last_became_active":"2023-10-09T06:40:43.080150+0000","last_became_peered":"2023-10-09T06:40:43.080150+0000","last_unstale":"2023-10-09T06:43:04.890441+0000","last_undegraded":"2023-10-09T06:43:04.890441+0000","last_fullsized":"2023-10-09T06:43:04.890441+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37650432,"num_objects":574,"num_object_clones":0,"num_object_copies":1722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":200864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":836,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692808+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:42:45.692808+0000","last_peered":"2023-10-09T06:42:45.692808+0000","last_clean":"2023-10-09T06:42:45.692808+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:42:45.692808+0000","last_undegraded":"2023-10-09T06:42:45.692808+0000","last_fullsized":"2023-10-09T06:42:45.692808+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":438,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722624+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:42:45.722624+0000","last_peered":"2023-10-09T06:42:45.722624+0000","last_clean":"2023-10-09T06:42:45.722624+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:42:45.722624+0000","last_undegraded":"2023-10-09T06:42:45.722624+0000","last_fullsized":"2023-10-09T06:42:45.722624+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":511,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693429+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:42:45.693429+0000","last_peered":"2023-10-09T06:42:45.693429+0000","last_clean":"2023-10-09T06:42:45.693429+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:42:45.693429+0000","last_undegraded":"2023-10-09T06:42:45.693429+0000","last_fullsized":"2023-10-09T06:42:45.693429+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"536'8042","reported_seq":8173,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.048581+0000","last_change":"2023-10-09T06:40:47.766598+0000","last_active":"2023-10-09T06:43:04.048581+0000","last_peered":"2023-10-09T06:43:04.048581+0000","last_clean":"2023-10-09T06:43:04.048581+0000","last_became_active":"2023-10-09T06:40:43.101954+0000","last_became_peered":"2023-10-09T06:40:43.101954+0000","last_unstale":"2023-10-09T06:43:04.048581+0000","last_undegraded":"2023-10-09T06:43:04.048581+0000","last_fullsized":"2023-10-09T06:43:04.048581+0000","mapping_epoch":489,"log_start":"493'843","ondisk_log_start":"493'843","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4698","last_scrub_stamp":"2023-10-09T06:40:47.766356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.766356+0000","objects_scrubbed":2441,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:35.858607+0000","scrub_duration":1.024576309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40042496,"num_objects":611,"num_object_clones":0,"num_object_copies":1833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3354,"num_write_kb":48768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":659,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693780+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:42:45.693780+0000","last_peered":"2023-10-09T06:42:45.693780+0000","last_clean":"2023-10-09T06:42:45.693780+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:42:45.693780+0000","last_undegraded":"2023-10-09T06:42:45.693780+0000","last_fullsized":"2023-10-09T06:42:45.693780+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":107,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761921+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:42:49.761921+0000","last_peered":"2023-10-09T06:42:49.761921+0000","last_clean":"2023-10-09T06:42:49.761921+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:42:49.761921+0000","last_undegraded":"2023-10-09T06:42:49.761921+0000","last_fullsized":"2023-10-09T06:42:49.761921+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":500,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754753+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:42:49.754753+0000","last_peered":"2023-10-09T06:42:49.754753+0000","last_clean":"2023-10-09T06:42:49.754753+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:42:49.754753+0000","last_undegraded":"2023-10-09T06:42:49.754753+0000","last_fullsized":"2023-10-09T06:42:49.754753+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"536'7971","reported_seq":8038,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.791281+0000","last_change":"2023-10-09T06:42:18.037795+0000","last_active":"2023-10-09T06:43:04.791281+0000","last_peered":"2023-10-09T06:43:04.791281+0000","last_clean":"2023-10-09T06:43:04.791281+0000","last_became_active":"2023-10-09T06:40:43.159937+0000","last_became_peered":"2023-10-09T06:40:43.159937+0000","last_unstale":"2023-10-09T06:43:04.791281+0000","last_undegraded":"2023-10-09T06:43:04.791281+0000","last_fullsized":"2023-10-09T06:43:04.791281+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7971,"log_dups_size":0,"ondisk_log_size":7971,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36503552,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5691,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":887,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722143+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:42:45.722143+0000","last_peered":"2023-10-09T06:42:45.722143+0000","last_clean":"2023-10-09T06:42:45.722143+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:42:45.722143+0000","last_undegraded":"2023-10-09T06:42:45.722143+0000","last_fullsized":"2023-10-09T06:42:45.722143+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":265,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.757148+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:42:49.757148+0000","last_peered":"2023-10-09T06:42:49.757148+0000","last_clean":"2023-10-09T06:42:49.757148+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:42:49.757148+0000","last_undegraded":"2023-10-09T06:42:49.757148+0000","last_fullsized":"2023-10-09T06:42:49.757148+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":515,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693380+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:42:45.693380+0000","last_peered":"2023-10-09T06:42:45.693380+0000","last_clean":"2023-10-09T06:42:45.693380+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:42:45.693380+0000","last_undegraded":"2023-10-09T06:42:45.693380+0000","last_fullsized":"2023-10-09T06:42:45.693380+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"535'8018","reported_seq":8188,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.639614+0000","last_change":"2023-10-09T06:40:45.736296+0000","last_active":"2023-10-09T06:43:02.639614+0000","last_peered":"2023-10-09T06:43:02.639614+0000","last_clean":"2023-10-09T06:43:02.639614+0000","last_became_active":"2023-10-09T06:40:43.168245+0000","last_became_peered":"2023-10-09T06:40:43.168245+0000","last_unstale":"2023-10-09T06:43:02.639614+0000","last_undegraded":"2023-10-09T06:43:02.639614+0000","last_fullsized":"2023-10-09T06:43:02.639614+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4658","last_scrub_stamp":"2023-10-09T06:40:45.736230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:45.736230+0000","objects_scrubbed":2357,"log_size":8018,"log_dups_size":0,"ondisk_log_size":8018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:31.145769+0000","scrub_duration":0.58060363000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3369,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":571,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756699+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:42:49.756699+0000","last_peered":"2023-10-09T06:42:49.756699+0000","last_clean":"2023-10-09T06:42:49.756699+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:42:49.756699+0000","last_undegraded":"2023-10-09T06:42:49.756699+0000","last_fullsized":"2023-10-09T06:42:49.756699+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":492,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693312+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:42:45.693312+0000","last_peered":"2023-10-09T06:42:45.693312+0000","last_clean":"2023-10-09T06:42:45.693312+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:42:45.693312+0000","last_undegraded":"2023-10-09T06:42:45.693312+0000","last_fullsized":"2023-10-09T06:42:45.693312+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":512,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693558+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:42:45.693558+0000","last_peered":"2023-10-09T06:42:45.693558+0000","last_clean":"2023-10-09T06:42:45.693558+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:42:45.693558+0000","last_undegraded":"2023-10-09T06:42:45.693558+0000","last_fullsized":"2023-10-09T06:42:45.693558+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"535'7887","reported_seq":8062,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.865297+0000","last_change":"2023-10-09T06:40:51.573557+0000","last_active":"2023-10-09T06:43:02.865297+0000","last_peered":"2023-10-09T06:43:02.865297+0000","last_clean":"2023-10-09T06:43:02.865297+0000","last_became_active":"2023-10-09T06:40:43.081959+0000","last_became_peered":"2023-10-09T06:40:43.081959+0000","last_unstale":"2023-10-09T06:43:02.865297+0000","last_undegraded":"2023-10-09T06:43:02.865297+0000","last_fullsized":"2023-10-09T06:43:02.865297+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'4739","last_scrub_stamp":"2023-10-09T06:40:51.573471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:51.573471+0000","objects_scrubbed":2521,"log_size":7887,"log_dups_size":0,"ondisk_log_size":7887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:30.924217+0000","scrub_duration":0.34142884899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38600704,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3148,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":874,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973655+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:43:02.973655+0000","last_peered":"2023-10-09T06:43:02.973655+0000","last_clean":"2023-10-09T06:43:02.973655+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:43:02.973655+0000","last_undegraded":"2023-10-09T06:43:02.973655+0000","last_fullsized":"2023-10-09T06:43:02.973655+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":516,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692666+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:42:45.692666+0000","last_peered":"2023-10-09T06:42:45.692666+0000","last_clean":"2023-10-09T06:42:45.692666+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:42:45.692666+0000","last_undegraded":"2023-10-09T06:42:45.692666+0000","last_fullsized":"2023-10-09T06:42:45.692666+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":495,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693919+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:42:45.693919+0000","last_peered":"2023-10-09T06:42:45.693919+0000","last_clean":"2023-10-09T06:42:45.693919+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:42:45.693919+0000","last_undegraded":"2023-10-09T06:42:45.693919+0000","last_fullsized":"2023-10-09T06:42:45.693919+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"536'7915","reported_seq":8042,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.789670+0000","last_change":"2023-10-09T06:40:47.192403+0000","last_active":"2023-10-09T06:43:04.789670+0000","last_peered":"2023-10-09T06:43:04.789670+0000","last_clean":"2023-10-09T06:43:04.789670+0000","last_became_active":"2023-10-09T06:40:43.074962+0000","last_became_peered":"2023-10-09T06:40:43.074962+0000","last_unstale":"2023-10-09T06:43:04.789670+0000","last_undegraded":"2023-10-09T06:43:04.789670+0000","last_fullsized":"2023-10-09T06:43:04.789670+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4664","last_scrub_stamp":"2023-10-09T06:40:47.192137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.192137+0000","objects_scrubbed":2418,"log_size":7915,"log_dups_size":0,"ondisk_log_size":7915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:24.365070+0000","scrub_duration":0.455376642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3254,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":497,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693231+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:42:45.693231+0000","last_peered":"2023-10-09T06:42:45.693231+0000","last_clean":"2023-10-09T06:42:45.693231+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:42:45.693231+0000","last_undegraded":"2023-10-09T06:42:45.693231+0000","last_fullsized":"2023-10-09T06:42:45.693231+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":240,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756553+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:42:49.756553+0000","last_peered":"2023-10-09T06:42:49.756553+0000","last_clean":"2023-10-09T06:42:49.756553+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:42:49.756553+0000","last_undegraded":"2023-10-09T06:42:49.756553+0000","last_fullsized":"2023-10-09T06:42:49.756553+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":493,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761402+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:42:49.761402+0000","last_peered":"2023-10-09T06:42:49.761402+0000","last_clean":"2023-10-09T06:42:49.761402+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:42:49.761402+0000","last_undegraded":"2023-10-09T06:42:49.761402+0000","last_fullsized":"2023-10-09T06:42:49.761402+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"536'8407","reported_seq":8473,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.862800+0000","last_change":"2023-10-09T06:42:25.028645+0000","last_active":"2023-10-09T06:43:04.862800+0000","last_peered":"2023-10-09T06:43:04.862800+0000","last_clean":"2023-10-09T06:43:04.862800+0000","last_became_active":"2023-10-09T06:40:43.080919+0000","last_became_peered":"2023-10-09T06:40:43.080919+0000","last_unstale":"2023-10-09T06:43:04.862800+0000","last_undegraded":"2023-10-09T06:43:04.862800+0000","last_fullsized":"2023-10-09T06:43:04.862800+0000","mapping_epoch":489,"log_start":"493'72","ondisk_log_start":"493'72","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8335,"log_dups_size":0,"ondisk_log_size":8335,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32931840,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6016,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":827,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692899+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:42:45.692899+0000","last_peered":"2023-10-09T06:42:45.692899+0000","last_clean":"2023-10-09T06:42:45.692899+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:42:45.692899+0000","last_undegraded":"2023-10-09T06:42:45.692899+0000","last_fullsized":"2023-10-09T06:42:45.692899+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":545,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758628+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:42:49.758628+0000","last_peered":"2023-10-09T06:42:49.758628+0000","last_clean":"2023-10-09T06:42:49.758628+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:42:49.758628+0000","last_undegraded":"2023-10-09T06:42:49.758628+0000","last_fullsized":"2023-10-09T06:42:49.758628+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756862+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:42:49.756862+0000","last_peered":"2023-10-09T06:42:49.756862+0000","last_clean":"2023-10-09T06:42:49.756862+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:42:49.756862+0000","last_undegraded":"2023-10-09T06:42:49.756862+0000","last_fullsized":"2023-10-09T06:42:49.756862+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":479,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758854+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:42:49.758854+0000","last_peered":"2023-10-09T06:42:49.758854+0000","last_clean":"2023-10-09T06:42:49.758854+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:42:49.758854+0000","last_undegraded":"2023-10-09T06:42:49.758854+0000","last_fullsized":"2023-10-09T06:42:49.758854+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"535'11377","reported_seq":11431,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.737459+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:43:02.737459+0000","last_peered":"2023-10-09T06:43:02.737459+0000","last_clean":"2023-10-09T06:43:02.737459+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:43:02.737459+0000","last_undegraded":"2023-10-09T06:43:02.737459+0000","last_fullsized":"2023-10-09T06:43:02.737459+0000","mapping_epoch":489,"log_start":"509'6032","ondisk_log_start":"509'6032","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5345,"log_dups_size":0,"ondisk_log_size":5345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76087296,"num_objects":1161,"num_object_clones":0,"num_object_copies":3483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11377,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":716,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753328+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:42:49.753328+0000","last_peered":"2023-10-09T06:42:49.753328+0000","last_clean":"2023-10-09T06:42:49.753328+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:42:49.753328+0000","last_undegraded":"2023-10-09T06:42:49.753328+0000","last_fullsized":"2023-10-09T06:42:49.753328+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":268,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758993+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:42:49.758993+0000","last_peered":"2023-10-09T06:42:49.758993+0000","last_clean":"2023-10-09T06:42:49.758993+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:42:49.758993+0000","last_undegraded":"2023-10-09T06:42:49.758993+0000","last_fullsized":"2023-10-09T06:42:49.758993+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":516,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692937+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:42:45.692937+0000","last_peered":"2023-10-09T06:42:45.692937+0000","last_clean":"2023-10-09T06:42:45.692937+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:42:45.692937+0000","last_undegraded":"2023-10-09T06:42:45.692937+0000","last_fullsized":"2023-10-09T06:42:45.692937+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":509,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758551+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:42:49.758551+0000","last_peered":"2023-10-09T06:42:49.758551+0000","last_clean":"2023-10-09T06:42:49.758551+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:42:49.758551+0000","last_undegraded":"2023-10-09T06:42:49.758551+0000","last_fullsized":"2023-10-09T06:42:49.758551+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"535'11307","reported_seq":11362,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.838283+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:43:02.838283+0000","last_peered":"2023-10-09T06:43:02.838283+0000","last_clean":"2023-10-09T06:43:02.838283+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:43:02.838283+0000","last_undegraded":"2023-10-09T06:43:02.838283+0000","last_fullsized":"2023-10-09T06:43:02.838283+0000","mapping_epoch":489,"log_start":"502'4900","ondisk_log_start":"502'4900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351768,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11307,"num_write_kb":397121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":877,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722286+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:42:45.722286+0000","last_peered":"2023-10-09T06:42:45.722286+0000","last_clean":"2023-10-09T06:42:45.722286+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:42:45.722286+0000","last_undegraded":"2023-10-09T06:42:45.722286+0000","last_fullsized":"2023-10-09T06:42:45.722286+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":530,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.757829+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:42:49.757829+0000","last_peered":"2023-10-09T06:42:49.757829+0000","last_clean":"2023-10-09T06:42:49.757829+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:42:49.757829+0000","last_undegraded":"2023-10-09T06:42:49.757829+0000","last_fullsized":"2023-10-09T06:42:49.757829+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973628+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:43:02.973628+0000","last_peered":"2023-10-09T06:43:02.973628+0000","last_clean":"2023-10-09T06:43:02.973628+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:43:02.973628+0000","last_undegraded":"2023-10-09T06:43:02.973628+0000","last_fullsized":"2023-10-09T06:43:02.973628+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":539,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761468+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:42:49.761468+0000","last_peered":"2023-10-09T06:42:49.761468+0000","last_clean":"2023-10-09T06:42:49.761468+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:42:49.761468+0000","last_undegraded":"2023-10-09T06:42:49.761468+0000","last_fullsized":"2023-10-09T06:42:49.761468+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"535'11597","reported_seq":11651,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.688012+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:43:02.688012+0000","last_peered":"2023-10-09T06:43:02.688012+0000","last_clean":"2023-10-09T06:43:02.688012+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:43:02.688012+0000","last_undegraded":"2023-10-09T06:43:02.688012+0000","last_fullsized":"2023-10-09T06:43:02.688012+0000","mapping_epoch":489,"log_start":"507'5400","ondisk_log_start":"507'5400","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11597,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":344,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722676+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:42:45.722676+0000","last_peered":"2023-10-09T06:42:45.722676+0000","last_clean":"2023-10-09T06:42:45.722676+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:42:45.722676+0000","last_undegraded":"2023-10-09T06:42:45.722676+0000","last_fullsized":"2023-10-09T06:42:45.722676+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":858,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722371+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:42:45.722371+0000","last_peered":"2023-10-09T06:42:45.722371+0000","last_clean":"2023-10-09T06:42:45.722371+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:42:45.722371+0000","last_undegraded":"2023-10-09T06:42:45.722371+0000","last_fullsized":"2023-10-09T06:42:45.722371+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":249,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756711+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:42:49.756711+0000","last_peered":"2023-10-09T06:42:49.756711+0000","last_clean":"2023-10-09T06:42:49.756711+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:42:49.756711+0000","last_undegraded":"2023-10-09T06:42:49.756711+0000","last_fullsized":"2023-10-09T06:42:49.756711+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":497,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693874+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:42:45.693874+0000","last_peered":"2023-10-09T06:42:45.693874+0000","last_clean":"2023-10-09T06:42:45.693874+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:42:45.693874+0000","last_undegraded":"2023-10-09T06:42:45.693874+0000","last_fullsized":"2023-10-09T06:42:45.693874+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"536'11686","reported_seq":11743,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.888478+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:43:04.888478+0000","last_peered":"2023-10-09T06:43:04.888478+0000","last_clean":"2023-10-09T06:43:04.888478+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:43:04.888478+0000","last_undegraded":"2023-10-09T06:43:04.888478+0000","last_fullsized":"2023-10-09T06:43:04.888478+0000","mapping_epoch":489,"log_start":"508'5600","ondisk_log_start":"508'5600","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71303168,"num_objects":1088,"num_object_clones":0,"num_object_copies":3264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11686,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":848,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758968+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:42:49.758968+0000","last_peered":"2023-10-09T06:42:49.758968+0000","last_clean":"2023-10-09T06:42:49.758968+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:42:49.758968+0000","last_undegraded":"2023-10-09T06:42:49.758968+0000","last_fullsized":"2023-10-09T06:42:49.758968+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":398,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722228+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:42:45.722228+0000","last_peered":"2023-10-09T06:42:45.722228+0000","last_clean":"2023-10-09T06:42:45.722228+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:42:45.722228+0000","last_undegraded":"2023-10-09T06:42:45.722228+0000","last_fullsized":"2023-10-09T06:42:45.722228+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":213,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.753255+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:42:49.753255+0000","last_peered":"2023-10-09T06:42:49.753255+0000","last_clean":"2023-10-09T06:42:49.753255+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:42:49.753255+0000","last_undegraded":"2023-10-09T06:42:49.753255+0000","last_fullsized":"2023-10-09T06:42:49.753255+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":524,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:54.900105+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:42:54.900105+0000","last_peered":"2023-10-09T06:42:54.900105+0000","last_clean":"2023-10-09T06:42:54.900105+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:42:54.900105+0000","last_undegraded":"2023-10-09T06:42:54.900105+0000","last_fullsized":"2023-10-09T06:42:54.900105+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"536'8075","reported_seq":8202,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.002393+0000","last_change":"2023-10-09T06:40:48.261698+0000","last_active":"2023-10-09T06:43:04.002393+0000","last_peered":"2023-10-09T06:43:04.002393+0000","last_clean":"2023-10-09T06:43:04.002393+0000","last_became_active":"2023-10-09T06:40:43.090550+0000","last_became_peered":"2023-10-09T06:40:43.090550+0000","last_unstale":"2023-10-09T06:43:04.002393+0000","last_undegraded":"2023-10-09T06:43:04.002393+0000","last_fullsized":"2023-10-09T06:43:04.002393+0000","mapping_epoch":489,"log_start":"493'841","ondisk_log_start":"493'841","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4775","last_scrub_stamp":"2023-10-09T06:40:48.261467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:48.261467+0000","objects_scrubbed":2451,"log_size":7234,"log_dups_size":0,"ondisk_log_size":7234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:45.927807+0000","scrub_duration":0.48199046000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3305,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":857,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693125+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:42:45.693125+0000","last_peered":"2023-10-09T06:42:45.693125+0000","last_clean":"2023-10-09T06:42:45.693125+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:42:45.693125+0000","last_undegraded":"2023-10-09T06:42:45.693125+0000","last_fullsized":"2023-10-09T06:42:45.693125+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":398,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.722716+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:42:45.722716+0000","last_peered":"2023-10-09T06:42:45.722716+0000","last_clean":"2023-10-09T06:42:45.722716+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:42:45.722716+0000","last_undegraded":"2023-10-09T06:42:45.722716+0000","last_fullsized":"2023-10-09T06:42:45.722716+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":535,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758470+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:42:49.758470+0000","last_peered":"2023-10-09T06:42:49.758470+0000","last_clean":"2023-10-09T06:42:49.758470+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:42:49.758470+0000","last_undegraded":"2023-10-09T06:42:49.758470+0000","last_fullsized":"2023-10-09T06:42:49.758470+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":522,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754878+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:42:49.754878+0000","last_peered":"2023-10-09T06:42:49.754878+0000","last_clean":"2023-10-09T06:42:49.754878+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:42:49.754878+0000","last_undegraded":"2023-10-09T06:42:49.754878+0000","last_fullsized":"2023-10-09T06:42:49.754878+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"536'11670","reported_seq":11727,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:05.760428+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:43:05.760428+0000","last_peered":"2023-10-09T06:43:05.760428+0000","last_clean":"2023-10-09T06:43:05.760428+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:43:05.760428+0000","last_undegraded":"2023-10-09T06:43:05.760428+0000","last_fullsized":"2023-10-09T06:43:05.760428+0000","mapping_epoch":489,"log_start":"509'6300","ondisk_log_start":"509'6300","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67502080,"num_objects":1030,"num_object_clones":0,"num_object_copies":3090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11670,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":858,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693428+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:42:45.693428+0000","last_peered":"2023-10-09T06:42:45.693428+0000","last_clean":"2023-10-09T06:42:45.693428+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:42:45.693428+0000","last_undegraded":"2023-10-09T06:42:45.693428+0000","last_fullsized":"2023-10-09T06:42:45.693428+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":270,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.763384+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:42:49.763384+0000","last_peered":"2023-10-09T06:42:49.763384+0000","last_clean":"2023-10-09T06:42:49.763384+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:42:49.763384+0000","last_undegraded":"2023-10-09T06:42:49.763384+0000","last_fullsized":"2023-10-09T06:42:49.763384+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":502,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758369+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:42:49.758369+0000","last_peered":"2023-10-09T06:42:49.758369+0000","last_clean":"2023-10-09T06:42:49.758369+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:42:49.758369+0000","last_undegraded":"2023-10-09T06:42:49.758369+0000","last_fullsized":"2023-10-09T06:42:49.758369+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":258,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756481+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:42:49.756481+0000","last_peered":"2023-10-09T06:42:49.756481+0000","last_clean":"2023-10-09T06:42:49.756481+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:42:49.756481+0000","last_undegraded":"2023-10-09T06:42:49.756481+0000","last_fullsized":"2023-10-09T06:42:49.756481+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"536'11615","reported_seq":11672,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.835977+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:43:04.835977+0000","last_peered":"2023-10-09T06:43:04.835977+0000","last_clean":"2023-10-09T06:43:04.835977+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:43:04.835977+0000","last_undegraded":"2023-10-09T06:43:04.835977+0000","last_fullsized":"2023-10-09T06:43:04.835977+0000","mapping_epoch":489,"log_start":"508'5600","ondisk_log_start":"508'5600","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11615,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":892,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.761997+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:42:49.761997+0000","last_peered":"2023-10-09T06:42:49.761997+0000","last_clean":"2023-10-09T06:42:49.761997+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:42:49.761997+0000","last_undegraded":"2023-10-09T06:42:49.761997+0000","last_fullsized":"2023-10-09T06:42:49.761997+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":519,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692984+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:42:45.692984+0000","last_peered":"2023-10-09T06:42:45.692984+0000","last_clean":"2023-10-09T06:42:45.692984+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:42:45.692984+0000","last_undegraded":"2023-10-09T06:42:45.692984+0000","last_fullsized":"2023-10-09T06:42:45.692984+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":505,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:54.900075+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:42:54.900075+0000","last_peered":"2023-10-09T06:42:54.900075+0000","last_clean":"2023-10-09T06:42:54.900075+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:42:54.900075+0000","last_undegraded":"2023-10-09T06:42:54.900075+0000","last_fullsized":"2023-10-09T06:42:54.900075+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":498,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693292+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:42:45.693292+0000","last_peered":"2023-10-09T06:42:45.693292+0000","last_clean":"2023-10-09T06:42:45.693292+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:42:45.693292+0000","last_undegraded":"2023-10-09T06:42:45.693292+0000","last_fullsized":"2023-10-09T06:42:45.693292+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"536'7961","reported_seq":8026,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:04.688731+0000","last_change":"2023-10-09T06:40:43.188245+0000","last_active":"2023-10-09T06:43:04.688731+0000","last_peered":"2023-10-09T06:43:04.688731+0000","last_clean":"2023-10-09T06:43:04.688731+0000","last_became_active":"2023-10-09T06:40:43.186495+0000","last_became_peered":"2023-10-09T06:40:43.186495+0000","last_unstale":"2023-10-09T06:43:04.688731+0000","last_undegraded":"2023-10-09T06:43:04.688731+0000","last_fullsized":"2023-10-09T06:43:04.688731+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34537472,"num_objects":527,"num_object_clones":0,"num_object_copies":1581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5681,"num_write_kb":198656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":818,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.757648+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:42:49.757648+0000","last_peered":"2023-10-09T06:42:49.757648+0000","last_clean":"2023-10-09T06:42:49.757648+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:42:49.757648+0000","last_undegraded":"2023-10-09T06:42:49.757648+0000","last_fullsized":"2023-10-09T06:42:49.757648+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":505,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754836+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:42:49.754836+0000","last_peered":"2023-10-09T06:42:49.754836+0000","last_clean":"2023-10-09T06:42:49.754836+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:42:49.754836+0000","last_undegraded":"2023-10-09T06:42:49.754836+0000","last_fullsized":"2023-10-09T06:42:49.754836+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":527,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.758359+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:42:49.758359+0000","last_peered":"2023-10-09T06:42:49.758359+0000","last_clean":"2023-10-09T06:42:49.758359+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:42:49.758359+0000","last_undegraded":"2023-10-09T06:42:49.758359+0000","last_fullsized":"2023-10-09T06:42:49.758359+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"535'7930","reported_seq":15556,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.723618+0000","last_change":"2023-10-09T06:42:38.062683+0000","last_active":"2023-10-09T06:43:02.723618+0000","last_peered":"2023-10-09T06:43:02.723618+0000","last_clean":"2023-10-09T06:43:02.723618+0000","last_became_active":"2023-10-09T06:42:35.898459+0000","last_became_peered":"2023-10-09T06:42:35.898459+0000","last_unstale":"2023-10-09T06:43:02.723618+0000","last_undegraded":"2023-10-09T06:43:02.723618+0000","last_fullsized":"2023-10-09T06:43:02.723618+0000","mapping_epoch":525,"log_start":"493'742","ondisk_log_start":"493'742","created":500,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7448","last_scrub_stamp":"2023-10-09T06:42:38.062621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:38.062621+0000","objects_scrubbed":1070,"log_size":7188,"log_dups_size":0,"ondisk_log_size":7188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:34.705165+0000","scrub_duration":0.30123573999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38076416,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":489,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":749,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756798+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:42:49.756798+0000","last_peered":"2023-10-09T06:42:49.756798+0000","last_clean":"2023-10-09T06:42:49.756798+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:42:49.756798+0000","last_undegraded":"2023-10-09T06:42:49.756798+0000","last_fullsized":"2023-10-09T06:42:49.756798+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":213,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693469+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:42:45.693469+0000","last_peered":"2023-10-09T06:42:45.693469+0000","last_clean":"2023-10-09T06:42:45.693469+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:42:45.693469+0000","last_undegraded":"2023-10-09T06:42:45.693469+0000","last_fullsized":"2023-10-09T06:42:45.693469+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":542,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692909+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:42:45.692909+0000","last_peered":"2023-10-09T06:42:45.692909+0000","last_clean":"2023-10-09T06:42:45.692909+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:42:45.692909+0000","last_undegraded":"2023-10-09T06:42:45.692909+0000","last_fullsized":"2023-10-09T06:42:45.692909+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"536'8080","reported_seq":18309,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:05.721453+0000","last_change":"2023-10-09T06:42:36.563651+0000","last_active":"2023-10-09T06:43:05.721453+0000","last_peered":"2023-10-09T06:43:05.721453+0000","last_clean":"2023-10-09T06:43:05.721453+0000","last_became_active":"2023-10-09T06:41:58.457422+0000","last_became_peered":"2023-10-09T06:41:58.457422+0000","last_unstale":"2023-10-09T06:43:05.721453+0000","last_undegraded":"2023-10-09T06:43:05.721453+0000","last_fullsized":"2023-10-09T06:43:05.721453+0000","mapping_epoch":519,"log_start":"493'894","ondisk_log_start":"493'894","created":500,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7520","last_scrub_stamp":"2023-10-09T06:42:36.563378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:36.563378+0000","objects_scrubbed":1096,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:24.101176+0000","scrub_duration":0.49466481600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34799616,"num_objects":531,"num_object_clones":0,"num_object_copies":1593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":565,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":213,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.693672+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:42:45.693672+0000","last_peered":"2023-10-09T06:42:45.693672+0000","last_clean":"2023-10-09T06:42:45.693672+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:42:45.693672+0000","last_undegraded":"2023-10-09T06:42:45.693672+0000","last_fullsized":"2023-10-09T06:42:45.693672+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":552,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692909+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:42:45.692909+0000","last_peered":"2023-10-09T06:42:45.692909+0000","last_clean":"2023-10-09T06:42:45.692909+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:42:45.692909+0000","last_undegraded":"2023-10-09T06:42:45.692909+0000","last_fullsized":"2023-10-09T06:42:45.692909+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"536'8146","reported_seq":10063,"reported_epoch":536,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:43:04.911418+0000","last_change":"2023-10-09T06:42:43.880894+0000","last_active":"2023-10-09T06:43:04.911418+0000","last_peered":"2023-10-09T06:43:04.911418+0000","last_clean":"2023-10-09T06:42:40.976884+0000","last_became_active":"2023-10-09T06:42:43.866227+0000","last_became_peered":"2023-10-09T06:42:43.866227+0000","last_unstale":"2023-10-09T06:43:04.911418+0000","last_undegraded":"2023-10-09T06:42:43.809722+0000","last_fullsized":"2023-10-09T06:42:43.686346+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8146,"log_dups_size":0,"ondisk_log_size":8146,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34603008,"num_objects":528,"num_object_clones":0,"num_object_copies":1584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5451,"num_objects_misplaced":2726,"num_objects_unfound":0,"num_objects_dirty":528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5826,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":30212096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[0,2],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":536},{"shards":"0,1,3,4","objects":1},{"shards":"0,2","objects":-9}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":891,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.729876+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:42:45.729876+0000","last_peered":"2023-10-09T06:42:45.729876+0000","last_clean":"2023-10-09T06:42:45.729876+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:42:45.729876+0000","last_undegraded":"2023-10-09T06:42:45.729876+0000","last_fullsized":"2023-10-09T06:42:45.729876+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":240,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.756969+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:42:49.756969+0000","last_peered":"2023-10-09T06:42:49.756969+0000","last_clean":"2023-10-09T06:42:49.756969+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:42:49.756969+0000","last_undegraded":"2023-10-09T06:42:49.756969+0000","last_fullsized":"2023-10-09T06:42:49.756969+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":257,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.757997+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:42:49.757997+0000","last_peered":"2023-10-09T06:42:49.757997+0000","last_clean":"2023-10-09T06:42:49.757997+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:42:49.757997+0000","last_undegraded":"2023-10-09T06:42:49.757997+0000","last_fullsized":"2023-10-09T06:42:49.757997+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"536'8175","reported_seq":11416,"reported_epoch":536,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T06:43:05.790423+0000","last_change":"2023-10-09T06:43:05.814497+0000","last_active":"2023-10-09T06:43:05.790423+0000","last_peered":"2023-10-09T06:43:05.790423+0000","last_clean":"2023-10-09T06:42:39.926512+0000","last_became_active":"2023-10-09T06:42:42.144898+0000","last_became_peered":"2023-10-09T06:42:42.144898+0000","last_unstale":"2023-10-09T06:43:05.790423+0000","last_undegraded":"2023-10-09T06:43:05.790133+0000","last_fullsized":"2023-10-09T06:42:41.983721+0000","mapping_epoch":529,"log_start":"493'1004","ondisk_log_start":"493'1004","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":7171,"log_dups_size":0,"ondisk_log_size":7171,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32866304,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5864,"num_write_kb":203680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1052,"num_bytes_recovered":66846720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":502}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":660,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.757950+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:42:49.757950+0000","last_peered":"2023-10-09T06:42:49.757950+0000","last_clean":"2023-10-09T06:42:49.757950+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:42:49.757950+0000","last_undegraded":"2023-10-09T06:42:49.757950+0000","last_fullsized":"2023-10-09T06:42:49.757950+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":535,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-09T06:42:49.754032+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:42:49.754032+0000","last_peered":"2023-10-09T06:42:49.754032+0000","last_clean":"2023-10-09T06:42:49.754032+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:42:49.754032+0000","last_undegraded":"2023-10-09T06:42:49.754032+0000","last_fullsized":"2023-10-09T06:42:49.754032+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":495,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-09T06:42:45.692443+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:42:45.692443+0000","last_peered":"2023-10-09T06:42:45.692443+0000","last_clean":"2023-10-09T06:42:45.692443+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:42:45.692443+0000","last_undegraded":"2023-10-09T06:42:45.692443+0000","last_fullsized":"2023-10-09T06:42:45.692443+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":1159331864,"num_objects":17691,"num_object_clones":0,"num_object_copies":53073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8457,"num_objects_misplaced":2726,"num_objects_unfound":0,"num_objects_dirty":17691,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146209,"num_write_kb":4557441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1513,"num_bytes_recovered":97058816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3849924608,"data_stored":3849912392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190723,"ondisk_log_size":190723,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234549,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1795032,"kb_used_data":289936,"kb_used_omap":0,"kb_used_meta":1505088,"kb_avail":91950120,"statfs":{"total":95995035648,"available":94156922880,"internally_reserved":0,"allocated":296894464,"data_stored":295815716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541210112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:18 2023","interfaces":[{"interface":"back","average":{"1min":4.5549999999999997,"5min":2.169,"15min":3.9420000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":55.783999999999999,"5min":55.783999999999999,"15min":215.97800000000001},"last":5.9210000000000003},{"interface":"front","average":{"1min":4.5439999999999996,"5min":2.129,"15min":3.996},"min":{"1min":0.245,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":55.771999999999998,"5min":55.771999999999998,"15min":215.577},"last":5.9340000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.8149999999999999,"15min":5.069},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23200000000000001},"max":{"1min":7.9080000000000004,"5min":169.51900000000001,"15min":208.626},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.1160000000000001,"5min":3.7410000000000001,"15min":4.1139999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":6.5730000000000004,"5min":169.101,"15min":181.24199999999999},"last":4.9450000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":5.5679999999999996,"15min":4.9640000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.8869999999999996,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":3.1070000000000002,"5min":5.8949999999999996,"15min":5.0579999999999998},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":15.355,"5min":228.291,"15min":228.291},"last":7.2859999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":4.4500000000000002,"15min":4.8659999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.8740000000000001,"5min":88.897999999999996,"15min":187.34200000000001},"last":7.3559999999999999},{"interface":"front","average":{"1min":1.952,"5min":4.4160000000000004,"15min":4.8529999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":6.4509999999999996,"5min":88.926000000000002,"15min":187.08199999999999},"last":2.6880000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.393000000000001,"5min":5.2069999999999999,"15min":2.9020000000000001},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":130.774,"5min":225.041,"15min":225.041},"last":1.179},{"interface":"front","average":{"1min":11.032999999999999,"5min":5.0499999999999998,"15min":2.7450000000000001},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.72999999999999,"5min":225.244,"15min":225.244},"last":2.6440000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.8820000000000001,"15min":2.4060000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.60299999999999998,"5min":219.923,"15min":219.923},"last":1.462},{"interface":"front","average":{"1min":0.82999999999999996,"5min":2.9460000000000002,"15min":2.3610000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":219.86000000000001,"15min":219.86000000000001},"last":1.2210000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.494999999999999,"5min":5.1429999999999998,"15min":3.2050000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.317},"max":{"1min":138.477,"5min":222.179,"15min":222.179},"last":2.9279999999999999},{"interface":"front","average":{"1min":11.484999999999999,"5min":5.04,"15min":3.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":135.244,"5min":223.261,"15min":223.261},"last":3.0009999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496185,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875652,"kb_used_data":371324,"kb_used_omap":0,"kb_used_meta":1504320,"kb_avail":91869500,"statfs":{"total":95995035648,"available":94074368000,"internally_reserved":0,"allocated":380235776,"data_stored":379176254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1540423680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8999999999999999,"15min":2.597},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":28.026,"5min":28.026,"15min":75.061999999999998},"last":2.2000000000000002},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.3979999999999997,"15min":3.6880000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":28.007000000000001,"5min":212.096,"15min":212.096},"last":2.214}]},{"osd":1,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":4.9169999999999998,"15min":5.0380000000000003},"min":{"1min":0.33000000000000002,"5min":0.27300000000000002,"15min":0.20899999999999999},"max":{"1min":3.8919999999999999,"5min":219.93199999999999,"15min":292.92200000000003},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.548,"5min":5.0259999999999998,"15min":5.0919999999999996},"min":{"1min":0.39300000000000002,"5min":0.222,"15min":0.222},"max":{"1min":8.3840000000000003,"5min":219.94800000000001,"15min":292.904},"last":0.374}]},{"osd":2,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.8970000000000002,"15min":3.79},"min":{"1min":0.34899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":6.6959999999999997,"5min":185.654,"15min":185.654},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.8460000000000001,"15min":3.593},"min":{"1min":0.38900000000000001,"5min":0.29699999999999999,"15min":0.27100000000000002},"max":{"1min":6.7130000000000001,"5min":185.67699999999999,"15min":185.67699999999999},"last":0.42699999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4859999999999998,"5min":5.2430000000000003,"15min":4.407},"min":{"1min":0.33200000000000002,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":80.168000000000006,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":5.4390000000000001,"5min":5.0700000000000003,"15min":4.3380000000000001},"min":{"1min":0.371,"5min":0.32100000000000001,"15min":0.318},"max":{"1min":80.242999999999995,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.70799999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.198,"15min":1.53},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":5.6890000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.27,"5min":3.254,"15min":2.177},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.223},"max":{"1min":8.2989999999999995,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.34200000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:42:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.3140000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.435,"5min":10.435,"15min":12.747999999999999},"last":0.442},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.4259999999999999,"15min":1.5329999999999999},"min":{"1min":0.312,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.3369999999999997,"5min":9.9139999999999997,"15min":16.536999999999999},"last":0.41999999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:42:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.0329999999999999,"15min":1.7170000000000001},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8300000000000001,"5min":8.8059999999999992,"15min":165.37},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1779999999999999,"15min":2.056},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.3610000000000007,"5min":8.5980000000000008,"15min":165.40199999999999},"last":0.45300000000000001}]}]},{"osd":1,"up_from":110,"seq":472446402923,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1712504,"kb_used_data":596016,"kb_used_omap":4,"kb_used_meta":1114363,"kb_avail":92032648,"statfs":{"total":95995035648,"available":94241431552,"internally_reserved":0,"allocated":610320384,"data_stored":608050938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1141108622},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.6719999999999997,"15min":5.4580000000000002},"min":{"1min":0.23200000000000001,"5min":0.182,"15min":0.182},"max":{"1min":114.97799999999999,"5min":114.97799999999999,"15min":269.79199999999997},"last":1.087},{"interface":"front","average":{"1min":10.832000000000001,"5min":6.5330000000000004,"15min":5.8339999999999996},"min":{"1min":0.21099999999999999,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":114.934,"5min":114.934,"15min":271.20999999999998},"last":3.589}]},{"osd":2,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.4139999999999999,"15min":2.2410000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.184},"max":{"1min":4.4699999999999998,"5min":8.0129999999999999,"15min":138.452},"last":0.34699999999999998},{"interface":"front","average":{"1min":4.016,"5min":2.8759999999999999,"15min":3.0659999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":20.085000000000001,"5min":20.085000000000001,"15min":138.47499999999999},"last":13.842000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:38 2023","interfaces":[{"interface":"back","average":{"1min":3.548,"5min":2.9089999999999998,"15min":2.8959999999999999},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":27.655999999999999,"5min":27.655999999999999,"15min":31.062000000000001},"last":9.8580000000000005},{"interface":"front","average":{"1min":3.177,"5min":1.9930000000000001,"15min":2.0270000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":27.745999999999999,"5min":27.745999999999999,"15min":27.745999999999999},"last":5.3879999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":3.7229999999999999,"15min":3.2930000000000001},"min":{"1min":0.378,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.9210000000000003,"5min":180.59700000000001,"15min":180.59700000000001},"last":3.7709999999999999},{"interface":"front","average":{"1min":1.643,"5min":3.552,"15min":2.9700000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":4.3159999999999998,"5min":180.70599999999999,"15min":180.70599999999999},"last":3.6400000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.589,"15min":2.5790000000000002},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.6160000000000001,"5min":15.83,"15min":119.408},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.8560000000000001,"15min":2.835},"min":{"1min":0.376,"5min":0.312,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":7.9470000000000001,"15min":119.483},"last":3.6659999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":8.7739999999999991,"5min":5.0730000000000004,"15min":3.9609999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":161.54900000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":3.6949999999999998},{"interface":"front","average":{"1min":8.8140000000000001,"5min":5.3399999999999999,"15min":4.3609999999999998},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":161.566,"5min":183.13900000000001,"15min":183.13900000000001},"last":3.798}]},{"osd":7,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":3.5670000000000002,"15min":4.0659999999999998},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":4.8890000000000002,"5min":178.74000000000001,"15min":178.74000000000001},"last":3.8239999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":2.9220000000000002,"15min":3.4119999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":5.3419999999999996,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.434}]}]},{"osd":0,"up_from":91,"seq":390842024356,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1706988,"kb_used_data":537764,"kb_used_omap":5,"kb_used_meta":1169210,"kb_avail":92038164,"statfs":{"total":95995035648,"available":94247079936,"internally_reserved":0,"allocated":550670336,"data_stored":548007136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1197271674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,1,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:42:51 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":8.0039999999999996,"15min":4.3869999999999996},"min":{"1min":0.22500000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":13.387,"5min":201.381,"15min":201.381},"last":4.6189999999999998},{"interface":"front","average":{"1min":2.1589999999999998,"5min":7.5019999999999998,"15min":4.609},"min":{"1min":0.24399999999999999,"5min":0.193,"15min":0.187},"max":{"1min":8.3219999999999992,"5min":201.44200000000001,"15min":201.44200000000001},"last":10.369999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":11.029,"5min":4.5430000000000001,"15min":4.0030000000000001},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":147.61099999999999,"5min":147.61099999999999,"15min":147.61099999999999},"last":10.398999999999999},{"interface":"front","average":{"1min":11.228999999999999,"5min":5.024,"15min":4.415},"min":{"1min":0.30199999999999999,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":150.12700000000001,"5min":150.12700000000001,"15min":150.12700000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":3.5019999999999998,"15min":4.9450000000000003},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":10.484999999999999,"5min":95.555000000000007,"15min":208.11199999999999},"last":7.444},{"interface":"front","average":{"1min":2.2799999999999998,"5min":3.6139999999999999,"15min":4.8719999999999999},"min":{"1min":0.33400000000000002,"5min":0.22900000000000001,"15min":0.19800000000000001},"max":{"1min":10.417,"5min":97.605000000000004,"15min":208.25899999999999},"last":7.6959999999999997}]},{"osd":4,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.9610000000000001,"15min":3.1360000000000001},"min":{"1min":0.32400000000000001,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":7.9649999999999999,"5min":14.183,"15min":200.654},"last":4.1029999999999998},{"interface":"front","average":{"1min":2.0049999999999999,"5min":2.0960000000000001,"15min":3.4750000000000001},"min":{"1min":0.376,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":8.2850000000000001,"5min":11.738,"15min":200.86600000000001},"last":7.8339999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":2.1539999999999999,"15min":2.1619999999999999},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":8.0120000000000005,"5min":14.263,"15min":80.885999999999996},"last":4.1870000000000003},{"interface":"front","average":{"1min":1.712,"5min":1.7809999999999999,"15min":2.2469999999999999},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.31900000000000001},"max":{"1min":8.4019999999999992,"5min":8.4019999999999992,"15min":80.438999999999993},"last":5.3730000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.9610000000000001,"15min":3.2469999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":120.572},"last":5.4349999999999996},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.0600000000000001,"15min":3.0129999999999999},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":8.0449999999999999,"5min":14.308999999999999,"15min":120.64400000000001},"last":4.1580000000000004}]},{"osd":7,"last update":"Mon Oct 9 06:42:23 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.9379999999999999,"15min":2.5019999999999998},"min":{"1min":0.42199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":8.2490000000000006,"5min":16.911000000000001,"15min":122.018},"last":7.742},{"interface":"front","average":{"1min":1.931,"5min":2.3889999999999998,"15min":2.9049999999999998},"min":{"1min":0.36199999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.4570000000000007,"5min":9.9190000000000005,"15min":122.32299999999999},"last":5.4119999999999999}]}]},{"osd":2,"up_from":128,"seq":549755814225,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1548796,"kb_used_data":511668,"kb_used_omap":7,"kb_used_meta":1036984,"kb_avail":92196356,"statfs":{"total":95995035648,"available":94409068544,"internally_reserved":0,"allocated":523948032,"data_stored":521732513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1061871697},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":4.2869999999999999,"5min":6.5650000000000004,"15min":6.5469999999999997},"min":{"1min":0.254,"5min":0.189,"15min":0.189},"max":{"1min":14.374000000000001,"5min":214.595,"15min":214.595},"last":2.3380000000000001},{"interface":"front","average":{"1min":3.786,"5min":6.1200000000000001,"15min":6.226},"min":{"1min":0.221,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":17.053000000000001,"5min":214.94300000000001,"15min":214.94300000000001},"last":2.3199999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.6360000000000001,"15min":4.3070000000000004},"min":{"1min":0.217,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":12.773999999999999,"5min":80.141000000000005,"15min":140.452},"last":3.2480000000000002},{"interface":"front","average":{"1min":3.1120000000000001,"5min":2.665,"15min":4.5149999999999997},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":11.930999999999999,"5min":79.474000000000004,"15min":139.75299999999999},"last":16.939}]},{"osd":3,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":5.7050000000000001,"5min":3.5979999999999999,"15min":3.7909999999999999},"min":{"1min":0.30099999999999999,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":37.965000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":5.5890000000000004,"5min":3.5419999999999998,"15min":4.0739999999999998},"min":{"1min":0.27800000000000002,"5min":0.27700000000000002,"15min":0.26700000000000002},"max":{"1min":37.990000000000002,"5min":79.614000000000004,"15min":91.685000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":2.036,"15min":2.3069999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.5679999999999996,"5min":79.643000000000001,"15min":79.643000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.2120000000000002,"15min":2.54},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":4.4539999999999997,"5min":82.872,"15min":82.872},"last":2.8100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.994,"15min":2.3570000000000002},"min":{"1min":0.33500000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7040000000000006,"5min":82.834999999999994,"15min":82.834999999999994},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.1539999999999999,"5min":2.1720000000000002,"15min":2.4950000000000001},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":4.4109999999999996,"5min":82.921999999999997,"15min":82.921999999999997},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.3039999999999998,"15min":3.2010000000000001},"min":{"1min":0.36099999999999999,"5min":0.313,"15min":0.312},"max":{"1min":4.4880000000000004,"5min":82.899000000000001,"15min":123.47799999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.5209999999999999,"15min":3.1850000000000001},"min":{"1min":0.39200000000000002,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":6.3689999999999998,"5min":80.364999999999995,"15min":122.517},"last":2.3039999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:42:11 2023","interfaces":[{"interface":"back","average":{"1min":8.9900000000000002,"5min":3.8100000000000001,"15min":3.298},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":132.06399999999999,"5min":132.06399999999999,"15min":132.06399999999999},"last":3.2930000000000001},{"interface":"front","average":{"1min":8.7509999999999994,"5min":3.4420000000000002,"15min":2.9460000000000002},"min":{"1min":0.35899999999999999,"5min":0.35499999999999998,"15min":0.34999999999999998},"max":{"1min":132.196,"5min":132.196,"15min":132.196},"last":0.50600000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127422,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1572196,"kb_used_data":653148,"kb_used_omap":2,"kb_used_meta":918909,"kb_avail":92172956,"statfs":{"total":95995035648,"available":94385106944,"internally_reserved":0,"allocated":668823552,"data_stored":666747992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":940962869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6050000000000004,"5min":4.0170000000000003,"15min":4.4989999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19},"max":{"1min":95.281999999999996,"5min":95.281999999999996,"15min":161.821},"last":1.173},{"interface":"front","average":{"1min":9.0289999999999999,"5min":4.3010000000000002,"15min":4.4269999999999996},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":98.572000000000003,"5min":98.572000000000003,"15min":163.52000000000001},"last":6.71}]},{"osd":1,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.8460000000000001,"5min":4.0039999999999996,"15min":4.4359999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":102.468,"5min":102.468,"15min":184.012},"last":6.1600000000000001},{"interface":"front","average":{"1min":9.3949999999999996,"5min":4.399,"15min":5.2839999999999998},"min":{"1min":0.41399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":102.58,"5min":102.58,"15min":184.703},"last":1.1539999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":2.9449999999999998,"15min":3.9870000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.9900000000000002,"5min":118.93899999999999,"15min":182.44900000000001},"last":7.8940000000000001},{"interface":"front","average":{"1min":1.6830000000000001,"5min":3.1070000000000002,"15min":4.3250000000000002},"min":{"1min":0.22,"5min":0.219,"15min":0.219},"max":{"1min":14.9,"5min":118.97199999999999,"15min":182.56399999999999},"last":1.1000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.0329999999999999,"15min":3.3879999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.7390000000000008,"5min":133.80099999999999,"15min":178.22200000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.444,"5min":3.2650000000000001,"15min":3.4060000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7999999999999998,"5min":133.648,"15min":180.43600000000001},"last":2.3660000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.181,"15min":3.6440000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34000000000000002},"max":{"1min":7.8380000000000001,"5min":128.083,"15min":177.56299999999999},"last":2.4089999999999998},{"interface":"front","average":{"1min":2.2170000000000001,"5min":3.0569999999999999,"15min":3.0920000000000001},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":8.0570000000000004,"5min":128.11799999999999,"15min":177.59399999999999},"last":3.044}]},{"osd":6,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.895,"15min":2.7410000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3710000000000004,"5min":10.505000000000001,"15min":130.39599999999999},"last":2.4649999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6379999999999999,"15min":2.4279999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.5019999999999998,"5min":7.5949999999999998,"15min":130.67699999999999},"last":3.2440000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.4470000000000001,"15min":3.0619999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.6850000000000005,"5min":99.248000000000005,"15min":121.613},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.6299999999999999,"15min":3.4569999999999999},"min":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":3.2690000000000001,"5min":99.457999999999998,"15min":121.559},"last":2.4369999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019458,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2042508,"kb_used_data":554756,"kb_used_omap":0,"kb_used_meta":1487744,"kb_avail":91702644,"statfs":{"total":95995035648,"available":93903507456,"internally_reserved":0,"allocated":568070144,"data_stored":567002807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1523449856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.823,"15min":6.4610000000000003},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":69.165000000000006,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.34100000000000003},{"interface":"front","average":{"1min":5.2089999999999996,"5min":3.1459999999999999,"15min":6.6760000000000002},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":69.147999999999996,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.36299999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":5.7619999999999996,"5min":3.9820000000000002,"15min":4.1699999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.22900000000000001},"max":{"1min":69.644000000000005,"5min":163.392,"15min":163.392},"last":0.377},{"interface":"front","average":{"1min":6.1920000000000002,"5min":4.085,"15min":4.2439999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":69.572000000000003,"5min":163.42400000000001,"15min":163.42400000000001},"last":4.3819999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":5.1680000000000001,"15min":4.3579999999999997},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.3180000000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.25,"5min":5.4630000000000001,"15min":4.3079999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.26400000000000001},"max":{"1min":3.3140000000000001,"5min":175.875,"15min":175.875},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":4.8929999999999998,"15min":6.1379999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":29.042999999999999,"5min":108.19799999999999,"15min":161.19300000000001},"last":7.8239999999999998},{"interface":"front","average":{"1min":3.2509999999999999,"5min":4.7560000000000002,"15min":5.9329999999999998},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":29.192,"5min":112.21899999999999,"15min":161.934},"last":7.3630000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:41:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.006,"15min":0.96299999999999997},"min":{"1min":0.28599999999999998,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":3.1070000000000002,"5min":9.3979999999999997,"15min":9.3979999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.198,"15min":1.087},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.7599999999999998,"5min":8.4600000000000009,"15min":9.5009999999999994},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:42:25 2023","interfaces":[{"interface":"back","average":{"1min":5.1189999999999998,"5min":1.788,"15min":2.2549999999999999},"min":{"1min":0.317,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":80.549000000000007,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":5.258,"5min":1.8049999999999999,"15min":2.4750000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":83.218000000000004,"5min":83.218000000000004,"15min":163.381},"last":0.45300000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":4.984,"5min":1.6930000000000001,"15min":1.9379999999999999},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":76.081000000000003,"5min":76.081000000000003,"15min":106.801},"last":0.42099999999999999},{"interface":"front","average":{"1min":4.8639999999999999,"5min":1.6879999999999999,"15min":1.9730000000000001},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":76.283000000000001,"5min":76.283000000000001,"15min":106.827},"last":0.54200000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757823,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1876292,"kb_used_data":384508,"kb_used_omap":0,"kb_used_meta":1491776,"kb_avail":91868860,"statfs":{"total":95995035648,"available":94073712640,"internally_reserved":0,"allocated":393736192,"data_stored":392673299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1527578624},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.4609999999999999,"15min":6.0819999999999999},"min":{"1min":0.23200000000000001,"5min":0.22900000000000001,"15min":0.20799999999999999},"max":{"1min":4.5460000000000003,"5min":78.171999999999997,"15min":294.89699999999999},"last":2.2170000000000001},{"interface":"front","average":{"1min":2.097,"5min":3.7440000000000002,"15min":6.2889999999999997},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":7.4630000000000001,"5min":78.230000000000004,"15min":294.30399999999997},"last":7.9660000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":10.257999999999999,"15min":7.2409999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.22},"max":{"1min":5.8120000000000003,"5min":153.91300000000001,"15min":188.34100000000001},"last":4.4669999999999996},{"interface":"front","average":{"1min":2.2349999999999999,"5min":10.214,"15min":7.2910000000000004},"min":{"1min":0.31,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1369999999999996,"5min":154.023,"15min":188.839},"last":0.66000000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":5.0970000000000004,"15min":4.1429999999999998},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.49,"5min":100.46599999999999,"15min":120.613},"last":2.8490000000000002},{"interface":"front","average":{"1min":2.2309999999999999,"5min":5.2919999999999998,"15min":3.9540000000000002},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.4610000000000003,"5min":100.56999999999999,"15min":120.962},"last":1.157}]},{"osd":3,"last update":"Mon Oct 9 06:42:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":5.8540000000000001,"15min":6.5890000000000004},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":5.6340000000000003,"5min":220.41800000000001,"15min":220.41800000000001},"last":2.085},{"interface":"front","average":{"1min":2.4390000000000001,"5min":5.7930000000000001,"15min":6.4100000000000001},"min":{"1min":0.442,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.4210000000000003,"5min":220.45500000000001,"15min":220.45500000000001},"last":1.9550000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:42:56 2023","interfaces":[{"interface":"back","average":{"1min":5.2729999999999997,"5min":1.7669999999999999,"15min":1.696},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.247},"max":{"1min":74.725999999999999,"5min":74.725999999999999,"15min":79.043000000000006},"last":2.0329999999999999},{"interface":"front","average":{"1min":5.2530000000000001,"5min":1.968,"15min":1.72},"min":{"1min":0.318,"5min":0.251,"15min":0.251},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":79.165999999999997},"last":1.486}]},{"osd":6,"last update":"Mon Oct 9 06:42:16 2023","interfaces":[{"interface":"back","average":{"1min":6.3520000000000003,"5min":2.0299999999999998,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.251},"max":{"1min":81.254000000000005,"5min":81.254000000000005,"15min":81.254000000000005},"last":1.6439999999999999},{"interface":"front","average":{"1min":6.0030000000000001,"5min":1.891,"15min":1.9159999999999999},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":79.216999999999999,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Oct 9 06:42:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1719999999999999,"15min":1.577},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":4.718,"5min":11.146000000000001,"15min":78.890000000000001},"last":2.113},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.1180000000000001,"15min":1.4039999999999999},"min":{"1min":0.379,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":2.7069999999999999,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.623}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":528941056,"data_stored":528941056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":592838656,"data_stored":592838656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":493748224,"data_stored":493748224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":640679936,"data_stored":640679936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":561844224,"data_stored":561840152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":379195392,"data_stored":379191320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":362614784,"data_stored":362610712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":290062336,"data_stored":290062336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:43:11.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T06:43:11.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:11 smithi150 ceph-mon[194178]: pgmap v1658: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 996 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 8457/47868 objects degraded (17.667%); 2726/47868 objects misplaced (5.695%); 3.3 MiB/s, 56 objects/s recovering 2023-10-09T06:43:11.436 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:11.471 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:11.533 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:11.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:11.624 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:11.638 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:11.639 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:11.754 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:11.941 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:11.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:11.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:12.052 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:12.142 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:12.256 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:12] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:43:12.480 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:12.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:12.589 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:12.627 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:12.682 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:12.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:12.791 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:12.832 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:12.887 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:12.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:12.995 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:13.076 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:13 smithi107 ceph-mon[220921]: pgmap v1659: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 878 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8222/42171 objects degraded (19.497%); 2609/42171 objects misplaced (6.187%); 2.7 MiB/s, 72 objects/s recovering 2023-10-09T06:43:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:13 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 8222/42171 objects degraded (19.497%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:13 smithi150 ceph-mon[194178]: pgmap v1659: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 878 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8222/42171 objects degraded (19.497%); 2609/42171 objects misplaced (6.187%); 2.7 MiB/s, 72 objects/s recovering 2023-10-09T06:43:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:13 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 8222/42171 objects degraded (19.497%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:13 smithi107 ceph-mon[212750]: pgmap v1659: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 878 MiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 8222/42171 objects degraded (19.497%); 2609/42171 objects misplaced (6.187%); 2.7 MiB/s, 72 objects/s recovering 2023-10-09T06:43:13.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:13 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 8222/42171 objects degraded (19.497%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:13.394 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:43:14.052 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:14.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:14.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:14.164 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:14.236 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:14.841 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:14.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:14.900 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:14.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:14.982 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:15.039 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:15.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:15.148 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:15.178 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:15.235 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:15.285 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:15 smithi107 ceph-mon[220921]: pgmap v1660: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 803 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8222/38568 objects degraded (21.318%); 2609/38568 objects misplaced (6.765%); 2.5 MiB/s, 67 objects/s recovering 2023-10-09T06:43:15.285 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:15 smithi150 ceph-mon[194178]: pgmap v1660: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 803 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8222/38568 objects degraded (21.318%); 2609/38568 objects misplaced (6.765%); 2.5 MiB/s, 67 objects/s recovering 2023-10-09T06:43:15.285 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:15 smithi107 ceph-mon[212750]: pgmap v1660: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 803 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 8222/38568 objects degraded (21.318%); 2609/38568 objects misplaced (6.765%); 2.5 MiB/s, 67 objects/s recovering 2023-10-09T06:43:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:15.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:15.387 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:15.447 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:15.452 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:16.089 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:43:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:16 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1376006441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:16.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:16 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:16.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:16 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1376006441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:16.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:16 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:16 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1376006441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:16.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:16 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:17.043 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:17.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:17.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: pgmap v1661: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 685 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7984/32925 objects degraded (24.249%); 2489/32925 objects misplaced (7.560%); 2.9 MiB/s, 70 objects/s recovering 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:43:17.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: pgmap v1661: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 685 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7984/32925 objects degraded (24.249%); 2489/32925 objects misplaced (7.560%); 2.9 MiB/s, 70 objects/s recovering 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:43:17.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:17.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:17.155 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:17.233 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: pgmap v1661: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 685 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7984/32925 objects degraded (24.249%); 2489/32925 objects misplaced (7.560%); 2.9 MiB/s, 70 objects/s recovering 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: osdmap e537: 8 total, 8 up, 8 in 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: failed to encode map e537 with expected crc 2023-10-09T06:43:17.407 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:17.408 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:17.775 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:43:17.775 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:43:18.115 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:18.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:18.221 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:18.329 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:18.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:18 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7984/32925 objects degraded (24.249%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:18.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:18 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7984/32925 objects degraded (24.249%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:18.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:18 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7984/32925 objects degraded (24.249%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:19.375 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:19 smithi107 ceph-mon[212750]: pgmap v1663: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 656 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7984/31515 objects degraded (25.334%); 2489/31515 objects misplaced (7.898%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:19.376 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:19 smithi107 ceph-mon[220921]: pgmap v1663: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 656 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7984/31515 objects degraded (25.334%); 2489/31515 objects misplaced (7.898%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:19.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:19 smithi150 ceph-mon[194178]: pgmap v1663: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 656 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7984/31515 objects degraded (25.334%); 2489/31515 objects misplaced (7.898%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:20.074 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:20.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:20.190 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.227 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:20.301 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:20.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:20.409 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:20.462 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:20.519 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:20.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:20.538 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:43:20.627 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:20.680 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:20.730 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:20.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:20.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:20.838 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:20.933 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:21.092 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:21.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:21.204 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:21.242 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:21.299 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:21.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:21.371 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:21.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:21 smithi107 ceph-mon[212750]: pgmap v1664: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 559 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7984/26868 objects degraded (29.716%); 2489/26868 objects misplaced (9.264%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:21.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:21 smithi107 ceph-mon[220921]: pgmap v1664: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 559 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7984/26868 objects degraded (29.716%); 2489/26868 objects misplaced (9.264%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:21.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:21 smithi150 ceph-mon[194178]: pgmap v1664: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 559 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7984/26868 objects degraded (29.716%); 2489/26868 objects misplaced (9.264%); 1.5 MiB/s, 76 objects/s recovering 2023-10-09T06:43:21.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:21.407 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:43:21.436 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:43:21.446 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:21.482 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:43:21.488 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:43:21.488 INFO:teuthology.orchestra.run.smithi107.stdout:{"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7984/26868 objects degraded (29.716%), 3 pgs degraded","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2214,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":537,"num_osds":8,"num_up_osds":8,"osd_up_since":1696832203,"num_in_osds":8,"osd_in_since":1696832954,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":126},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":8956,"data_bytes":585897912,"bytes_used":12628885504,"bytes_avail":755331399680,"bytes_total":767960285184,"degraded_objects":7984,"degraded_total":26868,"degraded_ratio":0.29715646866160489,"misplaced_objects":2489,"misplaced_total":26868,"misplaced_ratio":0.092638082477296407,"recovering_objects_per_sec":76,"recovering_bytes_per_sec":1552415,"recovering_keys_per_sec":0,"num_objects_recovered":764,"num_bytes_recovered":15532032,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":699},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi107.dqtsqc","status":"up:active","gid":24560}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.107:8443/","prometheus":"http://172.21.15.107:9283/"}},"servicemap":{"epoch":29,"modified":"2023-10-09T06:39:15.531037+0000","services":{}},"progress_events":{"107f254f-0084-430c-93f7-0df701c1cf7f":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"e42c139a-be3e-4b01-b8d9-083be03fc529":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T06:43:21.502 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:21.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:21.524 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:43:21.566 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:43:21.608 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:21.620 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:43:21.685 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:43:21.725 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:43:21.766 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:21.768 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:43:21.805 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:43:21.806 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:43:22.124 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:22] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-10-09T06:43:22.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1397188335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:22 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1919844016' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:43:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1397188335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:22 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1919844016' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:43:22.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1397188335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:22.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:22 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1919844016' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T06:43:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:23 smithi107 ceph-mon[212750]: pgmap v1665: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 444 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7760/21342 objects degraded (36.360%); 2377/21342 objects misplaced (11.138%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:23.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:23 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7760/21342 objects degraded (36.360%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:23.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:23 smithi107 ceph-mon[220921]: pgmap v1665: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 444 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7760/21342 objects degraded (36.360%); 2377/21342 objects misplaced (11.138%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:23.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:23 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7760/21342 objects degraded (36.360%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:23 smithi150 ceph-mon[194178]: pgmap v1665: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 444 MiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 7760/21342 objects degraded (36.360%); 2377/21342 objects misplaced (11.138%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:23.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:23 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7760/21342 objects degraded (36.360%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:23.735 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:23.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:23.838 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:23.942 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:24.256 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T06:43:24.256 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:43:24.289 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:24.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:24.413 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:24.567 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:24.698 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:24.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:24.813 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:24.854 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:24.897 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:24.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:25.004 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:25.030 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:25.098 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:25.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:25.204 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:25.294 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:25.405 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:25 smithi107 ceph-mon[212750]: pgmap v1666: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 367 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 7760/17676 objects degraded (43.901%); 2377/17676 objects misplaced (13.448%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:25.406 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:25 smithi107 ceph-mon[220921]: pgmap v1666: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 367 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 7760/17676 objects degraded (43.901%); 2377/17676 objects misplaced (13.448%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:25.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:25 smithi150 ceph-mon[194178]: pgmap v1666: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 367 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 7760/17676 objects degraded (43.901%); 2377/17676 objects misplaced (13.448%); 1.4 MiB/s, 78 objects/s recovering 2023-10-09T06:43:25.441 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:25.473 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:25.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:25.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:25.586 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:25.617 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:25.638 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:25.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:25.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:25.817 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:25.898 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:26.111 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:26.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:26.220 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:26.258 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:26.321 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:43:26.327 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:26.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:26.432 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:26.542 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:26.937 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:43:26.937 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:43:26.938 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:43:26.938 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:43:26.938 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:43:27.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:27 smithi107 ceph-mon[220921]: pgmap v1667: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 248 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7562/11943 objects degraded (63.317%); 2278/11943 objects misplaced (19.074%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:43:27.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:27 smithi107 ceph-mon[212750]: pgmap v1667: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 248 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7562/11943 objects degraded (63.317%); 2278/11943 objects misplaced (19.074%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:43:27.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:27 smithi150 ceph-mon[194178]: pgmap v1667: 129 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 248 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 7562/11943 objects degraded (63.317%); 2278/11943 objects misplaced (19.074%); 1.3 MiB/s, 76 objects/s recovering 2023-10-09T06:43:28.457 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:28 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7562/11943 objects degraded (63.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:28.459 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:28 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7562/11943 objects degraded (63.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:28.650 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:28.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:28 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7562/11943 objects degraded (63.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:28.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:28.762 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:28.797 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:28.866 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:28.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:28.973 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:28.989 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Removed 100770 objects 2023-10-09T06:43:28.989 INFO:tasks.radosbench.radosbench.0.smithi107.stdout:Clean up completed and total clean up time :133.835 2023-10-09T06:43:29.015 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:29.255 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:29.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:29.287 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:29 smithi107 ceph-mon[212750]: pgmap v1668: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 222 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7562/10707 objects degraded (70.627%); 2278/10707 objects misplaced (21.276%); 1.1 MiB/s, 69 objects/s recovering 2023-10-09T06:43:29.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:29 smithi107 ceph-mon[220921]: pgmap v1668: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 222 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7562/10707 objects degraded (70.627%); 2278/10707 objects misplaced (21.276%); 1.1 MiB/s, 69 objects/s recovering 2023-10-09T06:43:29.288 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:43:29.360 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:29.387 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:29.431 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:29.437 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:29 smithi150 ceph-mon[194178]: pgmap v1668: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 222 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 7562/10707 objects degraded (70.627%); 2278/10707 objects misplaced (21.276%); 1.1 MiB/s, 69 objects/s recovering 2023-10-09T06:43:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:29.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:29.539 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:29.564 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:29.565 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:29.609 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:29.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:29.717 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:29.743 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:29.788 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:29.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:29.893 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:29.919 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:29.963 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:29.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:30.071 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:30.098 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:30.114 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:30.141 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:30.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:30.180 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:43:30.204 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:43:30.207 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:43:30.220 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:43:30.223 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:43:30.248 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:30.286 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:30.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:43:30.713 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-10-09T06:43:30.713 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: pgmap v1669: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 120 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7562/5814 objects degraded (130.065%); 2278/5814 objects misplaced (39.181%); 1.1 MiB/s, 67 objects/s recovering 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[212750]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: pgmap v1669: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 120 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7562/5814 objects degraded (130.065%); 2278/5814 objects misplaced (39.181%); 1.1 MiB/s, 67 objects/s recovering 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:31 smithi107 ceph-mon[220921]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: pgmap v1669: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 120 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7562/5814 objects degraded (130.065%); 2278/5814 objects misplaced (39.181%); 1.1 MiB/s, 67 objects/s recovering 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2829914470' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: osdmap e538: 8 total, 8 up, 8 in 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: from='client.37317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: failed to encode map e538 with expected crc 2023-10-09T06:43:31.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:31 smithi150 ceph-mon[194178]: failed to encode map e538 with expected crc 2023-10-09T06:43:32.161 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:32] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T06:43:32.248 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:32.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:32.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:32.353 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:32.379 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:32.422 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:32.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:32 smithi150 ceph-mon[194178]: failed to encode map e538 with expected crc 2023-10-09T06:43:32.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:32.425 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:32.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:32.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:32.527 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[212750]: failed to encode map e538 with expected crc 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[220921]: failed to encode map e538 with expected crc 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:32.553 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:32.555 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:32.752 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:32.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:32.857 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:32.883 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:32.925 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:32.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:33.031 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:33.058 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:33.102 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:33.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:33.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:33.208 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.235 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:33.280 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:33.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:33.385 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:33.413 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:33.457 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:33.460 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: pgmap v1671: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 26 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 7384/1299 objects degraded (568.437%); 2189/1299 objects misplaced (168.514%); 1.2 MiB/s, 126 objects/s recovering 2023-10-09T06:43:33.460 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.461 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:43:33.461 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.461 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.461 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:33 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 7384/1299 objects degraded (568.437%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:33.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:33.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:33.564 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:33.591 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:33.634 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:33.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: pgmap v1671: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 26 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 7384/1299 objects degraded (568.437%); 2189/1299 objects misplaced (168.514%); 1.2 MiB/s, 126 objects/s recovering 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 7384/1299 objects degraded (568.437%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:33.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: pgmap v1671: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 26 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 7384/1299 objects degraded (568.437%); 2189/1299 objects misplaced (168.514%); 1.2 MiB/s, 126 objects/s recovering 2023-10-09T06:43:33.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: osdmap e539: 8 total, 8 up, 8 in 2023-10-09T06:43:33.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: failed to encode map e539 with expected crc 2023-10-09T06:43:33.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:33 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 7384/1299 objects degraded (568.437%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:33.739 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:33.771 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1670,"stamp":"2023-10-09T06:43:30.169617+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":48011192,"num_objects":749,"num_object_clones":0,"num_object_copies":2247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7384,"num_objects_misplaced":2189,"num_objects_unfound":0,"num_objects_dirty":749,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163274,"num_write_kb":4557470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3231,"num_bytes_recovered":132261836,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":196728,"ondisk_log_size":196728,"up":387,"acting":384,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":394,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10938572,"kb_used_data":700108,"kb_used_omap":20,"kb_used_meta":10238251,"kb_avail":739022644,"statfs":{"total":767960285184,"available":756759187456,"internally_reserved":0,"allocated":716910592,"data_stored":703358319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10483969678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,5,7,2,2,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-537886720,"num_objects":-8207,"num_object_clones":0,"num_object_copies":-24621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-600,"num_objects_misplaced":-300,"num_objects_unfound":0,"num_objects_dirty":-8207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":900,"num_bytes_recovered":19660800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.511243"},"pg_stats":[{"pgid":"14.17","version":"537'8381","reported_seq":8564,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.583746+0000","last_change":"2023-10-09T06:40:52.372943+0000","last_active":"2023-10-09T06:43:27.583746+0000","last_peered":"2023-10-09T06:43:27.583746+0000","last_clean":"2023-10-09T06:43:27.583746+0000","last_became_active":"2023-10-09T06:40:43.185643+0000","last_became_peered":"2023-10-09T06:40:43.185643+0000","last_unstale":"2023-10-09T06:43:27.583746+0000","last_undegraded":"2023-10-09T06:43:27.583746+0000","last_fullsized":"2023-10-09T06:43:27.583746+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4783","last_scrub_stamp":"2023-10-09T06:40:52.372858+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:52.372858+0000","objects_scrubbed":2511,"log_size":8381,"log_dups_size":0,"ondisk_log_size":8381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:22.222358+0000","scrub_duration":0.27153366200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1900544,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3598,"num_write_kb":35712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.18","version":"77'161","reported_seq":831,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973991+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:43:02.973991+0000","last_peered":"2023-10-09T06:43:02.973991+0000","last_clean":"2023-10-09T06:43:02.973991+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:43:02.973991+0000","last_undegraded":"2023-10-09T06:43:02.973991+0000","last_fullsized":"2023-10-09T06:43:02.973991+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":516,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967763+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:43:02.967763+0000","last_peered":"2023-10-09T06:43:02.967763+0000","last_clean":"2023-10-09T06:43:02.967763+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:43:02.967763+0000","last_undegraded":"2023-10-09T06:43:02.967763+0000","last_fullsized":"2023-10-09T06:43:02.967763+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974811+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:43:02.974811+0000","last_peered":"2023-10-09T06:43:02.974811+0000","last_clean":"2023-10-09T06:43:02.974811+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:43:02.974811+0000","last_undegraded":"2023-10-09T06:43:02.974811+0000","last_fullsized":"2023-10-09T06:43:02.974811+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"537'8298","reported_seq":8434,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:43:28.955364+0000","last_change":"2023-10-09T06:42:45.839310+0000","last_active":"2023-10-09T06:43:28.955364+0000","last_peered":"2023-10-09T06:43:28.955364+0000","last_clean":"2023-10-09T06:42:43.293915+0000","last_became_active":"2023-10-09T06:42:45.820286+0000","last_became_peered":"2023-10-09T06:42:45.820286+0000","last_unstale":"2023-10-09T06:43:28.955364+0000","last_undegraded":"2023-10-09T06:42:45.719211+0000","last_fullsized":"2023-10-09T06:42:45.692966+0000","mapping_epoch":533,"log_start":"493'1290","ondisk_log_start":"493'1290","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7291","last_scrub_stamp":"2023-10-09T06:42:37.253113+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:37.253113+0000","objects_scrubbed":1009,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:53.601935+0000","scrub_duration":0.250396232,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3006,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1009,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[3,6],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6","objects":-897}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":792,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967675+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:43:02.967675+0000","last_peered":"2023-10-09T06:43:02.967675+0000","last_clean":"2023-10-09T06:43:02.967675+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:43:02.967675+0000","last_undegraded":"2023-10-09T06:43:02.967675+0000","last_fullsized":"2023-10-09T06:43:02.967675+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":534,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.970857+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:43:02.970857+0000","last_peered":"2023-10-09T06:43:02.970857+0000","last_clean":"2023-10-09T06:43:02.970857+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:43:02.970857+0000","last_undegraded":"2023-10-09T06:43:02.970857+0000","last_fullsized":"2023-10-09T06:43:02.970857+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":539,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966349+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:43:02.966349+0000","last_peered":"2023-10-09T06:43:02.966349+0000","last_clean":"2023-10-09T06:43:02.966349+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:43:02.966349+0000","last_undegraded":"2023-10-09T06:43:02.966349+0000","last_fullsized":"2023-10-09T06:43:02.966349+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"537'8415","reported_seq":8483,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.915698+0000","last_change":"2023-10-09T06:40:43.168550+0000","last_active":"2023-10-09T06:43:28.915698+0000","last_peered":"2023-10-09T06:43:28.915698+0000","last_clean":"2023-10-09T06:43:28.915698+0000","last_became_active":"2023-10-09T06:40:43.166693+0000","last_became_peered":"2023-10-09T06:40:43.166693+0000","last_unstale":"2023-10-09T06:43:28.915698+0000","last_undegraded":"2023-10-09T06:43:28.915698+0000","last_fullsized":"2023-10-09T06:43:28.915698+0000","mapping_epoch":500,"log_start":"493'24","ondisk_log_start":"493'24","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8391,"log_dups_size":0,"ondisk_log_size":8391,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6154,"num_write_kb":197504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":784,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967675+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:43:02.967675+0000","last_peered":"2023-10-09T06:43:02.967675+0000","last_clean":"2023-10-09T06:43:02.967675+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:43:02.967675+0000","last_undegraded":"2023-10-09T06:43:02.967675+0000","last_fullsized":"2023-10-09T06:43:02.967675+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974177+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:43:02.974177+0000","last_peered":"2023-10-09T06:43:02.974177+0000","last_clean":"2023-10-09T06:43:02.974177+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:43:02.974177+0000","last_undegraded":"2023-10-09T06:43:02.974177+0000","last_fullsized":"2023-10-09T06:43:02.974177+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":526,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974753+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:43:02.974753+0000","last_peered":"2023-10-09T06:43:02.974753+0000","last_clean":"2023-10-09T06:43:02.974753+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:43:02.974753+0000","last_undegraded":"2023-10-09T06:43:02.974753+0000","last_fullsized":"2023-10-09T06:43:02.974753+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":645,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975979+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:43:02.975979+0000","last_peered":"2023-10-09T06:43:02.975979+0000","last_clean":"2023-10-09T06:43:02.975979+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:43:02.975979+0000","last_undegraded":"2023-10-09T06:43:02.975979+0000","last_fullsized":"2023-10-09T06:43:02.975979+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":213,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966691+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:43:02.966691+0000","last_peered":"2023-10-09T06:43:02.966691+0000","last_clean":"2023-10-09T06:43:02.966691+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:43:02.966691+0000","last_undegraded":"2023-10-09T06:43:02.966691+0000","last_fullsized":"2023-10-09T06:43:02.966691+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974300+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:43:02.974300+0000","last_peered":"2023-10-09T06:43:02.974300+0000","last_clean":"2023-10-09T06:43:02.974300+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:43:02.974300+0000","last_undegraded":"2023-10-09T06:43:02.974300+0000","last_fullsized":"2023-10-09T06:43:02.974300+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":682,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975812+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:43:02.975812+0000","last_peered":"2023-10-09T06:43:02.975812+0000","last_clean":"2023-10-09T06:43:02.975812+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:43:02.975812+0000","last_undegraded":"2023-10-09T06:43:02.975812+0000","last_fullsized":"2023-10-09T06:43:02.975812+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974467+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:43:02.974467+0000","last_peered":"2023-10-09T06:43:02.974467+0000","last_clean":"2023-10-09T06:43:02.974467+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:43:02.974467+0000","last_undegraded":"2023-10-09T06:43:02.974467+0000","last_fullsized":"2023-10-09T06:43:02.974467+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974934+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:43:02.974934+0000","last_peered":"2023-10-09T06:43:02.974934+0000","last_clean":"2023-10-09T06:43:02.974934+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:43:02.974934+0000","last_undegraded":"2023-10-09T06:43:02.974934+0000","last_fullsized":"2023-10-09T06:43:02.974934+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"537'8892","reported_seq":8959,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.881047+0000","last_change":"2023-10-09T06:42:32.156372+0000","last_active":"2023-10-09T06:43:28.881047+0000","last_peered":"2023-10-09T06:43:28.881047+0000","last_clean":"2023-10-09T06:43:28.881047+0000","last_became_active":"2023-10-09T06:40:43.180666+0000","last_became_peered":"2023-10-09T06:40:43.180666+0000","last_unstale":"2023-10-09T06:43:28.881047+0000","last_undegraded":"2023-10-09T06:43:28.881047+0000","last_fullsized":"2023-10-09T06:43:28.881047+0000","mapping_epoch":500,"log_start":"493'723","ondisk_log_start":"493'723","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8169,"log_dups_size":0,"ondisk_log_size":8169,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1540096,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6502,"num_write_kb":208800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":666,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966549+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:43:02.966549+0000","last_peered":"2023-10-09T06:43:02.966549+0000","last_clean":"2023-10-09T06:43:02.966549+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:43:02.966549+0000","last_undegraded":"2023-10-09T06:43:02.966549+0000","last_fullsized":"2023-10-09T06:43:02.966549+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":418,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976240+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:43:02.976240+0000","last_peered":"2023-10-09T06:43:02.976240+0000","last_clean":"2023-10-09T06:43:02.976240+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:43:02.976240+0000","last_undegraded":"2023-10-09T06:43:02.976240+0000","last_fullsized":"2023-10-09T06:43:02.976240+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":476,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.086518+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:43:16.086518+0000","last_peered":"2023-10-09T06:43:16.086518+0000","last_clean":"2023-10-09T06:43:16.086518+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:43:16.086518+0000","last_undegraded":"2023-10-09T06:43:16.086518+0000","last_fullsized":"2023-10-09T06:43:16.086518+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"537'8531","reported_seq":8661,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.885005+0000","last_change":"2023-10-09T06:40:50.305746+0000","last_active":"2023-10-09T06:43:28.885005+0000","last_peered":"2023-10-09T06:43:28.885005+0000","last_clean":"2023-10-09T06:43:28.885005+0000","last_became_active":"2023-10-09T06:40:43.093584+0000","last_became_peered":"2023-10-09T06:40:43.093584+0000","last_unstale":"2023-10-09T06:43:28.885005+0000","last_undegraded":"2023-10-09T06:43:28.885005+0000","last_fullsized":"2023-10-09T06:43:28.885005+0000","mapping_epoch":500,"log_start":"494'1694","ondisk_log_start":"494'1694","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"503'4832","last_scrub_stamp":"2023-10-09T06:40:50.305626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:50.305626+0000","objects_scrubbed":2516,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:57.896131+0000","scrub_duration":0.56208410600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3708,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":647,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971389+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:43:02.971389+0000","last_peered":"2023-10-09T06:43:02.971389+0000","last_clean":"2023-10-09T06:43:02.971389+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:43:02.971389+0000","last_undegraded":"2023-10-09T06:43:02.971389+0000","last_fullsized":"2023-10-09T06:43:02.971389+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":543,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.970124+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:43:02.970124+0000","last_peered":"2023-10-09T06:43:02.970124+0000","last_clean":"2023-10-09T06:43:02.970124+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:43:02.970124+0000","last_undegraded":"2023-10-09T06:43:02.970124+0000","last_fullsized":"2023-10-09T06:43:02.970124+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.969454+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:43:02.969454+0000","last_peered":"2023-10-09T06:43:02.969454+0000","last_clean":"2023-10-09T06:43:02.969454+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:43:02.969454+0000","last_undegraded":"2023-10-09T06:43:02.969454+0000","last_fullsized":"2023-10-09T06:43:02.969454+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":811,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.965916+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:43:02.965916+0000","last_peered":"2023-10-09T06:43:02.965916+0000","last_clean":"2023-10-09T06:43:02.965916+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:43:02.965916+0000","last_undegraded":"2023-10-09T06:43:02.965916+0000","last_fullsized":"2023-10-09T06:43:02.965916+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":386,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975493+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:43:02.975493+0000","last_peered":"2023-10-09T06:43:02.975493+0000","last_clean":"2023-10-09T06:43:02.975493+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:43:02.975493+0000","last_undegraded":"2023-10-09T06:43:02.975493+0000","last_fullsized":"2023-10-09T06:43:02.975493+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976086+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:43:02.976086+0000","last_peered":"2023-10-09T06:43:02.976086+0000","last_clean":"2023-10-09T06:43:02.976086+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:43:02.976086+0000","last_undegraded":"2023-10-09T06:43:02.976086+0000","last_fullsized":"2023-10-09T06:43:02.976086+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":809,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974075+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:43:02.974075+0000","last_peered":"2023-10-09T06:43:02.974075+0000","last_clean":"2023-10-09T06:43:02.974075+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:43:02.974075+0000","last_undegraded":"2023-10-09T06:43:02.974075+0000","last_fullsized":"2023-10-09T06:43:02.974075+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":537,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:14.915493+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:43:14.915493+0000","last_peered":"2023-10-09T06:43:14.915493+0000","last_clean":"2023-10-09T06:43:14.915493+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:43:14.915493+0000","last_undegraded":"2023-10-09T06:43:14.915493+0000","last_fullsized":"2023-10-09T06:43:14.915493+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":357,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976199+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:43:02.976199+0000","last_peered":"2023-10-09T06:43:02.976199+0000","last_clean":"2023-10-09T06:43:02.976199+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:43:02.976199+0000","last_undegraded":"2023-10-09T06:43:02.976199+0000","last_fullsized":"2023-10-09T06:43:02.976199+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":865,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973576+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:43:02.973576+0000","last_peered":"2023-10-09T06:43:02.973576+0000","last_clean":"2023-10-09T06:43:02.973576+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:43:02.973576+0000","last_undegraded":"2023-10-09T06:43:02.973576+0000","last_fullsized":"2023-10-09T06:43:02.973576+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":380,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975060+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:43:02.975060+0000","last_peered":"2023-10-09T06:43:02.975060+0000","last_clean":"2023-10-09T06:43:02.975060+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:43:02.975060+0000","last_undegraded":"2023-10-09T06:43:02.975060+0000","last_fullsized":"2023-10-09T06:43:02.975060+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":213,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966013+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:43:02.966013+0000","last_peered":"2023-10-09T06:43:02.966013+0000","last_clean":"2023-10-09T06:43:02.966013+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:43:02.966013+0000","last_undegraded":"2023-10-09T06:43:02.966013+0000","last_fullsized":"2023-10-09T06:43:02.966013+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":651,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974674+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:43:02.974674+0000","last_peered":"2023-10-09T06:43:02.974674+0000","last_clean":"2023-10-09T06:43:02.974674+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:43:02.974674+0000","last_undegraded":"2023-10-09T06:43:02.974674+0000","last_fullsized":"2023-10-09T06:43:02.974674+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":615,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973231+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:43:02.973231+0000","last_peered":"2023-10-09T06:43:02.973231+0000","last_clean":"2023-10-09T06:43:02.973231+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:43:02.973231+0000","last_undegraded":"2023-10-09T06:43:02.973231+0000","last_fullsized":"2023-10-09T06:43:02.973231+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975053+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:43:02.975053+0000","last_peered":"2023-10-09T06:43:02.975053+0000","last_clean":"2023-10-09T06:43:02.975053+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:43:02.975053+0000","last_undegraded":"2023-10-09T06:43:02.975053+0000","last_fullsized":"2023-10-09T06:43:02.975053+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"537'8495","reported_seq":8639,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.855841+0000","last_change":"2023-10-09T06:42:35.324097+0000","last_active":"2023-10-09T06:43:27.855841+0000","last_peered":"2023-10-09T06:43:27.855841+0000","last_clean":"2023-10-09T06:43:27.855841+0000","last_became_active":"2023-10-09T06:40:43.094441+0000","last_became_peered":"2023-10-09T06:40:43.094441+0000","last_unstale":"2023-10-09T06:43:27.855841+0000","last_undegraded":"2023-10-09T06:43:27.855841+0000","last_fullsized":"2023-10-09T06:43:27.855841+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"525'7435","last_scrub_stamp":"2023-10-09T06:42:35.324027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:35.324027+0000","objects_scrubbed":1088,"log_size":8495,"log_dups_size":0,"ondisk_log_size":8495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:40.756290+0000","scrub_duration":0.32391870699999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1063,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":664,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975995+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:43:02.975995+0000","last_peered":"2023-10-09T06:43:02.975995+0000","last_clean":"2023-10-09T06:43:02.975995+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:43:02.975995+0000","last_undegraded":"2023-10-09T06:43:02.975995+0000","last_fullsized":"2023-10-09T06:43:02.975995+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":422,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975995+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:43:02.975995+0000","last_peered":"2023-10-09T06:43:02.975995+0000","last_clean":"2023-10-09T06:43:02.975995+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:43:02.975995+0000","last_undegraded":"2023-10-09T06:43:02.975995+0000","last_fullsized":"2023-10-09T06:43:02.975995+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":320,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974966+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:43:02.974966+0000","last_peered":"2023-10-09T06:43:02.974966+0000","last_clean":"2023-10-09T06:43:02.974966+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:43:02.974966+0000","last_undegraded":"2023-10-09T06:43:02.974966+0000","last_fullsized":"2023-10-09T06:43:02.974966+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"537'8665","reported_seq":8734,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.962888+0000","last_change":"2023-10-09T06:42:15.187041+0000","last_active":"2023-10-09T06:43:28.962888+0000","last_peered":"2023-10-09T06:43:28.962888+0000","last_clean":"2023-10-09T06:43:28.962888+0000","last_became_active":"2023-10-09T06:40:43.165023+0000","last_became_peered":"2023-10-09T06:40:43.165023+0000","last_unstale":"2023-10-09T06:43:28.962888+0000","last_undegraded":"2023-10-09T06:43:28.962888+0000","last_fullsized":"2023-10-09T06:43:28.962888+0000","mapping_epoch":489,"log_start":"493'402","ondisk_log_start":"493'402","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8263,"log_dups_size":0,"ondisk_log_size":8263,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":589824,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6345,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":925,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:09.146197+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:43:09.146197+0000","last_peered":"2023-10-09T06:43:09.146197+0000","last_clean":"2023-10-09T06:43:09.146197+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:43:09.146197+0000","last_undegraded":"2023-10-09T06:43:09.146197+0000","last_fullsized":"2023-10-09T06:43:09.146197+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":384,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976075+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:43:02.976075+0000","last_peered":"2023-10-09T06:43:02.976075+0000","last_clean":"2023-10-09T06:43:02.976075+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:43:02.976075+0000","last_undegraded":"2023-10-09T06:43:02.976075+0000","last_fullsized":"2023-10-09T06:43:02.976075+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":534,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967798+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:43:02.967798+0000","last_peered":"2023-10-09T06:43:02.967798+0000","last_clean":"2023-10-09T06:43:02.967798+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:43:02.967798+0000","last_undegraded":"2023-10-09T06:43:02.967798+0000","last_fullsized":"2023-10-09T06:43:02.967798+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"537'8544","reported_seq":8613,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.932397+0000","last_change":"2023-10-09T06:42:26.019283+0000","last_active":"2023-10-09T06:43:28.932397+0000","last_peered":"2023-10-09T06:43:28.932397+0000","last_clean":"2023-10-09T06:43:28.932397+0000","last_became_active":"2023-10-09T06:40:43.080150+0000","last_became_peered":"2023-10-09T06:40:43.080150+0000","last_unstale":"2023-10-09T06:43:28.932397+0000","last_undegraded":"2023-10-09T06:43:28.932397+0000","last_fullsized":"2023-10-09T06:43:28.932397+0000","mapping_epoch":489,"log_start":"493'217","ondisk_log_start":"493'217","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8327,"log_dups_size":0,"ondisk_log_size":8327,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1540096,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6253,"num_write_kb":200864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":838,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974670+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:43:02.974670+0000","last_peered":"2023-10-09T06:43:02.974670+0000","last_clean":"2023-10-09T06:43:02.974670+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:43:02.974670+0000","last_undegraded":"2023-10-09T06:43:02.974670+0000","last_fullsized":"2023-10-09T06:43:02.974670+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":440,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976110+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:43:02.976110+0000","last_peered":"2023-10-09T06:43:02.976110+0000","last_clean":"2023-10-09T06:43:02.976110+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:43:02.976110+0000","last_undegraded":"2023-10-09T06:43:02.976110+0000","last_fullsized":"2023-10-09T06:43:02.976110+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":513,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973599+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:43:02.973599+0000","last_peered":"2023-10-09T06:43:02.973599+0000","last_clean":"2023-10-09T06:43:02.973599+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:43:02.973599+0000","last_undegraded":"2023-10-09T06:43:02.973599+0000","last_fullsized":"2023-10-09T06:43:02.973599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"537'8653","reported_seq":8785,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.974974+0000","last_change":"2023-10-09T06:40:47.766598+0000","last_active":"2023-10-09T06:43:28.974974+0000","last_peered":"2023-10-09T06:43:28.974974+0000","last_clean":"2023-10-09T06:43:28.974974+0000","last_became_active":"2023-10-09T06:40:43.101954+0000","last_became_peered":"2023-10-09T06:40:43.101954+0000","last_unstale":"2023-10-09T06:43:28.974974+0000","last_undegraded":"2023-10-09T06:43:28.974974+0000","last_fullsized":"2023-10-09T06:43:28.974974+0000","mapping_epoch":489,"log_start":"494'2061","ondisk_log_start":"494'2061","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4698","last_scrub_stamp":"2023-10-09T06:40:47.766356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.766356+0000","objects_scrubbed":2441,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:46:35.858607+0000","scrub_duration":1.024576309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3965,"num_write_kb":48768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":661,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971386+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:43:02.971386+0000","last_peered":"2023-10-09T06:43:02.971386+0000","last_clean":"2023-10-09T06:43:02.971386+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:43:02.971386+0000","last_undegraded":"2023-10-09T06:43:02.971386+0000","last_fullsized":"2023-10-09T06:43:02.971386+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976168+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:43:02.976168+0000","last_peered":"2023-10-09T06:43:02.976168+0000","last_clean":"2023-10-09T06:43:02.976168+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:43:02.976168+0000","last_undegraded":"2023-10-09T06:43:02.976168+0000","last_fullsized":"2023-10-09T06:43:02.976168+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":501,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967740+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:43:02.967740+0000","last_peered":"2023-10-09T06:43:02.967740+0000","last_clean":"2023-10-09T06:43:02.967740+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:43:02.967740+0000","last_undegraded":"2023-10-09T06:43:02.967740+0000","last_fullsized":"2023-10-09T06:43:02.967740+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"537'8523","reported_seq":8591,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.908986+0000","last_change":"2023-10-09T06:42:18.037795+0000","last_active":"2023-10-09T06:43:28.908986+0000","last_peered":"2023-10-09T06:43:28.908986+0000","last_clean":"2023-10-09T06:43:28.908986+0000","last_became_active":"2023-10-09T06:40:43.159937+0000","last_became_peered":"2023-10-09T06:40:43.159937+0000","last_unstale":"2023-10-09T06:43:28.908986+0000","last_undegraded":"2023-10-09T06:43:28.908986+0000","last_fullsized":"2023-10-09T06:43:28.908986+0000","mapping_epoch":489,"log_start":"493'213","ondisk_log_start":"493'213","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8310,"log_dups_size":0,"ondisk_log_size":8310,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6243,"num_write_kb":199936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":889,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974875+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:43:02.974875+0000","last_peered":"2023-10-09T06:43:02.974875+0000","last_clean":"2023-10-09T06:43:02.974875+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:43:02.974875+0000","last_undegraded":"2023-10-09T06:43:02.974875+0000","last_fullsized":"2023-10-09T06:43:02.974875+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":267,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.078837+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:43:16.078837+0000","last_peered":"2023-10-09T06:43:16.078837+0000","last_clean":"2023-10-09T06:43:16.078837+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:43:16.078837+0000","last_undegraded":"2023-10-09T06:43:16.078837+0000","last_fullsized":"2023-10-09T06:43:16.078837+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":517,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.969967+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:43:02.969967+0000","last_peered":"2023-10-09T06:43:02.969967+0000","last_clean":"2023-10-09T06:43:02.969967+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:43:02.969967+0000","last_undegraded":"2023-10-09T06:43:02.969967+0000","last_fullsized":"2023-10-09T06:43:02.969967+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"537'8604","reported_seq":8776,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.684066+0000","last_change":"2023-10-09T06:40:45.736296+0000","last_active":"2023-10-09T06:43:27.684066+0000","last_peered":"2023-10-09T06:43:27.684066+0000","last_clean":"2023-10-09T06:43:27.684066+0000","last_became_active":"2023-10-09T06:40:43.168245+0000","last_became_peered":"2023-10-09T06:40:43.168245+0000","last_unstale":"2023-10-09T06:43:27.684066+0000","last_undegraded":"2023-10-09T06:43:27.684066+0000","last_fullsized":"2023-10-09T06:43:27.684066+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4658","last_scrub_stamp":"2023-10-09T06:40:45.736230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:45.736230+0000","objects_scrubbed":2357,"log_size":8604,"log_dups_size":0,"ondisk_log_size":8604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:33:31.145769+0000","scrub_duration":0.58060363000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1835008,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3955,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":572,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974393+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:43:02.974393+0000","last_peered":"2023-10-09T06:43:02.974393+0000","last_clean":"2023-10-09T06:43:02.974393+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:43:02.974393+0000","last_undegraded":"2023-10-09T06:43:02.974393+0000","last_fullsized":"2023-10-09T06:43:02.974393+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":494,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973520+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:43:02.973520+0000","last_peered":"2023-10-09T06:43:02.973520+0000","last_clean":"2023-10-09T06:43:02.973520+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:43:02.973520+0000","last_undegraded":"2023-10-09T06:43:02.973520+0000","last_fullsized":"2023-10-09T06:43:02.973520+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":514,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971258+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:43:02.971258+0000","last_peered":"2023-10-09T06:43:02.971258+0000","last_clean":"2023-10-09T06:43:02.971258+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:43:02.971258+0000","last_undegraded":"2023-10-09T06:43:02.971258+0000","last_fullsized":"2023-10-09T06:43:02.971258+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"537'8449","reported_seq":8626,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.856928+0000","last_change":"2023-10-09T06:40:51.573557+0000","last_active":"2023-10-09T06:43:27.856928+0000","last_peered":"2023-10-09T06:43:27.856928+0000","last_clean":"2023-10-09T06:43:27.856928+0000","last_became_active":"2023-10-09T06:40:43.081959+0000","last_became_peered":"2023-10-09T06:40:43.081959+0000","last_unstale":"2023-10-09T06:43:27.856928+0000","last_undegraded":"2023-10-09T06:43:27.856928+0000","last_fullsized":"2023-10-09T06:43:27.856928+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"503'4739","last_scrub_stamp":"2023-10-09T06:40:51.573471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:51.573471+0000","objects_scrubbed":2521,"log_size":8449,"log_dups_size":0,"ondisk_log_size":8449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:12:30.924217+0000","scrub_duration":0.34142884899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1769472,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3710,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":875,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:05.816631+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:43:05.816631+0000","last_peered":"2023-10-09T06:43:05.816631+0000","last_clean":"2023-10-09T06:43:05.816631+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:43:05.816631+0000","last_undegraded":"2023-10-09T06:43:05.816631+0000","last_fullsized":"2023-10-09T06:43:05.816631+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967677+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:43:02.967677+0000","last_peered":"2023-10-09T06:43:02.967677+0000","last_clean":"2023-10-09T06:43:02.967677+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:43:02.967677+0000","last_undegraded":"2023-10-09T06:43:02.967677+0000","last_fullsized":"2023-10-09T06:43:02.967677+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":497,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971562+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:43:02.971562+0000","last_peered":"2023-10-09T06:43:02.971562+0000","last_clean":"2023-10-09T06:43:02.971562+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:43:02.971562+0000","last_undegraded":"2023-10-09T06:43:02.971562+0000","last_fullsized":"2023-10-09T06:43:02.971562+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"537'8413","reported_seq":8541,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.863186+0000","last_change":"2023-10-09T06:40:47.192403+0000","last_active":"2023-10-09T06:43:28.863186+0000","last_peered":"2023-10-09T06:43:28.863186+0000","last_clean":"2023-10-09T06:43:28.863186+0000","last_became_active":"2023-10-09T06:40:43.074962+0000","last_became_peered":"2023-10-09T06:40:43.074962+0000","last_unstale":"2023-10-09T06:43:28.863186+0000","last_undegraded":"2023-10-09T06:43:28.863186+0000","last_fullsized":"2023-10-09T06:43:28.863186+0000","mapping_epoch":489,"log_start":"493'89","ondisk_log_start":"493'89","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"501'4664","last_scrub_stamp":"2023-10-09T06:40:47.192137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:47.192137+0000","objects_scrubbed":2418,"log_size":8324,"log_dups_size":0,"ondisk_log_size":8324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:24.365070+0000","scrub_duration":0.455376642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3752,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":499,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973491+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:43:02.973491+0000","last_peered":"2023-10-09T06:43:02.973491+0000","last_clean":"2023-10-09T06:43:02.973491+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:43:02.973491+0000","last_undegraded":"2023-10-09T06:43:02.973491+0000","last_fullsized":"2023-10-09T06:43:02.973491+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":241,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974575+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:43:02.974575+0000","last_peered":"2023-10-09T06:43:02.974575+0000","last_clean":"2023-10-09T06:43:02.974575+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:43:02.974575+0000","last_undegraded":"2023-10-09T06:43:02.974575+0000","last_fullsized":"2023-10-09T06:43:02.974575+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":494,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971508+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:43:02.971508+0000","last_peered":"2023-10-09T06:43:02.971508+0000","last_clean":"2023-10-09T06:43:02.971508+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:43:02.971508+0000","last_undegraded":"2023-10-09T06:43:02.971508+0000","last_fullsized":"2023-10-09T06:43:02.971508+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"537'8933","reported_seq":9000,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.951924+0000","last_change":"2023-10-09T06:42:25.028645+0000","last_active":"2023-10-09T06:43:28.951924+0000","last_peered":"2023-10-09T06:43:28.951924+0000","last_clean":"2023-10-09T06:43:28.951924+0000","last_became_active":"2023-10-09T06:40:43.080919+0000","last_became_peered":"2023-10-09T06:40:43.080919+0000","last_unstale":"2023-10-09T06:43:28.951924+0000","last_undegraded":"2023-10-09T06:43:28.951924+0000","last_fullsized":"2023-10-09T06:43:28.951924+0000","mapping_epoch":489,"log_start":"493'870","ondisk_log_start":"493'870","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8063,"log_dups_size":0,"ondisk_log_size":8063,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6542,"num_write_kb":208608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":829,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974839+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:43:02.974839+0000","last_peered":"2023-10-09T06:43:02.974839+0000","last_clean":"2023-10-09T06:43:02.974839+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:43:02.974839+0000","last_undegraded":"2023-10-09T06:43:02.974839+0000","last_fullsized":"2023-10-09T06:43:02.974839+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":546,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971319+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:43:02.971319+0000","last_peered":"2023-10-09T06:43:02.971319+0000","last_clean":"2023-10-09T06:43:02.971319+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:43:02.971319+0000","last_undegraded":"2023-10-09T06:43:02.971319+0000","last_fullsized":"2023-10-09T06:43:02.971319+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.079335+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:43:16.079335+0000","last_peered":"2023-10-09T06:43:16.079335+0000","last_clean":"2023-10-09T06:43:16.079335+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:43:16.079335+0000","last_undegraded":"2023-10-09T06:43:16.079335+0000","last_fullsized":"2023-10-09T06:43:16.079335+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":480,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973433+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:43:02.973433+0000","last_peered":"2023-10-09T06:43:02.973433+0000","last_clean":"2023-10-09T06:43:02.973433+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:43:02.973433+0000","last_undegraded":"2023-10-09T06:43:02.973433+0000","last_fullsized":"2023-10-09T06:43:02.973433+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"537'12478","reported_seq":12534,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.732591+0000","last_change":"2023-10-09T06:39:33.924982+0000","last_active":"2023-10-09T06:43:27.732591+0000","last_peered":"2023-10-09T06:43:27.732591+0000","last_clean":"2023-10-09T06:43:27.732591+0000","last_became_active":"2023-10-09T06:39:33.924673+0000","last_became_peered":"2023-10-09T06:39:33.924673+0000","last_unstale":"2023-10-09T06:43:27.732591+0000","last_undegraded":"2023-10-09T06:43:27.732591+0000","last_fullsized":"2023-10-09T06:43:27.732591+0000","mapping_epoch":489,"log_start":"511'7132","ondisk_log_start":"511'7132","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5346,"log_dups_size":0,"ondisk_log_size":5346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:58.877084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3932160,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12478,"num_write_kb":401216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":718,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.086487+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:43:16.086487+0000","last_peered":"2023-10-09T06:43:16.086487+0000","last_clean":"2023-10-09T06:43:16.086487+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:43:16.086487+0000","last_undegraded":"2023-10-09T06:43:16.086487+0000","last_fullsized":"2023-10-09T06:43:16.086487+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":270,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.086335+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:43:16.086335+0000","last_peered":"2023-10-09T06:43:16.086335+0000","last_clean":"2023-10-09T06:43:16.086335+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:43:16.086335+0000","last_undegraded":"2023-10-09T06:43:16.086335+0000","last_fullsized":"2023-10-09T06:43:16.086335+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":518,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.969372+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:43:02.969372+0000","last_peered":"2023-10-09T06:43:02.969372+0000","last_clean":"2023-10-09T06:43:02.969372+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:43:02.969372+0000","last_undegraded":"2023-10-09T06:43:02.969372+0000","last_fullsized":"2023-10-09T06:43:02.969372+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":510,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971235+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:43:02.971235+0000","last_peered":"2023-10-09T06:43:02.971235+0000","last_clean":"2023-10-09T06:43:02.971235+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:43:02.971235+0000","last_undegraded":"2023-10-09T06:43:02.971235+0000","last_fullsized":"2023-10-09T06:43:02.971235+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"537'12356","reported_seq":12413,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.860099+0000","last_change":"2023-10-09T06:39:33.927651+0000","last_active":"2023-10-09T06:43:27.860099+0000","last_peered":"2023-10-09T06:43:27.860099+0000","last_clean":"2023-10-09T06:43:27.860099+0000","last_became_active":"2023-10-09T06:39:33.927357+0000","last_became_peered":"2023-10-09T06:39:33.927357+0000","last_unstale":"2023-10-09T06:43:27.860099+0000","last_undegraded":"2023-10-09T06:43:27.860099+0000","last_fullsized":"2023-10-09T06:43:27.860099+0000","mapping_epoch":489,"log_start":"509'5900","ondisk_log_start":"509'5900","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:01:02.824552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3604504,"num_objects":56,"num_object_clones":0,"num_object_copies":168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12356,"num_write_kb":397121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":879,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975247+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:43:02.975247+0000","last_peered":"2023-10-09T06:43:02.975247+0000","last_clean":"2023-10-09T06:43:02.975247+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:43:02.975247+0000","last_undegraded":"2023-10-09T06:43:02.975247+0000","last_fullsized":"2023-10-09T06:43:02.975247+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":532,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.080005+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:43:16.080005+0000","last_peered":"2023-10-09T06:43:16.080005+0000","last_clean":"2023-10-09T06:43:16.080005+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:43:16.080005+0000","last_undegraded":"2023-10-09T06:43:16.080005+0000","last_fullsized":"2023-10-09T06:43:16.080005+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":519,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:05.816547+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:43:05.816547+0000","last_peered":"2023-10-09T06:43:05.816547+0000","last_clean":"2023-10-09T06:43:05.816547+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:43:05.816547+0000","last_undegraded":"2023-10-09T06:43:05.816547+0000","last_fullsized":"2023-10-09T06:43:05.816547+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":541,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.089931+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:43:16.089931+0000","last_peered":"2023-10-09T06:43:16.089931+0000","last_clean":"2023-10-09T06:43:16.089931+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:43:16.089931+0000","last_undegraded":"2023-10-09T06:43:16.089931+0000","last_fullsized":"2023-10-09T06:43:16.089931+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"537'12744","reported_seq":12800,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.597698+0000","last_change":"2023-10-09T06:39:33.925153+0000","last_active":"2023-10-09T06:43:27.597698+0000","last_peered":"2023-10-09T06:43:27.597698+0000","last_clean":"2023-10-09T06:43:27.597698+0000","last_became_active":"2023-10-09T06:39:33.924904+0000","last_became_peered":"2023-10-09T06:39:33.924904+0000","last_unstale":"2023-10-09T06:43:27.597698+0000","last_undegraded":"2023-10-09T06:43:27.597698+0000","last_fullsized":"2023-10-09T06:43:27.597698+0000","mapping_epoch":489,"log_start":"510'6600","ondisk_log_start":"510'6600","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:21.898563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4587520,"num_objects":70,"num_object_clones":0,"num_object_copies":210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12744,"num_write_kb":410048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":346,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975399+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:43:02.975399+0000","last_peered":"2023-10-09T06:43:02.975399+0000","last_clean":"2023-10-09T06:43:02.975399+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:43:02.975399+0000","last_undegraded":"2023-10-09T06:43:02.975399+0000","last_fullsized":"2023-10-09T06:43:02.975399+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":860,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975045+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:43:02.975045+0000","last_peered":"2023-10-09T06:43:02.975045+0000","last_clean":"2023-10-09T06:43:02.975045+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:43:02.975045+0000","last_undegraded":"2023-10-09T06:43:02.975045+0000","last_fullsized":"2023-10-09T06:43:02.975045+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":250,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974567+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:43:02.974567+0000","last_peered":"2023-10-09T06:43:02.974567+0000","last_clean":"2023-10-09T06:43:02.974567+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:43:02.974567+0000","last_undegraded":"2023-10-09T06:43:02.974567+0000","last_fullsized":"2023-10-09T06:43:02.974567+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":499,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974425+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:43:02.974425+0000","last_peered":"2023-10-09T06:43:02.974425+0000","last_clean":"2023-10-09T06:43:02.974425+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:43:02.974425+0000","last_undegraded":"2023-10-09T06:43:02.974425+0000","last_fullsized":"2023-10-09T06:43:02.974425+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"537'12774","reported_seq":12832,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.985824+0000","last_change":"2023-10-09T06:39:33.919958+0000","last_active":"2023-10-09T06:43:28.985824+0000","last_peered":"2023-10-09T06:43:28.985824+0000","last_clean":"2023-10-09T06:43:28.985824+0000","last_became_active":"2023-10-09T06:39:33.919655+0000","last_became_peered":"2023-10-09T06:39:33.919655+0000","last_unstale":"2023-10-09T06:43:28.985824+0000","last_undegraded":"2023-10-09T06:43:28.985824+0000","last_fullsized":"2023-10-09T06:43:28.985824+0000","mapping_epoch":489,"log_start":"510'6600","ondisk_log_start":"510'6600","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:39:07.136388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12774,"num_write_kb":408768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":849,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.973702+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:43:02.973702+0000","last_peered":"2023-10-09T06:43:02.973702+0000","last_clean":"2023-10-09T06:43:02.973702+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:43:02.973702+0000","last_undegraded":"2023-10-09T06:43:02.973702+0000","last_fullsized":"2023-10-09T06:43:02.973702+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":400,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974835+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:43:02.974835+0000","last_peered":"2023-10-09T06:43:02.974835+0000","last_clean":"2023-10-09T06:43:02.974835+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:43:02.974835+0000","last_undegraded":"2023-10-09T06:43:02.974835+0000","last_fullsized":"2023-10-09T06:43:02.974835+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":214,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966094+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:43:02.966094+0000","last_peered":"2023-10-09T06:43:02.966094+0000","last_clean":"2023-10-09T06:43:02.966094+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:43:02.966094+0000","last_undegraded":"2023-10-09T06:43:02.966094+0000","last_fullsized":"2023-10-09T06:43:02.966094+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":525,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966419+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:43:02.966419+0000","last_peered":"2023-10-09T06:43:02.966419+0000","last_clean":"2023-10-09T06:43:02.966419+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:43:02.966419+0000","last_undegraded":"2023-10-09T06:43:02.966419+0000","last_fullsized":"2023-10-09T06:43:02.966419+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"537'8679","reported_seq":8807,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.975375+0000","last_change":"2023-10-09T06:40:48.261698+0000","last_active":"2023-10-09T06:43:28.975375+0000","last_peered":"2023-10-09T06:43:28.975375+0000","last_clean":"2023-10-09T06:43:28.975375+0000","last_became_active":"2023-10-09T06:40:43.090550+0000","last_became_peered":"2023-10-09T06:40:43.090550+0000","last_unstale":"2023-10-09T06:43:28.975375+0000","last_undegraded":"2023-10-09T06:43:28.975375+0000","last_fullsized":"2023-10-09T06:43:28.975375+0000","mapping_epoch":489,"log_start":"494'2043","ondisk_log_start":"494'2043","created":489,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"502'4775","last_scrub_stamp":"2023-10-09T06:40:48.261467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:40:48.261467+0000","objects_scrubbed":2451,"log_size":6636,"log_dups_size":0,"ondisk_log_size":6636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:45.927807+0000","scrub_duration":0.48199046000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3909,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":859,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975160+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:43:02.975160+0000","last_peered":"2023-10-09T06:43:02.975160+0000","last_clean":"2023-10-09T06:43:02.975160+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:43:02.975160+0000","last_undegraded":"2023-10-09T06:43:02.975160+0000","last_fullsized":"2023-10-09T06:43:02.975160+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":400,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976190+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:43:02.976190+0000","last_peered":"2023-10-09T06:43:02.976190+0000","last_clean":"2023-10-09T06:43:02.976190+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:43:02.976190+0000","last_undegraded":"2023-10-09T06:43:02.976190+0000","last_fullsized":"2023-10-09T06:43:02.976190+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":536,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971167+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:43:02.971167+0000","last_peered":"2023-10-09T06:43:02.971167+0000","last_clean":"2023-10-09T06:43:02.971167+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:43:02.971167+0000","last_undegraded":"2023-10-09T06:43:02.971167+0000","last_fullsized":"2023-10-09T06:43:02.971167+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":523,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967866+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:43:02.967866+0000","last_peered":"2023-10-09T06:43:02.967866+0000","last_clean":"2023-10-09T06:43:02.967866+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:43:02.967866+0000","last_undegraded":"2023-10-09T06:43:02.967866+0000","last_fullsized":"2023-10-09T06:43:02.967866+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"537'12540","reported_seq":12598,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:25.818554+0000","last_change":"2023-10-09T06:39:33.927683+0000","last_active":"2023-10-09T06:43:25.818554+0000","last_peered":"2023-10-09T06:43:25.818554+0000","last_clean":"2023-10-09T06:43:25.818554+0000","last_became_active":"2023-10-09T06:39:33.927389+0000","last_became_peered":"2023-10-09T06:39:33.927389+0000","last_unstale":"2023-10-09T06:43:25.818554+0000","last_undegraded":"2023-10-09T06:43:25.818554+0000","last_fullsized":"2023-10-09T06:43:25.818554+0000","mapping_epoch":489,"log_start":"511'7000","ondisk_log_start":"511'7000","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":5540,"log_dups_size":0,"ondisk_log_size":5540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:22:36.138757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10485760,"num_objects":160,"num_object_clones":0,"num_object_copies":480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12540,"num_write_kb":406400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":860,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971140+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:43:02.971140+0000","last_peered":"2023-10-09T06:43:02.971140+0000","last_clean":"2023-10-09T06:43:02.971140+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:43:02.971140+0000","last_undegraded":"2023-10-09T06:43:02.971140+0000","last_fullsized":"2023-10-09T06:43:02.971140+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":271,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975183+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:43:02.975183+0000","last_peered":"2023-10-09T06:43:02.975183+0000","last_clean":"2023-10-09T06:43:02.975183+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:43:02.975183+0000","last_undegraded":"2023-10-09T06:43:02.975183+0000","last_fullsized":"2023-10-09T06:43:02.975183+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":504,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:16.087822+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:43:16.087822+0000","last_peered":"2023-10-09T06:43:16.087822+0000","last_clean":"2023-10-09T06:43:16.087822+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:43:16.087822+0000","last_undegraded":"2023-10-09T06:43:16.087822+0000","last_fullsized":"2023-10-09T06:43:16.087822+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974203+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:43:02.974203+0000","last_peered":"2023-10-09T06:43:02.974203+0000","last_clean":"2023-10-09T06:43:02.974203+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:43:02.974203+0000","last_undegraded":"2023-10-09T06:43:02.974203+0000","last_fullsized":"2023-10-09T06:43:02.974203+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"537'12696","reported_seq":12754,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.963463+0000","last_change":"2023-10-09T06:39:33.926478+0000","last_active":"2023-10-09T06:43:28.963463+0000","last_peered":"2023-10-09T06:43:28.963463+0000","last_clean":"2023-10-09T06:43:28.963463+0000","last_became_active":"2023-10-09T06:39:33.926165+0000","last_became_peered":"2023-10-09T06:39:33.926165+0000","last_unstale":"2023-10-09T06:43:28.963463+0000","last_undegraded":"2023-10-09T06:43:28.963463+0000","last_fullsized":"2023-10-09T06:43:28.963463+0000","mapping_epoch":489,"log_start":"510'6500","ondisk_log_start":"510'6500","created":489,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:34:45.712982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12696,"num_write_kb":406272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":893,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.976270+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:43:02.976270+0000","last_peered":"2023-10-09T06:43:02.976270+0000","last_clean":"2023-10-09T06:43:02.976270+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:43:02.976270+0000","last_undegraded":"2023-10-09T06:43:02.976270+0000","last_fullsized":"2023-10-09T06:43:02.976270+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":521,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974973+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:43:02.974973+0000","last_peered":"2023-10-09T06:43:02.974973+0000","last_clean":"2023-10-09T06:43:02.974973+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:43:02.974973+0000","last_undegraded":"2023-10-09T06:43:02.974973+0000","last_fullsized":"2023-10-09T06:43:02.974973+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":506,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966270+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:43:02.966270+0000","last_peered":"2023-10-09T06:43:02.966270+0000","last_clean":"2023-10-09T06:43:02.966270+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:43:02.966270+0000","last_undegraded":"2023-10-09T06:43:02.966270+0000","last_fullsized":"2023-10-09T06:43:02.966270+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":500,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967742+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:43:02.967742+0000","last_peered":"2023-10-09T06:43:02.967742+0000","last_clean":"2023-10-09T06:43:02.967742+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:43:02.967742+0000","last_undegraded":"2023-10-09T06:43:02.967742+0000","last_fullsized":"2023-10-09T06:43:02.967742+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"537'8493","reported_seq":8560,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:28.896722+0000","last_change":"2023-10-09T06:40:43.188245+0000","last_active":"2023-10-09T06:43:28.896722+0000","last_peered":"2023-10-09T06:43:28.896722+0000","last_clean":"2023-10-09T06:43:28.896722+0000","last_became_active":"2023-10-09T06:40:43.186495+0000","last_became_peered":"2023-10-09T06:40:43.186495+0000","last_unstale":"2023-10-09T06:43:28.896722+0000","last_undegraded":"2023-10-09T06:43:28.896722+0000","last_fullsized":"2023-10-09T06:43:28.896722+0000","mapping_epoch":500,"log_start":"493'200","ondisk_log_start":"493'200","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8293,"log_dups_size":0,"ondisk_log_size":8293,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6213,"num_write_kb":198656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":819,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974675+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:43:02.974675+0000","last_peered":"2023-10-09T06:43:02.974675+0000","last_clean":"2023-10-09T06:43:02.974675+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:43:02.974675+0000","last_undegraded":"2023-10-09T06:43:02.974675+0000","last_fullsized":"2023-10-09T06:43:02.974675+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":507,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:14.916456+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:43:14.916456+0000","last_peered":"2023-10-09T06:43:14.916456+0000","last_clean":"2023-10-09T06:43:14.916456+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:43:14.916456+0000","last_undegraded":"2023-10-09T06:43:14.916456+0000","last_fullsized":"2023-10-09T06:43:14.916456+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":528,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971045+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:43:02.971045+0000","last_peered":"2023-10-09T06:43:02.971045+0000","last_clean":"2023-10-09T06:43:02.971045+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:43:02.971045+0000","last_undegraded":"2023-10-09T06:43:02.971045+0000","last_fullsized":"2023-10-09T06:43:02.971045+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"537'8476","reported_seq":16104,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:27.688044+0000","last_change":"2023-10-09T06:42:38.062683+0000","last_active":"2023-10-09T06:43:27.688044+0000","last_peered":"2023-10-09T06:43:27.688044+0000","last_clean":"2023-10-09T06:43:27.688044+0000","last_became_active":"2023-10-09T06:42:35.898459+0000","last_became_peered":"2023-10-09T06:42:35.898459+0000","last_unstale":"2023-10-09T06:43:27.688044+0000","last_undegraded":"2023-10-09T06:43:27.688044+0000","last_fullsized":"2023-10-09T06:43:27.688044+0000","mapping_epoch":525,"log_start":"494'1816","ondisk_log_start":"494'1816","created":500,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7448","last_scrub_stamp":"2023-10-09T06:42:38.062621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:38.062621+0000","objects_scrubbed":1070,"log_size":6660,"log_dups_size":0,"ondisk_log_size":6660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:34.705165+0000","scrub_duration":0.30123573999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293760,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1035,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":750,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.977376+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:43:02.977376+0000","last_peered":"2023-10-09T06:43:02.977376+0000","last_clean":"2023-10-09T06:43:02.977376+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:43:02.977376+0000","last_undegraded":"2023-10-09T06:43:02.977376+0000","last_fullsized":"2023-10-09T06:43:02.977376+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":215,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.970046+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:43:02.970046+0000","last_peered":"2023-10-09T06:43:02.970046+0000","last_clean":"2023-10-09T06:43:02.970046+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:43:02.970046+0000","last_undegraded":"2023-10-09T06:43:02.970046+0000","last_fullsized":"2023-10-09T06:43:02.970046+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":544,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.969291+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:43:02.969291+0000","last_peered":"2023-10-09T06:43:02.969291+0000","last_clean":"2023-10-09T06:43:02.969291+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:43:02.969291+0000","last_undegraded":"2023-10-09T06:43:02.969291+0000","last_fullsized":"2023-10-09T06:43:02.969291+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"537'8518","reported_seq":18748,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T06:43:25.699999+0000","last_change":"2023-10-09T06:42:36.563651+0000","last_active":"2023-10-09T06:43:25.699999+0000","last_peered":"2023-10-09T06:43:25.699999+0000","last_clean":"2023-10-09T06:43:25.699999+0000","last_became_active":"2023-10-09T06:41:58.457422+0000","last_became_peered":"2023-10-09T06:41:58.457422+0000","last_unstale":"2023-10-09T06:43:25.699999+0000","last_undegraded":"2023-10-09T06:43:25.699999+0000","last_fullsized":"2023-10-09T06:43:25.699999+0000","mapping_epoch":519,"log_start":"494'1585","ondisk_log_start":"494'1585","created":500,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":5,"last_scrub":"526'7520","last_scrub_stamp":"2023-10-09T06:42:36.563378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:42:36.563378+0000","objects_scrubbed":1096,"log_size":6933,"log_dups_size":0,"ondisk_log_size":6933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:03:24.101176+0000","scrub_duration":0.49466481600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6094848,"num_objects":93,"num_object_clones":0,"num_object_copies":279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1003,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":215,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971342+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:43:02.971342+0000","last_peered":"2023-10-09T06:43:02.971342+0000","last_clean":"2023-10-09T06:43:02.971342+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:43:02.971342+0000","last_undegraded":"2023-10-09T06:43:02.971342+0000","last_fullsized":"2023-10-09T06:43:02.971342+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":554,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967845+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:43:02.967845+0000","last_peered":"2023-10-09T06:43:02.967845+0000","last_clean":"2023-10-09T06:43:02.967845+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:43:02.967845+0000","last_undegraded":"2023-10-09T06:43:02.967845+0000","last_fullsized":"2023-10-09T06:43:02.967845+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"537'8683","reported_seq":12748,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T06:43:28.932684+0000","last_change":"2023-10-09T06:42:43.880894+0000","last_active":"2023-10-09T06:43:28.932684+0000","last_peered":"2023-10-09T06:43:28.932684+0000","last_clean":"2023-10-09T06:42:40.976884+0000","last_became_active":"2023-10-09T06:42:43.866227+0000","last_became_peered":"2023-10-09T06:42:43.866227+0000","last_unstale":"2023-10-09T06:43:28.932684+0000","last_undegraded":"2023-10-09T06:42:43.809722+0000","last_fullsized":"2023-10-09T06:42:43.686346+0000","mapping_epoch":531,"log_start":"493'528","ondisk_log_start":"493'528","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4378,"num_objects_misplaced":2189,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6363,"num_write_kb":203328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":998,"num_bytes_recovered":65404928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[0,2],"avail_no_missing":["0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":893,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.975912+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:43:02.975912+0000","last_peered":"2023-10-09T06:43:02.975912+0000","last_clean":"2023-10-09T06:43:02.975912+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:43:02.975912+0000","last_undegraded":"2023-10-09T06:43:02.975912+0000","last_fullsized":"2023-10-09T06:43:02.975912+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":241,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.977530+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:43:02.977530+0000","last_peered":"2023-10-09T06:43:02.977530+0000","last_clean":"2023-10-09T06:43:02.977530+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:43:02.977530+0000","last_undegraded":"2023-10-09T06:43:02.977530+0000","last_fullsized":"2023-10-09T06:43:02.977530+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":258,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.974895+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:43:02.974895+0000","last_peered":"2023-10-09T06:43:02.974895+0000","last_clean":"2023-10-09T06:43:02.974895+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:43:02.974895+0000","last_undegraded":"2023-10-09T06:43:02.974895+0000","last_fullsized":"2023-10-09T06:43:02.974895+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"537'8614","reported_seq":15236,"reported_epoch":537,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T06:43:25.840439+0000","last_change":"2023-10-09T06:43:25.840641+0000","last_active":"2023-10-09T06:43:25.840439+0000","last_peered":"2023-10-09T06:43:25.840439+0000","last_clean":"2023-10-09T06:42:39.926512+0000","last_became_active":"2023-10-09T06:42:42.144898+0000","last_became_peered":"2023-10-09T06:42:42.144898+0000","last_unstale":"2023-10-09T06:43:25.840439+0000","last_undegraded":"2023-10-09T06:43:25.840439+0000","last_fullsized":"2023-10-09T06:42:41.983721+0000","mapping_epoch":529,"log_start":"494'1678","ondisk_log_start":"494'1678","created":500,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T06:39:32.904263+0000","last_clean_scrub_stamp":"2023-10-09T06:39:32.904263+0000","objects_scrubbed":0,"log_size":6936,"log_dups_size":0,"ondisk_log_size":6936,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4096000,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6303,"num_write_kb":203680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2179,"num_bytes_recovered":66846720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,2","objects":63}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":661,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.971154+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:43:02.971154+0000","last_peered":"2023-10-09T06:43:02.971154+0000","last_clean":"2023-10-09T06:43:02.971154+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:43:02.971154+0000","last_undegraded":"2023-10-09T06:43:02.971154+0000","last_fullsized":"2023-10-09T06:43:02.971154+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":536,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.966667+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:43:02.966667+0000","last_peered":"2023-10-09T06:43:02.966667+0000","last_clean":"2023-10-09T06:43:02.966667+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:43:02.966667+0000","last_undegraded":"2023-10-09T06:43:02.966667+0000","last_fullsized":"2023-10-09T06:43:02.966667+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":497,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-09T06:43:02.967525+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:43:02.967525+0000","last_peered":"2023-10-09T06:43:02.967525+0000","last_clean":"2023-10-09T06:43:02.967525+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:43:02.967525+0000","last_undegraded":"2023-10-09T06:43:02.967525+0000","last_fullsized":"2023-10-09T06:43:02.967525+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":47415320,"num_objects":725,"num_object_clones":0,"num_object_copies":2175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7384,"num_objects_misplaced":2189,"num_objects_unfound":0,"num_objects_dirty":725,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163213,"num_write_kb":4557441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3177,"num_bytes_recovered":132251648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":586625024,"data_stored":586612808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":191682,"ondisk_log_size":191682,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234554,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578032,"kb_used_data":77032,"kb_used_omap":0,"kb_used_meta":1500992,"kb_avail":92167120,"statfs":{"total":95995035648,"available":94379130880,"internally_reserved":0,"allocated":78880768,"data_stored":77793880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537015808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:19 2023","interfaces":[{"interface":"back","average":{"1min":7.6639999999999997,"5min":3.3919999999999999,"15min":3.629},"min":{"1min":0.26700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":76.727999999999994,"5min":76.727999999999994,"15min":196.28899999999999},"last":2.2210000000000001},{"interface":"front","average":{"1min":7.8079999999999998,"5min":3.4239999999999999,"15min":3.6840000000000002},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":76.712999999999994,"5min":76.712999999999994,"15min":196.333},"last":1.623}]},{"osd":1,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.8149999999999999,"15min":5.069},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23200000000000001},"max":{"1min":7.9080000000000004,"5min":169.51900000000001,"15min":208.626},"last":2.77},{"interface":"front","average":{"1min":2.1160000000000001,"5min":3.7410000000000001,"15min":4.1139999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":6.5730000000000004,"5min":169.101,"15min":181.24199999999999},"last":2.2799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":5.5679999999999996,"15min":4.9640000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.8869999999999996,"5min":228.24000000000001,"15min":228.24000000000001},"last":1.4530000000000001},{"interface":"front","average":{"1min":3.1070000000000002,"5min":5.8949999999999996,"15min":5.0579999999999998},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":15.355,"5min":228.291,"15min":228.291},"last":2.3079999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":4.4500000000000002,"15min":4.8659999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.8740000000000001,"5min":88.897999999999996,"15min":187.34200000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.952,"5min":4.4160000000000004,"15min":4.8529999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":6.4509999999999996,"5min":88.926000000000002,"15min":187.08199999999999},"last":0.74299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.393000000000001,"5min":5.2069999999999999,"15min":2.9020000000000001},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":130.774,"5min":225.041,"15min":225.041},"last":2.4969999999999999},{"interface":"front","average":{"1min":11.032999999999999,"5min":5.0499999999999998,"15min":2.7450000000000001},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.72999999999999,"5min":225.244,"15min":225.244},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.8820000000000001,"15min":2.4060000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.60299999999999998,"5min":219.923,"15min":219.923},"last":4.8520000000000003},{"interface":"front","average":{"1min":0.82999999999999996,"5min":2.9460000000000002,"15min":2.3610000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":219.86000000000001,"15min":219.86000000000001},"last":2.2559999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.494999999999999,"5min":5.1429999999999998,"15min":3.2050000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.317},"max":{"1min":138.477,"5min":222.179,"15min":222.179},"last":2.3340000000000001},{"interface":"front","average":{"1min":11.484999999999999,"5min":5.04,"15min":3.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":135.244,"5min":223.261,"15min":223.261},"last":3.722}]}]},{"osd":6,"up_from":46,"seq":197568496190,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1600796,"kb_used_data":96468,"kb_used_omap":0,"kb_used_meta":1504320,"kb_avail":92144356,"statfs":{"total":95995035648,"available":94355820544,"internally_reserved":0,"allocated":98783232,"data_stored":97715570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1540423680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.8999999999999999,"15min":2.597},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":28.026,"5min":28.026,"15min":75.061999999999998},"last":3.0329999999999999},{"interface":"front","average":{"1min":3.2490000000000001,"5min":4.3979999999999997,"15min":3.6880000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":28.007000000000001,"5min":212.096,"15min":212.096},"last":2.363}]},{"osd":1,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":3.238,"15min":4.0549999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.20899999999999999},"max":{"1min":40.067,"5min":157.852,"15min":219.93199999999999},"last":4.0250000000000004},{"interface":"front","average":{"1min":3.0270000000000001,"5min":3.1600000000000001,"15min":4.0800000000000001},"min":{"1min":0.27800000000000002,"5min":0.222,"15min":0.222},"max":{"1min":40.085999999999999,"5min":157.87100000000001,"15min":219.94800000000001},"last":4.0410000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":3.0249999999999999,"15min":3.8140000000000001},"min":{"1min":0.28899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":5.7939999999999996,"5min":158,"15min":185.654},"last":3.79},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.9359999999999999,"15min":3.617},"min":{"1min":0.312,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.819,"5min":158.017,"15min":185.67699999999999},"last":3.8149999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:42:33 2023","interfaces":[{"interface":"back","average":{"1min":5.4859999999999998,"5min":5.2430000000000003,"15min":4.407},"min":{"1min":0.33200000000000002,"5min":0.29699999999999999,"15min":0.28299999999999997},"max":{"1min":80.168000000000006,"5min":161.60300000000001,"15min":161.60300000000001},"last":5.0129999999999999},{"interface":"front","average":{"1min":5.4390000000000001,"5min":5.0700000000000003,"15min":4.3380000000000001},"min":{"1min":0.371,"5min":0.32100000000000001,"15min":0.318},"max":{"1min":80.242999999999995,"5min":161.56299999999999,"15min":161.56299999999999},"last":3.1110000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.089,"15min":1.5269999999999999},"min":{"1min":0.30399999999999999,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":2.4060000000000001,"5min":10.009,"15min":86.171999999999997},"last":2.1760000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":3.0939999999999999,"15min":2.1749999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":1.8200000000000001,"5min":212.03399999999999,"15min":212.03399999999999},"last":3.427}]},{"osd":5,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.3560000000000001,"15min":1.3500000000000001},"min":{"1min":0.29799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2880000000000003,"5min":10.435,"15min":12.747999999999999},"last":4.1699999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.1579999999999999,"15min":1.522},"min":{"1min":0.308,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.175,"5min":9.0060000000000002,"15min":16.536999999999999},"last":4.3689999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:42:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.0329999999999999,"15min":1.7170000000000001},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8300000000000001,"5min":8.8059999999999992,"15min":165.37},"last":3.0859999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1779999999999999,"15min":2.056},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.3610000000000007,"5min":8.5980000000000008,"15min":165.40199999999999},"last":3.238}]}]},{"osd":1,"up_from":110,"seq":472446402927,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1280200,"kb_used_data":157632,"kb_used_omap":4,"kb_used_meta":1122555,"kb_avail":92464952,"statfs":{"total":95995035648,"available":94684110848,"internally_reserved":0,"allocated":161415168,"data_stored":159137916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1149497230},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.6719999999999997,"15min":5.4580000000000002},"min":{"1min":0.23200000000000001,"5min":0.182,"15min":0.182},"max":{"1min":114.97799999999999,"5min":114.97799999999999,"15min":269.79199999999997},"last":10.369},{"interface":"front","average":{"1min":10.832000000000001,"5min":6.5330000000000004,"15min":5.8339999999999996},"min":{"1min":0.21099999999999999,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":114.934,"5min":114.934,"15min":271.20999999999998},"last":8.8689999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.4139999999999999,"15min":2.2410000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.184},"max":{"1min":4.4699999999999998,"5min":8.0129999999999999,"15min":138.452},"last":0.36399999999999999},{"interface":"front","average":{"1min":4.016,"5min":2.8759999999999999,"15min":3.0659999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":20.085000000000001,"5min":20.085000000000001,"15min":138.47499999999999},"last":8.8940000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:38 2023","interfaces":[{"interface":"back","average":{"1min":3.548,"5min":2.9089999999999998,"15min":2.8959999999999999},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.219},"max":{"1min":27.655999999999999,"5min":27.655999999999999,"15min":31.062000000000001},"last":1.224},{"interface":"front","average":{"1min":3.177,"5min":1.9930000000000001,"15min":2.0270000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.22700000000000001},"max":{"1min":27.745999999999999,"5min":27.745999999999999,"15min":27.745999999999999},"last":10.332000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":3.7229999999999999,"15min":3.2930000000000001},"min":{"1min":0.378,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.9210000000000003,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.643,"5min":3.552,"15min":2.9700000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":4.3159999999999998,"5min":180.70599999999999,"15min":180.70599999999999},"last":8.9239999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.589,"15min":2.5790000000000002},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.6160000000000001,"5min":15.83,"15min":119.408},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.8560000000000001,"15min":2.835},"min":{"1min":0.376,"5min":0.312,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":7.9470000000000001,"15min":119.483},"last":9.6820000000000004}]},{"osd":6,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":8.7739999999999991,"5min":5.0730000000000004,"15min":3.9609999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":161.54900000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":9.4760000000000009},{"interface":"front","average":{"1min":8.8140000000000001,"5min":5.3399999999999999,"15min":4.3609999999999998},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":161.566,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.50700000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":3.5670000000000002,"15min":4.0659999999999998},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":4.8890000000000002,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.2290000000000001,"5min":2.9220000000000002,"15min":3.4119999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":5.3419999999999996,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.58499999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024361,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186236,"kb_used_data":7156,"kb_used_omap":5,"kb_used_meta":1179066,"kb_avail":92558916,"statfs":{"total":95995035648,"available":94780329984,"internally_reserved":0,"allocated":7327744,"data_stored":4656738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1207364218},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:42:51 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":8.0039999999999996,"15min":4.3869999999999996},"min":{"1min":0.22500000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":13.387,"5min":201.381,"15min":201.381},"last":4.1449999999999996},{"interface":"front","average":{"1min":2.1589999999999998,"5min":7.5019999999999998,"15min":4.609},"min":{"1min":0.24399999999999999,"5min":0.193,"15min":0.187},"max":{"1min":8.3219999999999992,"5min":201.44200000000001,"15min":201.44200000000001},"last":3.9609999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":11.029,"5min":4.5430000000000001,"15min":4.0030000000000001},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":147.61099999999999,"5min":147.61099999999999,"15min":147.61099999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":11.228999999999999,"5min":5.024,"15min":4.415},"min":{"1min":0.30199999999999999,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":150.12700000000001,"5min":150.12700000000001,"15min":150.12700000000001},"last":3.7189999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.9929999999999999,"15min":4.2050000000000001},"min":{"1min":0.28599999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.4809999999999999,"5min":95.555000000000007,"15min":187.172},"last":48.216000000000001},{"interface":"front","average":{"1min":2.3580000000000001,"5min":3.0579999999999998,"15min":4.1619999999999999},"min":{"1min":0.32700000000000001,"5min":0.22900000000000001,"15min":0.19800000000000001},"max":{"1min":9.0839999999999996,"5min":97.605000000000004,"15min":194.851},"last":48.261000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":2.048,"15min":3.1400000000000001},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":4.1029999999999998,"5min":14.183,"15min":200.654},"last":4.0709999999999997},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.966,"15min":3.5190000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":7.8339999999999996,"5min":8.2850000000000001,"15min":200.86600000000001},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":2.1389999999999998,"15min":2.1560000000000001},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":4.1870000000000003,"5min":14.263,"15min":80.885999999999996},"last":4.0309999999999997},{"interface":"front","average":{"1min":2.048,"5min":1.722,"15min":2.2629999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":8.3689999999999998,"5min":8.4019999999999992,"15min":80.438999999999993},"last":0.42999999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":1.853,"15min":2.8799999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3290000000000006,"5min":8.4309999999999992,"15min":118.056},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.603,"5min":2.0339999999999998,"15min":2.6360000000000001},"min":{"1min":0.42699999999999999,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":4.1580000000000004,"5min":14.308999999999999,"15min":118.274},"last":4.2370000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.784,"15min":2.165},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.742,"5min":8.2490000000000006,"15min":22.879000000000001},"last":3.9249999999999998},{"interface":"front","average":{"1min":2.145,"5min":2.254,"15min":2.5299999999999998},"min":{"1min":0.44500000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.3960000000000008,"5min":9.9190000000000005,"15min":26.416},"last":0.48499999999999999}]}]},{"osd":2,"up_from":128,"seq":549755814229,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129544,"kb_used_data":76032,"kb_used_omap":7,"kb_used_meta":1053368,"kb_avail":92615608,"statfs":{"total":95995035648,"available":94838382592,"internally_reserved":0,"allocated":77856768,"data_stored":75637179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1078648913},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":10.237,"5min":7.4219999999999997,"15min":7.1289999999999996},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":89.287000000000006,"5min":214.595,"15min":214.595},"last":6.6299999999999999},{"interface":"front","average":{"1min":9.4120000000000008,"5min":6.9500000000000002,"15min":6.7539999999999996},"min":{"1min":0.251,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":89.225999999999999,"5min":214.94300000000001,"15min":214.94300000000001},"last":6.6959999999999997}]},{"osd":1,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":27.795000000000002,"5min":7.0369999999999999,"15min":6.0309999999999997},"min":{"1min":0.308,"5min":0.20499999999999999,"15min":0.20200000000000001},"max":{"1min":299.988,"5min":299.988,"15min":299.988},"last":0.79400000000000004},{"interface":"front","average":{"1min":28.745000000000001,"5min":7.2080000000000002,"15min":6.2800000000000002},"min":{"1min":0.25900000000000001,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":300.30500000000001,"5min":300.30500000000001,"15min":300.30500000000001},"last":3.2280000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:42:37 2023","interfaces":[{"interface":"back","average":{"1min":5.7050000000000001,"5min":3.5979999999999999,"15min":3.7909999999999999},"min":{"1min":0.30099999999999999,"5min":0.23200000000000001,"15min":0.20599999999999999},"max":{"1min":37.965000000000003,"5min":79.566000000000003,"15min":91.644000000000005},"last":2.093},{"interface":"front","average":{"1min":5.5890000000000004,"5min":3.5419999999999998,"15min":4.0739999999999998},"min":{"1min":0.27800000000000002,"5min":0.27700000000000002,"15min":0.26700000000000002},"max":{"1min":37.990000000000002,"5min":79.614000000000004,"15min":91.685000000000002},"last":1.216}]},{"osd":4,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.99099999999999999,"15min":2.2810000000000001},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.9299999999999997,"5min":9.5679999999999996,"15min":79.643000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.198,"15min":2.5539999999999998},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.0910000000000002,"5min":7.0910000000000002,"15min":82.872},"last":1.365}]},{"osd":5,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.107,"15min":2.3210000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":5.9809999999999999,"5min":9.7040000000000006,"15min":82.834999999999994},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.548,"5min":1.268,"15min":2.512},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":7.1369999999999996,"5min":7.1369999999999996,"15min":82.921999999999997},"last":1.4059999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.403,"15min":3.2250000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.313},"max":{"1min":7.1669999999999998,"5min":7.1669999999999998,"15min":123.47799999999999},"last":1.4450000000000001},{"interface":"front","average":{"1min":1.28,"5min":1.4950000000000001,"15min":3.1960000000000002},"min":{"1min":0.377,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":5.2359999999999998,"5min":9.3740000000000006,"15min":122.517},"last":0.628}]},{"osd":7,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.8210000000000002,"15min":3.2989999999999999},"min":{"1min":0.40699999999999997,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":5.2770000000000001,"5min":132.06399999999999,"15min":132.06399999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.198,"5min":2.6429999999999998,"15min":2.9399999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.34999999999999998},"max":{"1min":6.0099999999999998,"5min":132.196,"15min":132.196},"last":3.9990000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127427,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1105972,"kb_used_data":195244,"kb_used_omap":2,"kb_used_meta":910717,"kb_avail":92639180,"statfs":{"total":95995035648,"available":94862520320,"internally_reserved":0,"allocated":199929856,"data_stored":197846490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":932574261},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6050000000000004,"5min":4.0170000000000003,"15min":4.4989999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19},"max":{"1min":95.281999999999996,"5min":95.281999999999996,"15min":161.821},"last":2.883},{"interface":"front","average":{"1min":9.0289999999999999,"5min":4.3010000000000002,"15min":4.4269999999999996},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":98.572000000000003,"5min":98.572000000000003,"15min":163.52000000000001},"last":2.9990000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.8460000000000001,"5min":4.0039999999999996,"15min":4.4359999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":102.468,"5min":102.468,"15min":184.012},"last":15.465999999999999},{"interface":"front","average":{"1min":9.3949999999999996,"5min":4.399,"15min":5.2839999999999998},"min":{"1min":0.41399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":102.58,"5min":102.58,"15min":184.703},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":2.9449999999999998,"15min":3.9870000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.9900000000000002,"5min":118.93899999999999,"15min":182.44900000000001},"last":2.661},{"interface":"front","average":{"1min":1.6830000000000001,"5min":3.1070000000000002,"15min":4.3250000000000002},"min":{"1min":0.22,"5min":0.219,"15min":0.219},"max":{"1min":14.9,"5min":118.97199999999999,"15min":182.56399999999999},"last":0.69299999999999995}]},{"osd":4,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.0329999999999999,"15min":3.3879999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.7390000000000008,"5min":133.80099999999999,"15min":178.22200000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.444,"5min":3.2650000000000001,"15min":3.4060000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7999999999999998,"5min":133.648,"15min":180.43600000000001},"last":2.819}]},{"osd":5,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.181,"15min":3.6440000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34000000000000002},"max":{"1min":7.8380000000000001,"5min":128.083,"15min":177.56299999999999},"last":2.7829999999999999},{"interface":"front","average":{"1min":2.2170000000000001,"5min":3.0569999999999999,"15min":3.0920000000000001},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":8.0570000000000004,"5min":128.11799999999999,"15min":177.59399999999999},"last":2.7069999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.895,"15min":2.7410000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3710000000000004,"5min":10.505000000000001,"15min":130.39599999999999},"last":2.931},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6379999999999999,"15min":2.4279999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.5019999999999998,"5min":7.5949999999999998,"15min":130.67699999999999},"last":2.7429999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.4470000000000001,"15min":3.0619999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.6850000000000005,"5min":99.248000000000005,"15min":121.613},"last":0.371},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.6299999999999999,"15min":3.4569999999999999},"min":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":3.2690000000000001,"5min":99.457999999999998,"15min":121.559},"last":2.8450000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019463,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540708,"kb_used_data":57052,"kb_used_omap":0,"kb_used_meta":1483648,"kb_avail":92204444,"statfs":{"total":95995035648,"available":94417350656,"internally_reserved":0,"allocated":58421248,"data_stored":57345771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519255552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":7.3730000000000002,"5min":4.1280000000000001,"15min":6.9119999999999999},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":68.421999999999997,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.371},{"interface":"front","average":{"1min":7.258,"5min":4.3010000000000002,"15min":7.1159999999999997},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.23100000000000001},"max":{"1min":68.433000000000007,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.30099999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":4.0229999999999997,"15min":4.2130000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.4320000000000004,"5min":163.392,"15min":163.392},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.6679999999999999,"5min":4.1260000000000003,"15min":4.3019999999999996},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":5.3970000000000002,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":4.234,"15min":4.391},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.6879999999999997,"5min":175.97900000000001,"15min":175.97900000000001},"last":1.421},{"interface":"front","average":{"1min":1.7450000000000001,"5min":4.2599999999999998,"15min":4.3609999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.0110000000000001,"5min":175.875,"15min":175.875},"last":0.46200000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5169999999999999,"5min":4.8929999999999998,"15min":6.1379999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":29.042999999999999,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":3.2509999999999999,"5min":4.7560000000000002,"15min":5.9329999999999998},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.28799999999999998},"max":{"1min":29.192,"5min":112.21899999999999,"15min":161.934},"last":0.68300000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.77400000000000002,"15min":0.95899999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.252},"max":{"1min":6.7869999999999999,"5min":6.7869999999999999,"15min":9.3979999999999997},"last":1.881},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.91600000000000004,"15min":1.1180000000000001},"min":{"1min":0.30199999999999999,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.9820000000000002,"5min":5.9820000000000002,"15min":9.5009999999999994},"last":0.42399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.651,"15min":2.2959999999999998},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":5.3819999999999997,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.087,"5min":1.708,"15min":2.4969999999999999},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":6.734,"5min":83.218000000000004,"15min":163.381},"last":2.2200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:42:31 2023","interfaces":[{"interface":"back","average":{"1min":4.984,"5min":1.6930000000000001,"15min":1.9379999999999999},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.312},"max":{"1min":76.081000000000003,"5min":76.081000000000003,"15min":106.801},"last":0.56399999999999995},{"interface":"front","average":{"1min":4.8639999999999999,"5min":1.6879999999999999,"15min":1.9730000000000001},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":76.283000000000001,"5min":76.283000000000001,"15min":106.827},"last":0.505}]}]},{"osd":5,"up_from":38,"seq":163208757828,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1517084,"kb_used_data":33492,"kb_used_omap":0,"kb_used_meta":1483584,"kb_avail":92228068,"statfs":{"total":95995035648,"available":94441541632,"internally_reserved":0,"allocated":34295808,"data_stored":33224775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519190016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:21 2023","interfaces":[{"interface":"back","average":{"1min":5.4130000000000003,"5min":3.468,"15min":5.9489999999999998},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":62.884999999999998,"5min":66.799000000000007,"15min":294.89699999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":5.452,"5min":3.7549999999999999,"15min":6.1589999999999998},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.215},"max":{"1min":62.195,"5min":66.614000000000004,"15min":294.30399999999997},"last":2.1389999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:43:07 2023","interfaces":[{"interface":"back","average":{"1min":5.0510000000000002,"5min":7.8769999999999998,"15min":7.0119999999999996},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.22},"max":{"1min":69.287999999999997,"5min":153.91300000000001,"15min":188.34100000000001},"last":1.3819999999999999},{"interface":"front","average":{"1min":4.4950000000000001,"5min":7.8289999999999997,"15min":7.0359999999999996},"min":{"1min":0.27400000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":67.245999999999995,"5min":154.023,"15min":188.839},"last":0.53200000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:43:07 2023","interfaces":[{"interface":"back","average":{"1min":12.33,"5min":6.3639999999999999,"15min":4.7140000000000004},"min":{"1min":0.39100000000000001,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":145.20400000000001,"5min":145.20400000000001,"15min":145.20400000000001},"last":2.2650000000000001},{"interface":"front","average":{"1min":12.356999999999999,"5min":6.5720000000000001,"15min":4.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":145.71899999999999,"5min":145.71899999999999,"15min":145.71899999999999},"last":3.5019999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:43:22 2023","interfaces":[{"interface":"back","average":{"1min":6.8680000000000003,"5min":5.827,"15min":6.1989999999999998},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":63.578000000000003,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.311},{"interface":"front","average":{"1min":5.2649999999999997,"5min":5.7590000000000003,"15min":5.8719999999999999},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":62.113999999999997,"5min":220.45500000000001,"15min":220.45500000000001},"last":1.992}]},{"osd":4,"last update":"Mon Oct 9 06:42:56 2023","interfaces":[{"interface":"back","average":{"1min":5.2729999999999997,"5min":1.7669999999999999,"15min":1.696},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.247},"max":{"1min":74.725999999999999,"5min":74.725999999999999,"15min":79.043000000000006},"last":0.95599999999999996},{"interface":"front","average":{"1min":5.2530000000000001,"5min":1.968,"15min":1.72},"min":{"1min":0.318,"5min":0.251,"15min":0.251},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":79.165999999999997},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":2.0339999999999998,"15min":1.867},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.251},"max":{"1min":4.492,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.62},{"interface":"front","average":{"1min":1.165,"5min":1.8380000000000001,"15min":1.958},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.9690000000000003,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.56499999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:42:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1719999999999999,"15min":1.577},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":4.718,"5min":11.146000000000001,"15min":78.890000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.1180000000000001,"15min":1.4039999999999999},"min":{"1min":0.379,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":2.7069999999999999,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.79700000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":142802944,"data_stored":142802944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":52428800,"data_stored":52428800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":191692800,"data_stored":191692800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":29429760,"data_stored":29425688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":18747392,"data_stored":18743320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":80416768,"data_stored":80412696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":71106560,"data_stored":71106560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:43:33.781 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:33.881 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:33.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:33.987 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:34.013 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:34.056 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:34.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:34.062 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:34.161 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:34.204 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:34.342 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:34.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.344 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[212750]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[212750]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[220921]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[220921]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[220921]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.345 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:34 smithi107 ceph-mon[220921]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:34.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:34 smithi150 ceph-mon[194178]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:34 smithi150 ceph-mon[194178]: osdmap e540: 8 total, 8 up, 8 in 2023-10-09T06:43:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:34 smithi150 ceph-mon[194178]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:34 smithi150 ceph-mon[194178]: failed to encode map e540 with expected crc 2023-10-09T06:43:34.447 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:34.478 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:34.482 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:43:34.508 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:43:34.529 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:43:34.550 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:43:34.568 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:43:34.587 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:43:34.606 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:43:34.625 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:43:34.626 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.626 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:43:34.627 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:43:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:35 smithi107 ceph-mon[212750]: pgmap v1674: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 7384/309 objects degraded (2389.644%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 142 objects/s recovering 2023-10-09T06:43:35.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1006721546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:35.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:35 smithi107 ceph-mon[220921]: pgmap v1674: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 7384/309 objects degraded (2389.644%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 142 objects/s recovering 2023-10-09T06:43:35.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1006721546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:35 smithi150 ceph-mon[194178]: pgmap v1674: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 7384/309 objects degraded (2389.644%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 142 objects/s recovering 2023-10-09T06:43:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1006721546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:35.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:43:36.781 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:43:37.107 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:37.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:37.213 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:37.239 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:37.282 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:37.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:37.387 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:37.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:37 smithi107 ceph-mon[212750]: pgmap v1675: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 237 objects/s recovering 2023-10-09T06:43:37.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:37 smithi107 ceph-mon[220921]: pgmap v1675: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 237 objects/s recovering 2023-10-09T06:43:37.413 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:37.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:37 smithi150 ceph-mon[194178]: pgmap v1675: 129 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 949 KiB/s, 237 objects/s recovering 2023-10-09T06:43:37.813 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:37.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:37.918 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:37.944 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:37.987 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:37.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:38.092 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:38.118 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:38.162 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:38.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:38.265 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:38.292 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:38.336 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:38.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:38.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:38.441 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:38.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:38 smithi150 ceph-mon[194178]: Health check update: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:38.467 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:38.512 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:38.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:38.618 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:38.644 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:38 smithi107 ceph-mon[212750]: Health check update: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:38.645 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:38 smithi107 ceph-mon[220921]: Health check update: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T06:43:38.648 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:39.199 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:39.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:39.303 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:39.330 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:39.336 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:39.374 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:39.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:39.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:39 smithi107 ceph-mon[212750]: pgmap v1676: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 211 objects/s recovering 2023-10-09T06:43:39.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:39 smithi107 ceph-mon[220921]: pgmap v1676: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 211 objects/s recovering 2023-10-09T06:43:39.435 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:43:39.435 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:43:39.455 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:39.479 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:39.515 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:39.569 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:39.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:39.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:39 smithi150 ceph-mon[194178]: pgmap v1676: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 211 objects/s recovering 2023-10-09T06:43:39.675 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:39.704 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:40.258 INFO:teuthology.orchestra.run.smithi107.stderr:pool 'unique_pool_9' removed 2023-10-09T06:43:40.306 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:40.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:40.412 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:40.443 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:40.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:43:40.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:43:40.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T06:43:40.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:43:41.276 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:41.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:41.381 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:41.383 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:43:41.413 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:41.458 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: pgmap v1677: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 41 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 265 objects/s recovering 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:43:41.465 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:41 smithi150 ceph-mon[194178]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:41.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:41.567 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:41.595 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:41.640 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:41.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: pgmap v1677: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 41 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 265 objects/s recovering 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:43:41.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[212750]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: pgmap v1677: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 127 active+clean; 5.5 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 41 op/s; 5275/309 objects degraded (1707.120%); 2189/309 objects misplaced (708.414%); 0 B/s, 265 objects/s recovering 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2482736846' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: osdmap e541: 8 total, 8 up, 8 in 2023-10-09T06:43:41.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:41 smithi107 ceph-mon[220921]: failed to encode map e541 with expected crc 2023-10-09T06:43:41.746 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:41.772 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:41.814 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:41.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:41.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:41.952 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:42.261 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:42] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-10-09T06:43:42.422 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:43:42.422 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T06:43:42.422 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:43:42.426 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:43:42.426 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:43:42.427 DEBUG:teuthology.orchestra.run.smithi107:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:43:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:43:42.569 INFO:teuthology.orchestra.run.smithi107.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:43:42.570 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:43:42.628 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:43:42.628 DEBUG:teuthology.orchestra.run.smithi107:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:43:42.645 INFO:tasks.workunit:timeout=3h 2023-10-09T06:43:42.646 INFO:tasks.workunit:cleanup=True 2023-10-09T06:43:42.646 DEBUG:teuthology.orchestra.run.smithi107:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:43:42.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:42 smithi107 ceph-mon[212750]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 2 pgs degraded) 2023-10-09T06:43:42.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:42 smithi107 ceph-mon[220921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 2 pgs degraded) 2023-10-09T06:43:42.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:42 smithi150 ceph-mon[194178]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5275/309 objects degraded (1707.120%), 2 pgs degraded) 2023-10-09T06:43:42.657 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:42.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:42.761 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:42.788 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:42.834 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:42.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:42.861 INFO:tasks.workunit.client.0.smithi107.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:43:42.939 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:42.966 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:43.163 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:43.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:43.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:43 smithi107 ceph-mon[212750]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:43.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:43 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:43.263 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:43 smithi107 ceph-mon[220921]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:43.263 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:43 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:43.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:43.292 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:43.351 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:43.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:43.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:43.454 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:43.482 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:43.627 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:43.634 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:43 smithi150 ceph-mon[194178]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:43.634 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:43 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:43:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:43.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:43.678 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:43.679 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:43.736 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:43.764 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:43.808 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:43.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:43.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:43.949 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:44.064 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:44.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:44.169 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:44.176 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:43:44.178 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:43:44.200 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:44.275 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:43:44.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:44 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:44.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:44 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:44.510 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:44.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:44.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:44.615 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:44.641 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:44.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:44 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[212750]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:43:45.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: failed to encode map e542 with expected crc 2023-10-09T06:43:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:45.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:45 smithi107 ceph-mon[220921]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:43:45.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:43:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:43:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: failed to encode map e542 with expected crc 2023-10-09T06:43:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/330930496' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:43:45.655 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:45 smithi150 ceph-mon[194178]: osdmap e542: 8 total, 8 up, 8 in 2023-10-09T06:43:46.065 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:46.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:46.169 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:46.175 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1680,"stamp":"2023-10-09T06:43:43.668726+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5046,"ondisk_log_size":5046,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10656124,"kb_used_data":381692,"kb_used_omap":20,"kb_used_meta":10274347,"kb_avail":739305092,"statfs":{"total":767960285184,"available":757048414208,"internally_reserved":0,"allocated":390852608,"data_stored":377229381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10520931982},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":50,"apply_latency_ms":50,"commit_latency_ns":50000000,"apply_latency_ns":50000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-5177344,"num_objects":-79,"num_object_clones":0,"num_object_copies":-237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-7384,"num_objects_misplaced":-2189,"num_objects_unfound":0,"num_objects_dirty":-79,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-163869,"num_write_kb":-4557441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3943,"num_bytes_recovered":-132251648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002716"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":835,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.388793+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:43:33.388793+0000","last_peered":"2023-10-09T06:43:33.388793+0000","last_clean":"2023-10-09T06:43:33.388793+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:43:33.388793+0000","last_undegraded":"2023-10-09T06:43:33.388793+0000","last_fullsized":"2023-10-09T06:43:33.388793+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":520,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185630+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:43:33.185630+0000","last_peered":"2023-10-09T06:43:33.185630+0000","last_clean":"2023-10-09T06:43:33.185630+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:43:33.185630+0000","last_undegraded":"2023-10-09T06:43:33.185630+0000","last_fullsized":"2023-10-09T06:43:33.185630+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178076+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:43:32.178076+0000","last_peered":"2023-10-09T06:43:32.178076+0000","last_clean":"2023-10-09T06:43:32.178076+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:43:32.178076+0000","last_undegraded":"2023-10-09T06:43:32.178076+0000","last_fullsized":"2023-10-09T06:43:32.178076+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":796,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185595+0000","last_change":"2023-10-09T06:39:27.204332+0000","last_active":"2023-10-09T06:43:33.185595+0000","last_peered":"2023-10-09T06:43:33.185595+0000","last_clean":"2023-10-09T06:43:33.185595+0000","last_became_active":"2023-10-09T06:37:08.296329+0000","last_became_peered":"2023-10-09T06:37:08.296329+0000","last_unstale":"2023-10-09T06:43:33.185595+0000","last_undegraded":"2023-10-09T06:43:33.185595+0000","last_fullsized":"2023-10-09T06:43:33.185595+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.593473+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":538,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185977+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:43:33.185977+0000","last_peered":"2023-10-09T06:43:33.185977+0000","last_clean":"2023-10-09T06:43:33.185977+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:43:33.185977+0000","last_undegraded":"2023-10-09T06:43:33.185977+0000","last_fullsized":"2023-10-09T06:43:33.185977+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":542,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170743+0000","last_change":"2023-10-09T06:29:15.540951+0000","last_active":"2023-10-09T06:43:32.170743+0000","last_peered":"2023-10-09T06:43:32.170743+0000","last_clean":"2023-10-09T06:43:32.170743+0000","last_became_active":"2023-10-09T06:29:15.540510+0000","last_became_peered":"2023-10-09T06:29:15.540510+0000","last_unstale":"2023-10-09T06:43:32.170743+0000","last_undegraded":"2023-10-09T06:43:32.170743+0000","last_fullsized":"2023-10-09T06:43:32.170743+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:22:03.248889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":788,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185514+0000","last_change":"2023-10-09T06:32:26.333293+0000","last_active":"2023-10-09T06:43:33.185514+0000","last_peered":"2023-10-09T06:43:33.185514+0000","last_clean":"2023-10-09T06:43:33.185514+0000","last_became_active":"2023-10-09T06:32:19.741074+0000","last_became_peered":"2023-10-09T06:32:19.741074+0000","last_unstale":"2023-10-09T06:43:33.185514+0000","last_undegraded":"2023-10-09T06:43:33.185514+0000","last_fullsized":"2023-10-09T06:43:33.185514+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.333195+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:39.738513+0000","scrub_duration":0.012659137000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":261,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176218+0000","last_change":"2023-10-09T06:29:15.537800+0000","last_active":"2023-10-09T06:43:32.176218+0000","last_peered":"2023-10-09T06:43:32.176218+0000","last_clean":"2023-10-09T06:43:32.176218+0000","last_became_active":"2023-10-09T06:29:15.537519+0000","last_became_peered":"2023-10-09T06:29:15.537519+0000","last_unstale":"2023-10-09T06:43:32.176218+0000","last_undegraded":"2023-10-09T06:43:32.176218+0000","last_fullsized":"2023-10-09T06:43:32.176218+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:31.532136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":530,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.208031+0000","last_change":"2023-10-09T06:28:11.701800+0000","last_active":"2023-10-09T06:43:33.208031+0000","last_peered":"2023-10-09T06:43:33.208031+0000","last_clean":"2023-10-09T06:43:33.208031+0000","last_became_active":"2023-10-09T06:28:11.701471+0000","last_became_peered":"2023-10-09T06:28:11.701471+0000","last_unstale":"2023-10-09T06:43:33.208031+0000","last_undegraded":"2023-10-09T06:43:33.208031+0000","last_fullsized":"2023-10-09T06:43:33.208031+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:23:11.342832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":649,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390042+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:43:33.390042+0000","last_peered":"2023-10-09T06:43:33.390042+0000","last_clean":"2023-10-09T06:43:33.390042+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:43:33.390042+0000","last_undegraded":"2023-10-09T06:43:33.390042+0000","last_fullsized":"2023-10-09T06:43:33.390042+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170611+0000","last_change":"2023-10-09T06:29:15.536387+0000","last_active":"2023-10-09T06:43:32.170611+0000","last_peered":"2023-10-09T06:43:32.170611+0000","last_clean":"2023-10-09T06:43:32.170611+0000","last_became_active":"2023-10-09T06:29:15.536073+0000","last_became_peered":"2023-10-09T06:29:15.536073+0000","last_unstale":"2023-10-09T06:43:32.170611+0000","last_undegraded":"2023-10-09T06:43:32.170611+0000","last_fullsized":"2023-10-09T06:43:32.170611+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:04.447401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176445+0000","last_change":"2023-10-09T06:27:36.676917+0000","last_active":"2023-10-09T06:43:32.176445+0000","last_peered":"2023-10-09T06:43:32.176445+0000","last_clean":"2023-10-09T06:43:32.176445+0000","last_became_active":"2023-10-09T06:27:36.676520+0000","last_became_peered":"2023-10-09T06:27:36.676520+0000","last_unstale":"2023-10-09T06:43:32.176445+0000","last_undegraded":"2023-10-09T06:43:32.176445+0000","last_fullsized":"2023-10-09T06:43:32.176445+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:58.219529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":686,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390192+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:43:33.390192+0000","last_peered":"2023-10-09T06:43:33.390192+0000","last_clean":"2023-10-09T06:43:33.390192+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:43:33.390192+0000","last_undegraded":"2023-10-09T06:43:33.390192+0000","last_fullsized":"2023-10-09T06:43:33.390192+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389333+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:43:33.389333+0000","last_peered":"2023-10-09T06:43:33.389333+0000","last_clean":"2023-10-09T06:43:33.389333+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:43:33.389333+0000","last_undegraded":"2023-10-09T06:43:33.389333+0000","last_fullsized":"2023-10-09T06:43:33.389333+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.177259+0000","last_change":"2023-10-09T06:27:36.666275+0000","last_active":"2023-10-09T06:43:32.177259+0000","last_peered":"2023-10-09T06:43:32.177259+0000","last_clean":"2023-10-09T06:43:32.177259+0000","last_became_active":"2023-10-09T06:27:36.664955+0000","last_became_peered":"2023-10-09T06:27:36.664955+0000","last_unstale":"2023-10-09T06:43:32.177259+0000","last_undegraded":"2023-10-09T06:43:32.177259+0000","last_fullsized":"2023-10-09T06:43:32.177259+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:38.445250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":669,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170443+0000","last_change":"2023-10-09T06:39:30.150976+0000","last_active":"2023-10-09T06:43:32.170443+0000","last_peered":"2023-10-09T06:43:32.170443+0000","last_clean":"2023-10-09T06:43:32.170443+0000","last_became_active":"2023-10-09T06:37:08.331531+0000","last_became_peered":"2023-10-09T06:37:08.331531+0000","last_unstale":"2023-10-09T06:43:32.170443+0000","last_undegraded":"2023-10-09T06:43:32.170443+0000","last_fullsized":"2023-10-09T06:43:32.170443+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:07.337313+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":422,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390399+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:43:33.390399+0000","last_peered":"2023-10-09T06:43:33.390399+0000","last_clean":"2023-10-09T06:43:33.390399+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:43:33.390399+0000","last_undegraded":"2023-10-09T06:43:33.390399+0000","last_fullsized":"2023-10-09T06:43:33.390399+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":480,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186845+0000","last_change":"2023-10-09T06:15:11.869954+0000","last_active":"2023-10-09T06:43:33.186845+0000","last_peered":"2023-10-09T06:43:33.186845+0000","last_clean":"2023-10-09T06:43:33.186845+0000","last_became_active":"2023-10-09T06:15:11.869677+0000","last_became_peered":"2023-10-09T06:15:11.869677+0000","last_unstale":"2023-10-09T06:43:33.186845+0000","last_undegraded":"2023-10-09T06:43:33.186845+0000","last_fullsized":"2023-10-09T06:43:33.186845+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:10:53.623093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":651,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186731+0000","last_change":"2023-10-09T06:32:26.181941+0000","last_active":"2023-10-09T06:43:33.186731+0000","last_peered":"2023-10-09T06:43:33.186731+0000","last_clean":"2023-10-09T06:43:33.186731+0000","last_became_active":"2023-10-09T06:32:19.746962+0000","last_became_peered":"2023-10-09T06:32:19.746962+0000","last_unstale":"2023-10-09T06:43:33.186731+0000","last_undegraded":"2023-10-09T06:43:33.186731+0000","last_fullsized":"2023-10-09T06:43:33.186731+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:52:10.578076+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":547,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186072+0000","last_change":"2023-10-09T06:19:06.712070+0000","last_active":"2023-10-09T06:43:33.186072+0000","last_peered":"2023-10-09T06:43:33.186072+0000","last_clean":"2023-10-09T06:43:33.186072+0000","last_became_active":"2023-10-09T06:19:06.666382+0000","last_became_peered":"2023-10-09T06:19:06.666382+0000","last_unstale":"2023-10-09T06:43:33.186072+0000","last_undegraded":"2023-10-09T06:43:33.186072+0000","last_fullsized":"2023-10-09T06:43:33.186072+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:01.108845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":511,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185970+0000","last_change":"2023-10-09T06:23:17.727421+0000","last_active":"2023-10-09T06:43:33.185970+0000","last_peered":"2023-10-09T06:43:33.185970+0000","last_clean":"2023-10-09T06:43:33.185970+0000","last_became_active":"2023-10-09T06:23:17.727081+0000","last_became_peered":"2023-10-09T06:23:17.727081+0000","last_unstale":"2023-10-09T06:43:33.185970+0000","last_undegraded":"2023-10-09T06:43:33.185970+0000","last_fullsized":"2023-10-09T06:43:33.185970+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:41.440509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":815,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.181946+0000","last_change":"2023-10-09T06:32:08.037455+0000","last_active":"2023-10-09T06:43:33.181946+0000","last_peered":"2023-10-09T06:43:33.181946+0000","last_clean":"2023-10-09T06:43:33.181946+0000","last_became_active":"2023-10-09T06:32:03.479746+0000","last_became_peered":"2023-10-09T06:32:03.479746+0000","last_unstale":"2023-10-09T06:43:33.181946+0000","last_undegraded":"2023-10-09T06:43:33.181946+0000","last_fullsized":"2023-10-09T06:43:33.181946+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:51:01.114788+0000","scrub_duration":0.0077798679999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":390,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389580+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:43:33.389580+0000","last_peered":"2023-10-09T06:43:33.389580+0000","last_clean":"2023-10-09T06:43:33.389580+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:43:33.389580+0000","last_undegraded":"2023-10-09T06:43:33.389580+0000","last_fullsized":"2023-10-09T06:43:33.389580+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178629+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:43:32.178629+0000","last_peered":"2023-10-09T06:43:32.178629+0000","last_clean":"2023-10-09T06:43:32.178629+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:43:32.178629+0000","last_undegraded":"2023-10-09T06:43:32.178629+0000","last_fullsized":"2023-10-09T06:43:32.178629+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":813,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390302+0000","last_change":"2023-10-09T06:39:10.128211+0000","last_active":"2023-10-09T06:43:33.390302+0000","last_peered":"2023-10-09T06:43:33.390302+0000","last_clean":"2023-10-09T06:43:33.390302+0000","last_became_active":"2023-10-09T06:37:08.252359+0000","last_became_peered":"2023-10-09T06:37:08.252359+0000","last_unstale":"2023-10-09T06:43:33.390302+0000","last_undegraded":"2023-10-09T06:43:33.390302+0000","last_fullsized":"2023-10-09T06:43:33.390302+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:37.056056+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":540,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170168+0000","last_change":"2023-10-09T06:29:15.538446+0000","last_active":"2023-10-09T06:43:32.170168+0000","last_peered":"2023-10-09T06:43:32.170168+0000","last_clean":"2023-10-09T06:43:32.170168+0000","last_became_active":"2023-10-09T06:29:15.536914+0000","last_became_peered":"2023-10-09T06:29:15.536914+0000","last_unstale":"2023-10-09T06:43:32.170168+0000","last_undegraded":"2023-10-09T06:43:32.170168+0000","last_fullsized":"2023-10-09T06:43:32.170168+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:36.665548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":361,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390311+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:43:33.390311+0000","last_peered":"2023-10-09T06:43:33.390311+0000","last_clean":"2023-10-09T06:43:33.390311+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:43:33.390311+0000","last_undegraded":"2023-10-09T06:43:33.390311+0000","last_fullsized":"2023-10-09T06:43:33.390311+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":870,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:35.851596+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:43:35.851596+0000","last_peered":"2023-10-09T06:43:35.851596+0000","last_clean":"2023-10-09T06:43:35.851596+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:43:35.851596+0000","last_undegraded":"2023-10-09T06:43:35.851596+0000","last_fullsized":"2023-10-09T06:43:35.851596+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":384,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389219+0000","last_change":"2023-10-09T06:19:06.758915+0000","last_active":"2023-10-09T06:43:33.389219+0000","last_peered":"2023-10-09T06:43:33.389219+0000","last_clean":"2023-10-09T06:43:33.389219+0000","last_became_active":"2023-10-09T06:19:06.757729+0000","last_became_peered":"2023-10-09T06:19:06.757729+0000","last_unstale":"2023-10-09T06:43:33.389219+0000","last_undegraded":"2023-10-09T06:43:33.389219+0000","last_fullsized":"2023-10-09T06:43:33.389219+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:03.304786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":216,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170457+0000","last_change":"2023-10-09T06:29:15.540805+0000","last_active":"2023-10-09T06:43:32.170457+0000","last_peered":"2023-10-09T06:43:32.170457+0000","last_clean":"2023-10-09T06:43:32.170457+0000","last_became_active":"2023-10-09T06:29:15.536863+0000","last_became_peered":"2023-10-09T06:29:15.536863+0000","last_unstale":"2023-10-09T06:43:32.170457+0000","last_undegraded":"2023-10-09T06:43:32.170457+0000","last_fullsized":"2023-10-09T06:43:32.170457+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:36.094135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":655,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389228+0000","last_change":"2023-10-09T06:32:14.142542+0000","last_active":"2023-10-09T06:43:33.389228+0000","last_peered":"2023-10-09T06:43:33.389228+0000","last_clean":"2023-10-09T06:43:33.389228+0000","last_became_active":"2023-10-09T06:32:03.461466+0000","last_became_peered":"2023-10-09T06:32:03.461466+0000","last_unstale":"2023-10-09T06:43:33.389228+0000","last_undegraded":"2023-10-09T06:43:33.389228+0000","last_fullsized":"2023-10-09T06:43:33.389228+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:54:28.228759+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":619,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207322+0000","last_change":"2023-10-09T06:29:15.458037+0000","last_active":"2023-10-09T06:43:33.207322+0000","last_peered":"2023-10-09T06:43:33.207322+0000","last_clean":"2023-10-09T06:43:33.207322+0000","last_became_active":"2023-10-09T06:29:15.457695+0000","last_became_peered":"2023-10-09T06:29:15.457695+0000","last_unstale":"2023-10-09T06:43:33.207322+0000","last_undegraded":"2023-10-09T06:43:33.207322+0000","last_fullsized":"2023-10-09T06:43:33.207322+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:44.709828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178202+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:43:32.178202+0000","last_peered":"2023-10-09T06:43:32.178202+0000","last_clean":"2023-10-09T06:43:32.178202+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:43:32.178202+0000","last_undegraded":"2023-10-09T06:43:32.178202+0000","last_fullsized":"2023-10-09T06:43:32.178202+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":668,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390118+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:43:33.390118+0000","last_peered":"2023-10-09T06:43:33.390118+0000","last_clean":"2023-10-09T06:43:33.390118+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:43:33.390118+0000","last_undegraded":"2023-10-09T06:43:33.390118+0000","last_fullsized":"2023-10-09T06:43:33.390118+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":426,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390452+0000","last_change":"2023-10-09T06:29:15.535549+0000","last_active":"2023-10-09T06:43:33.390452+0000","last_peered":"2023-10-09T06:43:33.390452+0000","last_clean":"2023-10-09T06:43:33.390452+0000","last_became_active":"2023-10-09T06:29:15.535302+0000","last_became_peered":"2023-10-09T06:29:15.535302+0000","last_unstale":"2023-10-09T06:43:33.390452+0000","last_undegraded":"2023-10-09T06:43:33.390452+0000","last_fullsized":"2023-10-09T06:43:33.390452+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:05:57.251358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":324,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389051+0000","last_change":"2023-10-09T06:37:08.253232+0000","last_active":"2023-10-09T06:43:33.389051+0000","last_peered":"2023-10-09T06:43:33.389051+0000","last_clean":"2023-10-09T06:43:33.389051+0000","last_became_active":"2023-10-09T06:37:08.252960+0000","last_became_peered":"2023-10-09T06:37:08.252960+0000","last_unstale":"2023-10-09T06:43:33.389051+0000","last_undegraded":"2023-10-09T06:43:33.389051+0000","last_fullsized":"2023-10-09T06:43:33.389051+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:17.372569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":928,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178409+0000","last_change":"2023-10-09T06:32:12.100485+0000","last_active":"2023-10-09T06:43:32.178409+0000","last_peered":"2023-10-09T06:43:32.178409+0000","last_clean":"2023-10-09T06:43:32.178409+0000","last_became_active":"2023-10-09T06:32:10.606543+0000","last_became_peered":"2023-10-09T06:32:10.606543+0000","last_unstale":"2023-10-09T06:43:32.178409+0000","last_undegraded":"2023-10-09T06:43:32.178409+0000","last_fullsized":"2023-10-09T06:43:32.178409+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:47.173023+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":388,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389756+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:43:33.389756+0000","last_peered":"2023-10-09T06:43:33.389756+0000","last_clean":"2023-10-09T06:43:33.389756+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:43:33.389756+0000","last_undegraded":"2023-10-09T06:43:33.389756+0000","last_fullsized":"2023-10-09T06:43:33.389756+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":538,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185666+0000","last_change":"2023-10-09T06:27:36.670287+0000","last_active":"2023-10-09T06:43:33.185666+0000","last_peered":"2023-10-09T06:43:33.185666+0000","last_clean":"2023-10-09T06:43:33.185666+0000","last_became_active":"2023-10-09T06:27:36.669982+0000","last_became_peered":"2023-10-09T06:27:36.669982+0000","last_unstale":"2023-10-09T06:43:33.185666+0000","last_undegraded":"2023-10-09T06:43:33.185666+0000","last_fullsized":"2023-10-09T06:43:33.185666+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:41:10.868141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":842,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207952+0000","last_change":"2023-10-09T06:32:13.928719+0000","last_active":"2023-10-09T06:43:33.207952+0000","last_peered":"2023-10-09T06:43:33.207952+0000","last_clean":"2023-10-09T06:43:33.207952+0000","last_became_active":"2023-10-09T06:32:03.407575+0000","last_became_peered":"2023-10-09T06:32:03.407575+0000","last_unstale":"2023-10-09T06:43:33.207952+0000","last_undegraded":"2023-10-09T06:43:33.207952+0000","last_fullsized":"2023-10-09T06:43:33.207952+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:13.928650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:13.928650+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:20.374631+0000","scrub_duration":0.0015939249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":444,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390244+0000","last_change":"2023-10-09T06:28:11.716229+0000","last_active":"2023-10-09T06:43:33.390244+0000","last_peered":"2023-10-09T06:43:33.390244+0000","last_clean":"2023-10-09T06:43:33.390244+0000","last_became_active":"2023-10-09T06:28:11.703520+0000","last_became_peered":"2023-10-09T06:28:11.703520+0000","last_unstale":"2023-10-09T06:43:33.390244+0000","last_undegraded":"2023-10-09T06:43:33.390244+0000","last_fullsized":"2023-10-09T06:43:33.390244+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:59.610213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":517,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207967+0000","last_change":"2023-10-09T06:19:06.730367+0000","last_active":"2023-10-09T06:43:33.207967+0000","last_peered":"2023-10-09T06:43:33.207967+0000","last_clean":"2023-10-09T06:43:33.207967+0000","last_became_active":"2023-10-09T06:19:06.730047+0000","last_became_peered":"2023-10-09T06:19:06.730047+0000","last_unstale":"2023-10-09T06:43:33.207967+0000","last_undegraded":"2023-10-09T06:43:33.207967+0000","last_fullsized":"2023-10-09T06:43:33.207967+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:57.188365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":665,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186083+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:43:33.186083+0000","last_peered":"2023-10-09T06:43:33.186083+0000","last_clean":"2023-10-09T06:43:33.186083+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:43:33.186083+0000","last_undegraded":"2023-10-09T06:43:33.186083+0000","last_fullsized":"2023-10-09T06:43:33.186083+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178708+0000","last_change":"2023-10-09T06:37:08.349625+0000","last_active":"2023-10-09T06:43:32.178708+0000","last_peered":"2023-10-09T06:43:32.178708+0000","last_clean":"2023-10-09T06:43:32.178708+0000","last_became_active":"2023-10-09T06:37:08.349327+0000","last_became_peered":"2023-10-09T06:37:08.349327+0000","last_unstale":"2023-10-09T06:43:32.178708+0000","last_undegraded":"2023-10-09T06:43:32.178708+0000","last_fullsized":"2023-10-09T06:43:32.178708+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:21:16.164314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":504,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170933+0000","last_change":"2023-10-09T06:29:15.444625+0000","last_active":"2023-10-09T06:43:32.170933+0000","last_peered":"2023-10-09T06:43:32.170933+0000","last_clean":"2023-10-09T06:43:32.170933+0000","last_became_active":"2023-10-09T06:29:15.444392+0000","last_became_peered":"2023-10-09T06:29:15.444392+0000","last_unstale":"2023-10-09T06:43:32.170933+0000","last_undegraded":"2023-10-09T06:43:32.170933+0000","last_fullsized":"2023-10-09T06:43:32.170933+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:56:40.003821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":893,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389092+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:43:33.389092+0000","last_peered":"2023-10-09T06:43:33.389092+0000","last_clean":"2023-10-09T06:43:33.389092+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:43:33.389092+0000","last_undegraded":"2023-10-09T06:43:33.389092+0000","last_fullsized":"2023-10-09T06:43:33.389092+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":270,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.177889+0000","last_change":"2023-10-09T06:29:15.542228+0000","last_active":"2023-10-09T06:43:32.177889+0000","last_peered":"2023-10-09T06:43:32.177889+0000","last_clean":"2023-10-09T06:43:32.177889+0000","last_became_active":"2023-10-09T06:29:15.541871+0000","last_became_peered":"2023-10-09T06:29:15.541871+0000","last_unstale":"2023-10-09T06:43:32.177889+0000","last_undegraded":"2023-10-09T06:43:32.177889+0000","last_fullsized":"2023-10-09T06:43:32.177889+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:06.777030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":521,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185972+0000","last_change":"2023-10-09T06:27:36.633514+0000","last_active":"2023-10-09T06:43:33.185972+0000","last_peered":"2023-10-09T06:43:33.185972+0000","last_clean":"2023-10-09T06:43:33.185972+0000","last_became_active":"2023-10-09T06:27:36.633204+0000","last_became_peered":"2023-10-09T06:27:36.633204+0000","last_unstale":"2023-10-09T06:43:33.185972+0000","last_undegraded":"2023-10-09T06:43:33.185972+0000","last_fullsized":"2023-10-09T06:43:33.185972+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:42:49.388045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":576,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.187613+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:43:33.187613+0000","last_peered":"2023-10-09T06:43:33.187613+0000","last_clean":"2023-10-09T06:43:33.187613+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:43:33.187613+0000","last_undegraded":"2023-10-09T06:43:33.187613+0000","last_fullsized":"2023-10-09T06:43:33.187613+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":498,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207878+0000","last_change":"2023-10-09T06:13:02.834951+0000","last_active":"2023-10-09T06:43:33.207878+0000","last_peered":"2023-10-09T06:43:33.207878+0000","last_clean":"2023-10-09T06:43:33.207878+0000","last_became_active":"2023-10-09T06:13:02.834581+0000","last_became_peered":"2023-10-09T06:13:02.834581+0000","last_unstale":"2023-10-09T06:43:33.207878+0000","last_undegraded":"2023-10-09T06:43:33.207878+0000","last_fullsized":"2023-10-09T06:43:33.207878+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:32.018492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":518,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186106+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:43:33.186106+0000","last_peered":"2023-10-09T06:43:33.186106+0000","last_clean":"2023-10-09T06:43:33.186106+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:43:33.186106+0000","last_undegraded":"2023-10-09T06:43:33.186106+0000","last_fullsized":"2023-10-09T06:43:33.186106+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":879,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186361+0000","last_change":"2023-10-09T06:32:19.978480+0000","last_active":"2023-10-09T06:43:33.186361+0000","last_peered":"2023-10-09T06:43:33.186361+0000","last_clean":"2023-10-09T06:43:33.186361+0000","last_became_active":"2023-10-09T06:32:10.592743+0000","last_became_peered":"2023-10-09T06:32:10.592743+0000","last_unstale":"2023-10-09T06:43:33.186361+0000","last_undegraded":"2023-10-09T06:43:33.186361+0000","last_fullsized":"2023-10-09T06:43:33.186361+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:50:09.485967+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":522,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185573+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:43:33.185573+0000","last_peered":"2023-10-09T06:43:33.185573+0000","last_clean":"2023-10-09T06:43:33.185573+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:43:33.185573+0000","last_undegraded":"2023-10-09T06:43:33.185573+0000","last_fullsized":"2023-10-09T06:43:33.185573+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":501,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186195+0000","last_change":"2023-10-09T06:13:02.830416+0000","last_active":"2023-10-09T06:43:33.186195+0000","last_peered":"2023-10-09T06:43:33.186195+0000","last_clean":"2023-10-09T06:43:33.186195+0000","last_became_active":"2023-10-09T06:13:02.830076+0000","last_became_peered":"2023-10-09T06:13:02.830076+0000","last_unstale":"2023-10-09T06:43:33.186195+0000","last_undegraded":"2023-10-09T06:43:33.186195+0000","last_fullsized":"2023-10-09T06:43:33.186195+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:40.341568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":503,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207840+0000","last_change":"2023-10-09T06:39:13.969329+0000","last_active":"2023-10-09T06:43:33.207840+0000","last_peered":"2023-10-09T06:43:33.207840+0000","last_clean":"2023-10-09T06:43:33.207840+0000","last_became_active":"2023-10-09T06:37:08.251454+0000","last_became_peered":"2023-10-09T06:37:08.251454+0000","last_unstale":"2023-10-09T06:43:33.207840+0000","last_undegraded":"2023-10-09T06:43:33.207840+0000","last_fullsized":"2023-10-09T06:43:33.207840+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:13.969189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:13.969189+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:25.773858+0000","scrub_duration":0.001928859,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176374+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:43:32.176374+0000","last_peered":"2023-10-09T06:43:32.176374+0000","last_clean":"2023-10-09T06:43:32.176374+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:43:32.176374+0000","last_undegraded":"2023-10-09T06:43:32.176374+0000","last_fullsized":"2023-10-09T06:43:32.176374+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":498,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186487+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:43:33.186487+0000","last_peered":"2023-10-09T06:43:33.186487+0000","last_clean":"2023-10-09T06:43:33.186487+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:43:33.186487+0000","last_undegraded":"2023-10-09T06:43:33.186487+0000","last_fullsized":"2023-10-09T06:43:33.186487+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":833,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.208055+0000","last_change":"2023-10-09T06:32:15.917369+0000","last_active":"2023-10-09T06:43:33.208055+0000","last_peered":"2023-10-09T06:43:33.208055+0000","last_clean":"2023-10-09T06:43:33.208055+0000","last_became_active":"2023-10-09T06:32:10.607940+0000","last_became_peered":"2023-10-09T06:32:10.607940+0000","last_unstale":"2023-10-09T06:43:33.208055+0000","last_undegraded":"2023-10-09T06:43:33.208055+0000","last_fullsized":"2023-10-09T06:43:33.208055+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.917298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.917298+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:14.614342+0000","scrub_duration":0.0089320809999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":550,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186771+0000","last_change":"2023-10-09T06:29:15.456849+0000","last_active":"2023-10-09T06:43:33.186771+0000","last_peered":"2023-10-09T06:43:33.186771+0000","last_clean":"2023-10-09T06:43:33.186771+0000","last_became_active":"2023-10-09T06:29:15.456619+0000","last_became_peered":"2023-10-09T06:29:15.456619+0000","last_unstale":"2023-10-09T06:43:33.186771+0000","last_undegraded":"2023-10-09T06:43:33.186771+0000","last_fullsized":"2023-10-09T06:43:33.186771+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:35:40.342186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178041+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:43:32.178041+0000","last_peered":"2023-10-09T06:43:32.178041+0000","last_clean":"2023-10-09T06:43:32.178041+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:43:32.178041+0000","last_undegraded":"2023-10-09T06:43:32.178041+0000","last_fullsized":"2023-10-09T06:43:32.178041+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":484,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186641+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:43:33.186641+0000","last_peered":"2023-10-09T06:43:33.186641+0000","last_clean":"2023-10-09T06:43:33.186641+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:43:33.186641+0000","last_undegraded":"2023-10-09T06:43:33.186641+0000","last_fullsized":"2023-10-09T06:43:33.186641+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":721,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170270+0000","last_change":"2023-10-09T06:39:31.102253+0000","last_active":"2023-10-09T06:43:32.170270+0000","last_peered":"2023-10-09T06:43:32.170270+0000","last_clean":"2023-10-09T06:43:32.170270+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:43:32.170270+0000","last_undegraded":"2023-10-09T06:43:32.170270+0000","last_fullsized":"2023-10-09T06:43:32.170270+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:23.414128+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":274,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186727+0000","last_change":"2023-10-09T06:37:08.254481+0000","last_active":"2023-10-09T06:43:33.186727+0000","last_peered":"2023-10-09T06:43:33.186727+0000","last_clean":"2023-10-09T06:43:33.186727+0000","last_became_active":"2023-10-09T06:37:08.254205+0000","last_became_peered":"2023-10-09T06:37:08.254205+0000","last_unstale":"2023-10-09T06:43:33.186727+0000","last_undegraded":"2023-10-09T06:43:33.186727+0000","last_fullsized":"2023-10-09T06:43:33.186727+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:11.189708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":522,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185877+0000","last_change":"2023-10-09T06:29:15.448291+0000","last_active":"2023-10-09T06:43:33.185877+0000","last_peered":"2023-10-09T06:43:33.185877+0000","last_clean":"2023-10-09T06:43:33.185877+0000","last_became_active":"2023-10-09T06:29:15.447968+0000","last_became_peered":"2023-10-09T06:29:15.447968+0000","last_unstale":"2023-10-09T06:43:33.185877+0000","last_undegraded":"2023-10-09T06:43:33.185877+0000","last_fullsized":"2023-10-09T06:43:33.185877+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:46.926928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":514,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186695+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:43:33.186695+0000","last_peered":"2023-10-09T06:43:33.186695+0000","last_clean":"2023-10-09T06:43:33.186695+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:43:33.186695+0000","last_undegraded":"2023-10-09T06:43:33.186695+0000","last_fullsized":"2023-10-09T06:43:33.186695+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":883,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389389+0000","last_change":"2023-10-09T06:32:09.868805+0000","last_active":"2023-10-09T06:43:33.389389+0000","last_peered":"2023-10-09T06:43:33.389389+0000","last_clean":"2023-10-09T06:43:33.389389+0000","last_became_active":"2023-10-09T06:31:52.448350+0000","last_became_peered":"2023-10-09T06:31:52.448350+0000","last_unstale":"2023-10-09T06:43:33.389389+0000","last_undegraded":"2023-10-09T06:43:33.389389+0000","last_fullsized":"2023-10-09T06:43:33.389389+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:57.811343+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":535,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176094+0000","last_change":"2023-10-09T06:29:15.570668+0000","last_active":"2023-10-09T06:43:32.176094+0000","last_peered":"2023-10-09T06:43:32.176094+0000","last_clean":"2023-10-09T06:43:32.176094+0000","last_became_active":"2023-10-09T06:29:15.550988+0000","last_became_peered":"2023-10-09T06:29:15.550988+0000","last_unstale":"2023-10-09T06:43:32.176094+0000","last_undegraded":"2023-10-09T06:43:32.176094+0000","last_fullsized":"2023-10-09T06:43:32.176094+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:06:16.495679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":523,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185833+0000","last_change":"2023-10-09T06:15:11.868183+0000","last_active":"2023-10-09T06:43:33.185833+0000","last_peered":"2023-10-09T06:43:33.185833+0000","last_clean":"2023-10-09T06:43:33.185833+0000","last_became_active":"2023-10-09T06:15:11.867841+0000","last_became_peered":"2023-10-09T06:15:11.867841+0000","last_unstale":"2023-10-09T06:43:33.185833+0000","last_undegraded":"2023-10-09T06:43:33.185833+0000","last_fullsized":"2023-10-09T06:43:33.185833+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:31:37.716910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":545,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186578+0000","last_change":"2023-10-09T06:29:15.456889+0000","last_active":"2023-10-09T06:43:33.186578+0000","last_peered":"2023-10-09T06:43:33.186578+0000","last_clean":"2023-10-09T06:43:33.186578+0000","last_became_active":"2023-10-09T06:29:15.456572+0000","last_became_peered":"2023-10-09T06:29:15.456572+0000","last_unstale":"2023-10-09T06:43:33.186578+0000","last_undegraded":"2023-10-09T06:43:33.186578+0000","last_fullsized":"2023-10-09T06:43:33.186578+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:01.578949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":350,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389533+0000","last_change":"2023-10-09T06:37:08.251305+0000","last_active":"2023-10-09T06:43:33.389533+0000","last_peered":"2023-10-09T06:43:33.389533+0000","last_clean":"2023-10-09T06:43:33.389533+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:43:33.389533+0000","last_undegraded":"2023-10-09T06:43:33.389533+0000","last_fullsized":"2023-10-09T06:43:33.389533+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":864,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.389139+0000","last_change":"2023-10-09T06:32:10.851458+0000","last_active":"2023-10-09T06:43:33.389139+0000","last_peered":"2023-10-09T06:43:33.389139+0000","last_clean":"2023-10-09T06:43:33.389139+0000","last_became_active":"2023-10-09T06:31:52.455701+0000","last_became_peered":"2023-10-09T06:31:52.455701+0000","last_unstale":"2023-10-09T06:43:33.389139+0000","last_undegraded":"2023-10-09T06:43:33.389139+0000","last_fullsized":"2023-10-09T06:43:33.389139+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:53.965517+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.177791+0000","last_change":"2023-10-09T06:28:11.696245+0000","last_active":"2023-10-09T06:43:32.177791+0000","last_peered":"2023-10-09T06:43:32.177791+0000","last_clean":"2023-10-09T06:43:32.177791+0000","last_became_active":"2023-10-09T06:28:11.695770+0000","last_became_peered":"2023-10-09T06:28:11.695770+0000","last_unstale":"2023-10-09T06:43:32.177791+0000","last_undegraded":"2023-10-09T06:43:32.177791+0000","last_fullsized":"2023-10-09T06:43:32.177791+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:45:54.630917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":503,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207845+0000","last_change":"2023-10-09T06:19:06.758679+0000","last_active":"2023-10-09T06:43:33.207845+0000","last_peered":"2023-10-09T06:43:33.207845+0000","last_clean":"2023-10-09T06:43:33.207845+0000","last_became_active":"2023-10-09T06:19:06.758242+0000","last_became_peered":"2023-10-09T06:19:06.758242+0000","last_unstale":"2023-10-09T06:43:33.207845+0000","last_undegraded":"2023-10-09T06:43:33.207845+0000","last_fullsized":"2023-10-09T06:43:33.207845+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:12:09.670790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":853,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186434+0000","last_change":"2023-10-09T06:32:14.928395+0000","last_active":"2023-10-09T06:43:33.186434+0000","last_peered":"2023-10-09T06:43:33.186434+0000","last_clean":"2023-10-09T06:43:33.186434+0000","last_became_active":"2023-10-09T06:31:52.459601+0000","last_became_peered":"2023-10-09T06:31:52.459601+0000","last_unstale":"2023-10-09T06:43:33.186434+0000","last_undegraded":"2023-10-09T06:43:33.186434+0000","last_fullsized":"2023-10-09T06:43:33.186434+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:54.426603+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":404,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.388934+0000","last_change":"2023-10-09T06:29:15.536752+0000","last_active":"2023-10-09T06:43:33.388934+0000","last_peered":"2023-10-09T06:43:33.388934+0000","last_clean":"2023-10-09T06:43:33.388934+0000","last_became_active":"2023-10-09T06:29:15.536252+0000","last_became_peered":"2023-10-09T06:29:15.536252+0000","last_unstale":"2023-10-09T06:43:33.388934+0000","last_undegraded":"2023-10-09T06:43:33.388934+0000","last_fullsized":"2023-10-09T06:43:33.388934+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:11.705779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":217,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170168+0000","last_change":"2023-10-09T06:29:15.536986+0000","last_active":"2023-10-09T06:43:32.170168+0000","last_peered":"2023-10-09T06:43:32.170168+0000","last_clean":"2023-10-09T06:43:32.170168+0000","last_became_active":"2023-10-09T06:29:15.536678+0000","last_became_peered":"2023-10-09T06:29:15.536678+0000","last_unstale":"2023-10-09T06:43:32.170168+0000","last_undegraded":"2023-10-09T06:43:32.170168+0000","last_fullsized":"2023-10-09T06:43:32.170168+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:15.709878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":528,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170814+0000","last_change":"2023-10-09T06:29:15.534823+0000","last_active":"2023-10-09T06:43:32.170814+0000","last_peered":"2023-10-09T06:43:32.170814+0000","last_clean":"2023-10-09T06:43:32.170814+0000","last_became_active":"2023-10-09T06:29:15.533697+0000","last_became_peered":"2023-10-09T06:29:15.533697+0000","last_unstale":"2023-10-09T06:43:32.170814+0000","last_undegraded":"2023-10-09T06:43:32.170814+0000","last_fullsized":"2023-10-09T06:43:32.170814+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:39.506120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":863,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.207884+0000","last_change":"2023-10-09T06:32:16.935871+0000","last_active":"2023-10-09T06:43:33.207884+0000","last_peered":"2023-10-09T06:43:33.207884+0000","last_clean":"2023-10-09T06:43:33.207884+0000","last_became_active":"2023-10-09T06:32:10.603857+0000","last_became_peered":"2023-10-09T06:32:10.603857+0000","last_unstale":"2023-10-09T06:43:33.207884+0000","last_undegraded":"2023-10-09T06:43:33.207884+0000","last_fullsized":"2023-10-09T06:43:33.207884+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.935768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.935768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:53.590265+0000","scrub_duration":0.022475815999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":404,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390382+0000","last_change":"2023-10-09T06:29:15.536567+0000","last_active":"2023-10-09T06:43:33.390382+0000","last_peered":"2023-10-09T06:43:33.390382+0000","last_clean":"2023-10-09T06:43:33.390382+0000","last_became_active":"2023-10-09T06:29:15.536288+0000","last_became_peered":"2023-10-09T06:29:15.536288+0000","last_unstale":"2023-10-09T06:43:33.390382+0000","last_undegraded":"2023-10-09T06:43:33.390382+0000","last_fullsized":"2023-10-09T06:43:33.390382+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:28:04.998138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":540,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186617+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:43:33.186617+0000","last_peered":"2023-10-09T06:43:33.186617+0000","last_clean":"2023-10-09T06:43:33.186617+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:43:33.186617+0000","last_undegraded":"2023-10-09T06:43:33.186617+0000","last_fullsized":"2023-10-09T06:43:33.186617+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":526,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.171156+0000","last_change":"2023-10-09T06:29:15.534479+0000","last_active":"2023-10-09T06:43:32.171156+0000","last_peered":"2023-10-09T06:43:32.171156+0000","last_clean":"2023-10-09T06:43:32.171156+0000","last_became_active":"2023-10-09T06:29:15.534209+0000","last_became_peered":"2023-10-09T06:29:15.534209+0000","last_unstale":"2023-10-09T06:43:32.171156+0000","last_undegraded":"2023-10-09T06:43:32.171156+0000","last_fullsized":"2023-10-09T06:43:32.171156+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:01.584300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":864,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186052+0000","last_change":"2023-10-09T06:32:17.264203+0000","last_active":"2023-10-09T06:43:33.186052+0000","last_peered":"2023-10-09T06:43:33.186052+0000","last_clean":"2023-10-09T06:43:33.186052+0000","last_became_active":"2023-10-09T06:32:10.604673+0000","last_became_peered":"2023-10-09T06:32:10.604673+0000","last_unstale":"2023-10-09T06:43:33.186052+0000","last_undegraded":"2023-10-09T06:43:33.186052+0000","last_fullsized":"2023-10-09T06:43:33.186052+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:07.243552+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":274,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176166+0000","last_change":"2023-10-09T06:29:15.557945+0000","last_active":"2023-10-09T06:43:32.176166+0000","last_peered":"2023-10-09T06:43:32.176166+0000","last_clean":"2023-10-09T06:43:32.176166+0000","last_became_active":"2023-10-09T06:29:15.537433+0000","last_became_peered":"2023-10-09T06:29:15.537433+0000","last_unstale":"2023-10-09T06:43:32.176166+0000","last_undegraded":"2023-10-09T06:43:32.176166+0000","last_fullsized":"2023-10-09T06:43:32.176166+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:14.773121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":508,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186915+0000","last_change":"2023-10-09T06:13:02.833327+0000","last_active":"2023-10-09T06:43:33.186915+0000","last_peered":"2023-10-09T06:43:33.186915+0000","last_clean":"2023-10-09T06:43:33.186915+0000","last_became_active":"2023-10-09T06:13:02.832893+0000","last_became_peered":"2023-10-09T06:13:02.832893+0000","last_unstale":"2023-10-09T06:43:33.186915+0000","last_undegraded":"2023-10-09T06:43:33.186915+0000","last_fullsized":"2023-10-09T06:43:33.186915+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:40.226319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176308+0000","last_change":"2023-10-09T06:29:15.516595+0000","last_active":"2023-10-09T06:43:32.176308+0000","last_peered":"2023-10-09T06:43:32.176308+0000","last_clean":"2023-10-09T06:43:32.176308+0000","last_became_active":"2023-10-09T06:29:15.516226+0000","last_became_peered":"2023-10-09T06:29:15.516226+0000","last_unstale":"2023-10-09T06:43:32.176308+0000","last_undegraded":"2023-10-09T06:43:32.176308+0000","last_fullsized":"2023-10-09T06:43:32.176308+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:21.132712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":896,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178713+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:43:32.178713+0000","last_peered":"2023-10-09T06:43:32.178713+0000","last_clean":"2023-10-09T06:43:32.178713+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:43:32.178713+0000","last_undegraded":"2023-10-09T06:43:32.178713+0000","last_fullsized":"2023-10-09T06:43:32.178713+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":525,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.208102+0000","last_change":"2023-10-09T06:19:06.672726+0000","last_active":"2023-10-09T06:43:33.208102+0000","last_peered":"2023-10-09T06:43:33.208102+0000","last_clean":"2023-10-09T06:43:33.208102+0000","last_became_active":"2023-10-09T06:19:06.672389+0000","last_became_peered":"2023-10-09T06:19:06.672389+0000","last_unstale":"2023-10-09T06:43:33.208102+0000","last_undegraded":"2023-10-09T06:43:33.208102+0000","last_fullsized":"2023-10-09T06:43:33.208102+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:09.676491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":509,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170668+0000","last_change":"2023-10-09T06:29:15.534372+0000","last_active":"2023-10-09T06:43:32.170668+0000","last_peered":"2023-10-09T06:43:32.170668+0000","last_clean":"2023-10-09T06:43:32.170668+0000","last_became_active":"2023-10-09T06:29:15.532206+0000","last_became_peered":"2023-10-09T06:29:15.532206+0000","last_unstale":"2023-10-09T06:43:32.170668+0000","last_undegraded":"2023-10-09T06:43:32.170668+0000","last_fullsized":"2023-10-09T06:43:32.170668+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:25.939856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":504,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185644+0000","last_change":"2023-10-09T06:28:11.697595+0000","last_active":"2023-10-09T06:43:33.185644+0000","last_peered":"2023-10-09T06:43:33.185644+0000","last_clean":"2023-10-09T06:43:33.185644+0000","last_became_active":"2023-10-09T06:28:11.697321+0000","last_became_peered":"2023-10-09T06:28:11.697321+0000","last_unstale":"2023-10-09T06:43:33.185644+0000","last_undegraded":"2023-10-09T06:43:33.185644+0000","last_fullsized":"2023-10-09T06:43:33.185644+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:42.691027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":822,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.176777+0000","last_change":"2023-10-09T06:32:25.148405+0000","last_active":"2023-10-09T06:43:32.176777+0000","last_peered":"2023-10-09T06:43:32.176777+0000","last_clean":"2023-10-09T06:43:32.176777+0000","last_became_active":"2023-10-09T06:32:19.774966+0000","last_became_peered":"2023-10-09T06:32:19.774966+0000","last_unstale":"2023-10-09T06:43:32.176777+0000","last_undegraded":"2023-10-09T06:43:32.176777+0000","last_fullsized":"2023-10-09T06:43:32.176777+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:57.619289+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":510,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.171081+0000","last_change":"2023-10-09T06:29:15.534549+0000","last_active":"2023-10-09T06:43:32.171081+0000","last_peered":"2023-10-09T06:43:32.171081+0000","last_clean":"2023-10-09T06:43:32.171081+0000","last_became_active":"2023-10-09T06:29:15.534233+0000","last_became_peered":"2023-10-09T06:29:15.534233+0000","last_unstale":"2023-10-09T06:43:32.171081+0000","last_undegraded":"2023-10-09T06:43:32.171081+0000","last_fullsized":"2023-10-09T06:43:32.171081+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:17.687300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":532,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186529+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:43:33.186529+0000","last_peered":"2023-10-09T06:43:33.186529+0000","last_clean":"2023-10-09T06:43:33.186529+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:43:33.186529+0000","last_undegraded":"2023-10-09T06:43:33.186529+0000","last_fullsized":"2023-10-09T06:43:33.186529+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":753,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.177907+0000","last_change":"2023-10-09T06:32:20.145528+0000","last_active":"2023-10-09T06:43:32.177907+0000","last_peered":"2023-10-09T06:43:32.177907+0000","last_clean":"2023-10-09T06:43:32.177907+0000","last_became_active":"2023-10-09T06:32:19.760554+0000","last_became_peered":"2023-10-09T06:32:19.760554+0000","last_unstale":"2023-10-09T06:43:32.177907+0000","last_undegraded":"2023-10-09T06:43:32.177907+0000","last_fullsized":"2023-10-09T06:43:32.177907+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:08.769835+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":219,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185982+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:43:33.185982+0000","last_peered":"2023-10-09T06:43:33.185982+0000","last_clean":"2023-10-09T06:43:33.185982+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:43:33.185982+0000","last_undegraded":"2023-10-09T06:43:33.185982+0000","last_fullsized":"2023-10-09T06:43:33.185982+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":548,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185849+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:43:33.185849+0000","last_peered":"2023-10-09T06:43:33.185849+0000","last_clean":"2023-10-09T06:43:33.185849+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:43:33.185849+0000","last_undegraded":"2023-10-09T06:43:33.185849+0000","last_fullsized":"2023-10-09T06:43:33.185849+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":219,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186148+0000","last_change":"2023-10-09T06:37:08.252497+0000","last_active":"2023-10-09T06:43:33.186148+0000","last_peered":"2023-10-09T06:43:33.186148+0000","last_clean":"2023-10-09T06:43:33.186148+0000","last_became_active":"2023-10-09T06:37:08.252174+0000","last_became_peered":"2023-10-09T06:37:08.252174+0000","last_unstale":"2023-10-09T06:43:33.186148+0000","last_undegraded":"2023-10-09T06:43:33.186148+0000","last_fullsized":"2023-10-09T06:43:33.186148+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:11.679524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":558,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185847+0000","last_change":"2023-10-09T06:29:15.526425+0000","last_active":"2023-10-09T06:43:33.185847+0000","last_peered":"2023-10-09T06:43:33.185847+0000","last_clean":"2023-10-09T06:43:33.185847+0000","last_became_active":"2023-10-09T06:29:15.526038+0000","last_became_peered":"2023-10-09T06:29:15.526038+0000","last_unstale":"2023-10-09T06:43:33.185847+0000","last_undegraded":"2023-10-09T06:43:33.185847+0000","last_fullsized":"2023-10-09T06:43:33.185847+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:43:01.886002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":897,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.390271+0000","last_change":"2023-10-09T06:32:22.597972+0000","last_active":"2023-10-09T06:43:33.390271+0000","last_peered":"2023-10-09T06:43:33.390271+0000","last_clean":"2023-10-09T06:43:33.390271+0000","last_became_active":"2023-10-09T06:32:19.754160+0000","last_became_peered":"2023-10-09T06:32:19.754160+0000","last_unstale":"2023-10-09T06:43:33.390271+0000","last_undegraded":"2023-10-09T06:43:33.390271+0000","last_fullsized":"2023-10-09T06:43:33.390271+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:17:53.871522+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":244,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.178108+0000","last_change":"2023-10-09T06:27:36.651271+0000","last_active":"2023-10-09T06:43:32.178108+0000","last_peered":"2023-10-09T06:43:32.178108+0000","last_clean":"2023-10-09T06:43:32.178108+0000","last_became_active":"2023-10-09T06:27:36.638882+0000","last_became_peered":"2023-10-09T06:27:36.638882+0000","last_unstale":"2023-10-09T06:43:32.178108+0000","last_undegraded":"2023-10-09T06:43:32.178108+0000","last_fullsized":"2023-10-09T06:43:32.178108+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:04.901705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":261,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.177174+0000","last_change":"2023-10-09T06:29:15.570880+0000","last_active":"2023-10-09T06:43:32.177174+0000","last_peered":"2023-10-09T06:43:32.177174+0000","last_clean":"2023-10-09T06:43:32.177174+0000","last_became_active":"2023-10-09T06:29:15.554507+0000","last_became_peered":"2023-10-09T06:29:15.554507+0000","last_unstale":"2023-10-09T06:43:32.177174+0000","last_undegraded":"2023-10-09T06:43:32.177174+0000","last_fullsized":"2023-10-09T06:43:32.177174+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:37.934108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":665,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.186346+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:43:33.186346+0000","last_peered":"2023-10-09T06:43:33.186346+0000","last_clean":"2023-10-09T06:43:33.186346+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:43:33.186346+0000","last_undegraded":"2023-10-09T06:43:33.186346+0000","last_fullsized":"2023-10-09T06:43:33.186346+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":539,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T06:43:32.170567+0000","last_change":"2023-10-09T06:29:15.534273+0000","last_active":"2023-10-09T06:43:32.170567+0000","last_peered":"2023-10-09T06:43:32.170567+0000","last_clean":"2023-10-09T06:43:32.170567+0000","last_became_active":"2023-10-09T06:29:15.532970+0000","last_became_peered":"2023-10-09T06:29:15.532970+0000","last_unstale":"2023-10-09T06:43:32.170567+0000","last_undegraded":"2023-10-09T06:43:32.170567+0000","last_fullsized":"2023-10-09T06:43:32.170567+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:42.960845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":501,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T06:43:33.185427+0000","last_change":"2023-10-09T06:13:02.830526+0000","last_active":"2023-10-09T06:43:33.185427+0000","last_peered":"2023-10-09T06:43:33.185427+0000","last_clean":"2023-10-09T06:43:33.185427+0000","last_became_active":"2023-10-09T06:13:02.830208+0000","last_became_peered":"2023-10-09T06:13:02.830208+0000","last_unstale":"2023-10-09T06:43:33.185427+0000","last_undegraded":"2023-10-09T06:43:33.185427+0000","last_fullsized":"2023-10-09T06:43:33.185427+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:53.164539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":10188,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234557,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564120,"kb_used_data":59024,"kb_used_omap":0,"kb_used_meta":1505088,"kb_avail":92181032,"statfs":{"total":95995035648,"available":94393376768,"internally_reserved":0,"allocated":60440576,"data_stored":59345455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541210112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:19 2023","interfaces":[{"interface":"back","average":{"1min":7.6639999999999997,"5min":3.3919999999999999,"15min":3.629},"min":{"1min":0.26700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":76.727999999999994,"5min":76.727999999999994,"15min":196.28899999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":7.8079999999999998,"5min":3.4239999999999999,"15min":3.6840000000000002},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":76.712999999999994,"5min":76.712999999999994,"15min":196.333},"last":0.373}]},{"osd":1,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.8149999999999999,"15min":5.069},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23200000000000001},"max":{"1min":7.9080000000000004,"5min":169.51900000000001,"15min":208.626},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.1160000000000001,"5min":3.7410000000000001,"15min":4.1139999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":6.5730000000000004,"5min":169.101,"15min":181.24199999999999},"last":0.48499999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:43:00 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":5.5679999999999996,"15min":4.9640000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.8869999999999996,"5min":228.24000000000001,"15min":228.24000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.1070000000000002,"5min":5.8949999999999996,"15min":5.0579999999999998},"min":{"1min":0.40400000000000003,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":15.355,"5min":228.291,"15min":228.291},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":4.4500000000000002,"15min":4.8659999999999997},"min":{"1min":0.373,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":3.8740000000000001,"5min":88.897999999999996,"15min":187.34200000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.952,"5min":4.4160000000000004,"15min":4.8529999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":6.4509999999999996,"5min":88.926000000000002,"15min":187.08199999999999},"last":0.46200000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.393000000000001,"5min":5.2069999999999999,"15min":2.9020000000000001},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":130.774,"5min":225.041,"15min":225.041},"last":0.28399999999999997},{"interface":"front","average":{"1min":11.032999999999999,"5min":5.0499999999999998,"15min":2.7450000000000001},"min":{"1min":0.30299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":130.72999999999999,"5min":225.244,"15min":225.244},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.8820000000000001,"15min":2.4060000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.60299999999999998,"5min":219.923,"15min":219.923},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.82999999999999996,"5min":2.9460000000000002,"15min":2.3610000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:42:45 2023","interfaces":[{"interface":"back","average":{"1min":11.494999999999999,"5min":5.1429999999999998,"15min":3.2050000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.317},"max":{"1min":138.477,"5min":222.179,"15min":222.179},"last":0.51400000000000001},{"interface":"front","average":{"1min":11.484999999999999,"5min":5.04,"15min":3.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":135.244,"5min":223.261,"15min":223.261},"last":0.497}]}]},{"osd":6,"up_from":46,"seq":197568496193,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557888,"kb_used_data":53560,"kb_used_omap":0,"kb_used_meta":1504320,"kb_avail":92187264,"statfs":{"total":95995035648,"available":94399758336,"internally_reserved":0,"allocated":54845440,"data_stored":53773617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1540423680},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:36 2023","interfaces":[{"interface":"back","average":{"1min":5.5199999999999996,"5min":2.6890000000000001,"15min":2.9049999999999998},"min":{"1min":0.24399999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":45.613999999999997,"5min":45.613999999999997,"15min":75.061999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":5.6369999999999996,"5min":5.1180000000000003,"15min":3.9510000000000001},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":45.646999999999998,"5min":212.096,"15min":212.096},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":3.5139999999999998,"5min":3.238,"15min":4.0549999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.20899999999999999},"max":{"1min":40.067,"5min":157.852,"15min":219.93199999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":3.0270000000000001,"5min":3.1600000000000001,"15min":4.0800000000000001},"min":{"1min":0.27800000000000002,"5min":0.222,"15min":0.222},"max":{"1min":40.085999999999999,"5min":157.87100000000001,"15min":219.94800000000001},"last":0.47799999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":3.0249999999999999,"15min":3.8140000000000001},"min":{"1min":0.28899999999999998,"5min":0.25,"15min":0.25},"max":{"1min":5.7939999999999996,"5min":158,"15min":185.654},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.1599999999999999,"5min":2.9359999999999999,"15min":3.617},"min":{"1min":0.312,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.819,"5min":158.017,"15min":185.67699999999999},"last":0.45200000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:43:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":3.7839999999999998,"15min":4.3730000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.9530000000000003,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":2.3149999999999999,"5min":3.9009999999999998,"15min":4.2930000000000001},"min":{"1min":0.38500000000000001,"5min":0.33100000000000002,"15min":0.32100000000000001},"max":{"1min":9.0579999999999998,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.53000000000000003}]},{"osd":4,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.089,"15min":1.5269999999999999},"min":{"1min":0.30399999999999999,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":2.4060000000000001,"5min":10.009,"15min":86.171999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":3.0939999999999999,"15min":2.1749999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.223},"max":{"1min":1.8200000000000001,"5min":212.03399999999999,"15min":212.03399999999999},"last":0.58599999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.3560000000000001,"15min":1.3500000000000001},"min":{"1min":0.29799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2880000000000003,"5min":10.435,"15min":12.747999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.1579999999999999,"15min":1.522},"min":{"1min":0.308,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.175,"5min":9.0060000000000002,"15min":16.536999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:42:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":1.0329999999999999,"15min":1.7170000000000001},"min":{"1min":0.33800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.8300000000000001,"5min":8.8059999999999992,"15min":165.37},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.1779999999999999,"15min":2.056},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.3610000000000007,"5min":8.5980000000000008,"15min":165.40199999999999},"last":0.66200000000000003}]}]},{"osd":1,"up_from":110,"seq":472446402930,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214956,"kb_used_data":70244,"kb_used_omap":4,"kb_used_meta":1144699,"kb_avail":92530196,"statfs":{"total":95995035648,"available":94750920704,"internally_reserved":0,"allocated":71929856,"data_stored":69641093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1172172686},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.6719999999999997,"15min":5.4580000000000002},"min":{"1min":0.23200000000000001,"5min":0.182,"15min":0.182},"max":{"1min":114.97799999999999,"5min":114.97799999999999,"15min":269.79199999999997},"last":0.307},{"interface":"front","average":{"1min":10.832000000000001,"5min":6.5330000000000004,"15min":5.8339999999999996},"min":{"1min":0.21099999999999999,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":114.934,"5min":114.934,"15min":271.20999999999998},"last":0.48699999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.4139999999999999,"15min":2.2410000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.184},"max":{"1min":4.4699999999999998,"5min":8.0129999999999999,"15min":138.452},"last":0.55900000000000005},{"interface":"front","average":{"1min":4.016,"5min":2.8759999999999999,"15min":3.0659999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":20.085000000000001,"5min":20.085000000000001,"15min":138.47499999999999},"last":0.65000000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:43:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.7610000000000001,"15min":2.7429999999999999},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":9.8580000000000005,"5min":27.655999999999999,"15min":27.655999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.383,"5min":2.0880000000000001,"15min":1.9770000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":10.332000000000001,"5min":27.745999999999999,"15min":27.745999999999999},"last":0.51800000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":3.7229999999999999,"15min":3.2930000000000001},"min":{"1min":0.378,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.9210000000000003,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.643,"5min":3.552,"15min":2.9700000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":4.3159999999999998,"5min":180.70599999999999,"15min":180.70599999999999},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.589,"15min":2.5790000000000002},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":3.6160000000000001,"5min":15.83,"15min":119.408},"last":0.623},{"interface":"front","average":{"1min":1.6539999999999999,"5min":1.8560000000000001,"15min":2.835},"min":{"1min":0.376,"5min":0.312,"15min":0.312},"max":{"1min":4.3399999999999999,"5min":7.9470000000000001,"15min":119.483},"last":0.68100000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":8.7739999999999991,"5min":5.0730000000000004,"15min":3.9609999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":161.54900000000001,"5min":183.12100000000001,"15min":183.12100000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":8.8140000000000001,"5min":5.3399999999999999,"15min":4.3609999999999998},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.316},"max":{"1min":161.566,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:43:02 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":3.5670000000000002,"15min":4.0659999999999998},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":4.8890000000000002,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.2290000000000001,"5min":2.9220000000000002,"15min":3.4119999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":5.3419999999999996,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.66200000000000003}]}]},{"osd":0,"up_from":91,"seq":390842024363,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186276,"kb_used_data":7196,"kb_used_omap":5,"kb_used_meta":1179066,"kb_avail":92558876,"statfs":{"total":95995035648,"available":94780289024,"internally_reserved":0,"allocated":7368704,"data_stored":4682451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1207364218},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:42:51 2023","interfaces":[{"interface":"back","average":{"1min":3.2730000000000001,"5min":8.0039999999999996,"15min":4.3869999999999996},"min":{"1min":0.22500000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":13.387,"5min":201.381,"15min":201.381},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.1589999999999998,"5min":7.5019999999999998,"15min":4.609},"min":{"1min":0.24399999999999999,"5min":0.193,"15min":0.187},"max":{"1min":8.3219999999999992,"5min":201.44200000000001,"15min":201.44200000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":11.029,"5min":4.5430000000000001,"15min":4.0030000000000001},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20399999999999999},"max":{"1min":147.61099999999999,"5min":147.61099999999999,"15min":147.61099999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":11.228999999999999,"5min":5.024,"15min":4.415},"min":{"1min":0.30199999999999999,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":150.12700000000001,"5min":150.12700000000001,"15min":150.12700000000001},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":2.9929999999999999,"15min":4.2050000000000001},"min":{"1min":0.28599999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.4809999999999999,"5min":95.555000000000007,"15min":187.172},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.3580000000000001,"5min":3.0579999999999998,"15min":4.1619999999999999},"min":{"1min":0.32700000000000001,"5min":0.22900000000000001,"15min":0.19800000000000001},"max":{"1min":9.0839999999999996,"5min":97.605000000000004,"15min":194.851},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":2.048,"15min":3.1400000000000001},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":4.1029999999999998,"5min":14.183,"15min":200.654},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.966,"15min":3.5190000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":7.8339999999999996,"5min":8.2850000000000001,"15min":200.86600000000001},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":2.1389999999999998,"15min":2.1560000000000001},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":4.1870000000000003,"5min":14.263,"15min":80.885999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.048,"5min":1.722,"15min":2.2629999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":8.3689999999999998,"5min":8.4019999999999992,"15min":80.438999999999993},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":2.0179999999999998,"5min":1.853,"15min":2.8799999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3290000000000006,"5min":8.4309999999999992,"15min":118.056},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.603,"5min":2.0339999999999998,"15min":2.6360000000000001},"min":{"1min":0.42699999999999999,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":4.1580000000000004,"5min":14.308999999999999,"15min":118.274},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:43:23 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.784,"15min":2.165},"min":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.742,"5min":8.2490000000000006,"15min":22.879000000000001},"last":0.442},{"interface":"front","average":{"1min":2.145,"5min":2.254,"15min":2.5299999999999998},"min":{"1min":0.44500000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.3960000000000008,"5min":9.9190000000000005,"15min":26.416},"last":0.51300000000000001}]}]},{"osd":2,"up_from":128,"seq":549755814232,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073976,"kb_used_data":6640,"kb_used_omap":7,"kb_used_meta":1067320,"kb_avail":92671176,"statfs":{"total":95995035648,"available":94895284224,"internally_reserved":0,"allocated":6799360,"data_stored":4564103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1092935761},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":10.237,"5min":7.4219999999999997,"15min":7.1289999999999996},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":89.287000000000006,"5min":214.595,"15min":214.595},"last":0.46500000000000002},{"interface":"front","average":{"1min":9.4120000000000008,"5min":6.9500000000000002,"15min":6.7539999999999996},"min":{"1min":0.251,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":89.225999999999999,"5min":214.94300000000001,"15min":214.94300000000001},"last":0.53200000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":27.795000000000002,"5min":7.0369999999999999,"15min":6.0309999999999997},"min":{"1min":0.308,"5min":0.20499999999999999,"15min":0.20200000000000001},"max":{"1min":299.988,"5min":299.988,"15min":299.988},"last":0.51000000000000001},{"interface":"front","average":{"1min":28.745000000000001,"5min":7.2080000000000002,"15min":6.2800000000000002},"min":{"1min":0.25900000000000001,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":300.30500000000001,"5min":300.30500000000001,"15min":300.30500000000001},"last":0.49199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:43:40 2023","interfaces":[{"interface":"back","average":{"1min":7.4189999999999996,"5min":3.923,"15min":4.2000000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":74.317999999999998,"5min":74.317999999999998,"15min":91.644000000000005},"last":0.55600000000000005},{"interface":"front","average":{"1min":6.9880000000000004,"5min":3.7240000000000002,"15min":4.3120000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":74.355999999999995,"5min":74.355999999999995,"15min":91.685000000000002},"last":0.57599999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.99099999999999999,"15min":2.2810000000000001},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.9299999999999997,"5min":9.5679999999999996,"15min":79.643000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.198,"15min":2.5539999999999998},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.0910000000000002,"5min":7.0910000000000002,"15min":82.872},"last":0.85799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.107,"15min":2.3210000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":5.9809999999999999,"5min":9.7040000000000006,"15min":82.834999999999994},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.548,"5min":1.268,"15min":2.512},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.32000000000000001},"max":{"1min":7.1369999999999996,"5min":7.1369999999999996,"15min":82.921999999999997},"last":0.90300000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.403,"15min":3.2250000000000001},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.313},"max":{"1min":7.1669999999999998,"5min":7.1669999999999998,"15min":123.47799999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.28,"5min":1.4950000000000001,"15min":3.1960000000000002},"min":{"1min":0.377,"5min":0.36199999999999999,"15min":0.313},"max":{"1min":5.2359999999999998,"5min":9.3740000000000006,"15min":122.517},"last":0.871}]},{"osd":7,"last update":"Mon Oct 9 06:43:16 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":2.8210000000000002,"15min":3.2989999999999999},"min":{"1min":0.40699999999999997,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":5.2770000000000001,"5min":132.06399999999999,"15min":132.06399999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.198,"5min":2.6429999999999998,"15min":2.9399999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.34999999999999998},"max":{"1min":6.0099999999999998,"5min":132.196,"15min":132.196},"last":0.76400000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127429,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085268,"kb_used_data":174540,"kb_used_omap":2,"kb_used_meta":910717,"kb_avail":92659884,"statfs":{"total":95995035648,"available":94883721216,"internally_reserved":0,"allocated":178728960,"data_stored":176637782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":932574261},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6050000000000004,"5min":4.0170000000000003,"15min":4.4989999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19},"max":{"1min":95.281999999999996,"5min":95.281999999999996,"15min":161.821},"last":0.26600000000000001},{"interface":"front","average":{"1min":9.0289999999999999,"5min":4.3010000000000002,"15min":4.4269999999999996},"min":{"1min":0.21099999999999999,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":98.572000000000003,"5min":98.572000000000003,"15min":163.52000000000001},"last":0.372}]},{"osd":1,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":7.8460000000000001,"5min":4.0039999999999996,"15min":4.4359999999999999},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":102.468,"5min":102.468,"15min":184.012},"last":0.30199999999999999},{"interface":"front","average":{"1min":9.3949999999999996,"5min":4.399,"15min":5.2839999999999998},"min":{"1min":0.41399999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":102.58,"5min":102.58,"15min":184.703},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":2.9449999999999998,"15min":3.9870000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.9900000000000002,"5min":118.93899999999999,"15min":182.44900000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.6830000000000001,"5min":3.1070000000000002,"15min":4.3250000000000002},"min":{"1min":0.22,"5min":0.219,"15min":0.219},"max":{"1min":14.9,"5min":118.97199999999999,"15min":182.56399999999999},"last":0.35499999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":3.0329999999999999,"15min":3.3879999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.7390000000000008,"5min":133.80099999999999,"15min":178.22200000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.444,"5min":3.2650000000000001,"15min":3.4060000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7999999999999998,"5min":133.648,"15min":180.43600000000001},"last":0.52300000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.181,"15min":3.6440000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34000000000000002},"max":{"1min":7.8380000000000001,"5min":128.083,"15min":177.56299999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.2170000000000001,"5min":3.0569999999999999,"15min":3.0920000000000001},"min":{"1min":0.33300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":8.0570000000000004,"5min":128.11799999999999,"15min":177.59399999999999},"last":0.59299999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":1.895,"15min":2.7410000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.3710000000000004,"5min":10.505000000000001,"15min":130.39599999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6379999999999999,"15min":2.4279999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.5019999999999998,"5min":7.5949999999999998,"15min":130.67699999999999},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:42:40 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.4470000000000001,"15min":3.0619999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":9.6850000000000005,"5min":99.248000000000005,"15min":121.613},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.0249999999999999,"5min":2.6299999999999999,"15min":3.4569999999999999},"min":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":3.2690000000000001,"5min":99.457999999999998,"15min":121.559},"last":0.61299999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019466,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484680,"kb_used_data":5120,"kb_used_omap":0,"kb_used_meta":1479552,"kb_avail":92260472,"statfs":{"total":95995035648,"available":94474723328,"internally_reserved":0,"allocated":5242880,"data_stored":4163242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1515061248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":7.3730000000000002,"5min":4.1280000000000001,"15min":6.9119999999999999},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":68.421999999999997,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":7.258,"5min":4.3010000000000002,"15min":7.1159999999999997},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.23100000000000001},"max":{"1min":68.433000000000007,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":4.0229999999999997,"15min":4.2130000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.4320000000000004,"5min":163.392,"15min":163.392},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.6679999999999999,"5min":4.1260000000000003,"15min":4.3019999999999996},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":5.3970000000000002,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.52200000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":4.234,"15min":4.391},"min":{"1min":0.30399999999999999,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":6.6879999999999997,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.7450000000000001,"5min":4.2599999999999998,"15min":4.3609999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.0110000000000001,"5min":175.875,"15min":175.875},"last":0.57499999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:43:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":3.2170000000000001,"15min":6.0979999999999999},"min":{"1min":0.34799999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8239999999999998,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.984,"5min":3.0659999999999998,"15min":5.8730000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":7.3630000000000004,"5min":112.21899999999999,"15min":161.934},"last":0.47799999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:43:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.77400000000000002,"15min":0.95899999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.252},"max":{"1min":6.7869999999999999,"5min":6.7869999999999999,"15min":9.3979999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.91600000000000004,"15min":1.1180000000000001},"min":{"1min":0.30199999999999999,"5min":0.27700000000000002,"15min":0.26600000000000001},"max":{"1min":5.9820000000000002,"5min":5.9820000000000002,"15min":9.5009999999999994},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.651,"15min":2.2959999999999998},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":5.3819999999999997,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.087,"5min":1.708,"15min":2.4969999999999999},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":6.734,"5min":83.218000000000004,"15min":163.381},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.5900000000000001,"15min":1.9419999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.7619999999999996,"5min":76.081000000000003,"15min":106.801},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.552,"15min":1.9950000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.3529999999999998,"5min":76.283000000000001,"15min":106.827},"last":0.59299999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757831,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1488960,"kb_used_data":5368,"kb_used_omap":0,"kb_used_meta":1483584,"kb_avail":92256192,"statfs":{"total":95995035648,"available":94470340608,"internally_reserved":0,"allocated":5496832,"data_stored":4421638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519190016},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:21 2023","interfaces":[{"interface":"back","average":{"1min":5.4130000000000003,"5min":3.468,"15min":5.9489999999999998},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":62.884999999999998,"5min":66.799000000000007,"15min":294.89699999999999},"last":0.441},{"interface":"front","average":{"1min":5.452,"5min":3.7549999999999999,"15min":6.1589999999999998},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.215},"max":{"1min":62.195,"5min":66.614000000000004,"15min":294.30399999999997},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 06:43:07 2023","interfaces":[{"interface":"back","average":{"1min":5.0510000000000002,"5min":7.8769999999999998,"15min":7.0119999999999996},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.22},"max":{"1min":69.287999999999997,"5min":153.91300000000001,"15min":188.34100000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.4950000000000001,"5min":7.8289999999999997,"15min":7.0359999999999996},"min":{"1min":0.27400000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":67.245999999999995,"5min":154.023,"15min":188.839},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:43:07 2023","interfaces":[{"interface":"back","average":{"1min":12.33,"5min":6.3639999999999999,"15min":4.7140000000000004},"min":{"1min":0.39100000000000001,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":145.20400000000001,"5min":145.20400000000001,"15min":145.20400000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":12.356999999999999,"5min":6.5720000000000001,"15min":4.5369999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":145.71899999999999,"5min":145.71899999999999,"15min":145.71899999999999},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:43:22 2023","interfaces":[{"interface":"back","average":{"1min":6.8680000000000003,"5min":5.827,"15min":6.1989999999999998},"min":{"1min":0.371,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":63.578000000000003,"5min":220.41800000000001,"15min":220.41800000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":5.2649999999999997,"5min":5.7590000000000003,"15min":5.8719999999999999},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":62.113999999999997,"5min":220.45500000000001,"15min":220.45500000000001},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:42:56 2023","interfaces":[{"interface":"back","average":{"1min":5.2729999999999997,"5min":1.7669999999999999,"15min":1.696},"min":{"1min":0.26100000000000001,"5min":0.255,"15min":0.247},"max":{"1min":74.725999999999999,"5min":74.725999999999999,"15min":79.043000000000006},"last":0.27300000000000002},{"interface":"front","average":{"1min":5.2530000000000001,"5min":1.968,"15min":1.72},"min":{"1min":0.318,"5min":0.251,"15min":0.251},"max":{"1min":74.777000000000001,"5min":74.777000000000001,"15min":79.165999999999997},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:43:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":2.0339999999999998,"15min":1.867},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.251},"max":{"1min":4.492,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.165,"5min":1.8380000000000001,"15min":1.958},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.9690000000000003,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.38200000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:42:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.1719999999999999,"15min":1.577},"min":{"1min":0.33900000000000002,"5min":0.318,"15min":0.29099999999999998},"max":{"1min":4.718,"5min":11.146000000000001,"15min":78.890000000000001},"last":0.311},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.1180000000000001,"15min":1.4039999999999999},"min":{"1min":0.379,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":2.7069999999999999,"5min":9.8070000000000004,"15min":79.138999999999996},"last":0.33700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:43:46.219 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:43:46.219 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:43:46.219 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 2, 4, 6, 7, 3, 5, 0] 2023-10-09T06:43:46.219 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T06:43:46.220 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:46.238 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:46.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:46.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:46.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[212750]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:46.327 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:46.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[220921]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[220921]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[220921]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:46.328 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:46.345 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:46.372 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:46.415 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:46.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:46.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:46.521 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:46.548 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:46 smithi150 ceph-mon[194178]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:46 smithi150 ceph-mon[194178]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:46 smithi150 ceph-mon[194178]: failed to encode map e542 with expected crc 2023-10-09T06:43:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:46.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:47.177 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:43:47.200 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:43:47.216 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:43:47.236 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:43:47.255 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:43:47.274 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:43:47.294 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:43:47.312 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.313 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:43:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[212750]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:47.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2341577314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:47.651 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[220921]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:47.652 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2341577314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:47 smithi150 ceph-mon[194178]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:43:47.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2341577314' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:43:47.876 DEBUG:teuthology.orchestra.run.smithi107:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:43:48.080 INFO:tasks.workunit.client.0.smithi107.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:43:48.286 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:43:48.286 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:43:48.893 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:48.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:48.997 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:49.024 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:48 smithi107 ceph-mon[212750]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:49.024 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:48 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:49.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:48 smithi107 ceph-mon[220921]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:49.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:48 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:49.027 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:48 smithi150 ceph-mon[194178]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:49.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:48 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:51.122 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:50 smithi107 ceph-mon[212750]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:51.123 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:50 smithi107 ceph-mon[220921]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:51.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:50 smithi150 ceph-mon[194178]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:51.204 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:51.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:51.310 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:51.344 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:51.394 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:51.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:51.452 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:43:51.453 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:43:51.453 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:43:51.453 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:43:51.475 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:43:51.500 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:51.545 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:51.558 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:43:51.605 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:43:51.605 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:43:51.606 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:43:51.644 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:51.645 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:43:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:51.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:51.660 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T06:43:51.661 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:43:51.709 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:51.729 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T06:43:51.730 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T06:43:51.730 DEBUG:teuthology.orchestra.run.smithi107:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T06:43:51.748 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:51.780 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:51.831 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:51.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:51.865 INFO:tasks.workunit.client.0.smithi107.stdout:Running main() from gmock_main.cc 2023-10-09T06:43:51.866 INFO:tasks.workunit.client.0.smithi107.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T06:43:51.866 INFO:tasks.workunit.client.0.smithi107.stdout:[----------] Global test environment set-up. 2023-10-09T06:43:51.866 INFO:tasks.workunit.client.0.smithi107.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T06:43:51.937 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:51.965 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:52.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:43:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:43:52] "GET /metrics HTTP/1.1" 200 34056 "" "Prometheus/2.43.0" 2023-10-09T06:43:52.660 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:43:52.827 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:52.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:52.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:52 smithi107 ceph-mon[212750]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:52.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:52 smithi107 ceph-mon[220921]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:52.931 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:52.961 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:53.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:52 smithi150 ceph-mon[194178]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:53.212 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:43:53.497 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:43:53.609 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:53.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:53.715 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:43:53.726 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:53.754 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:53.793 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3131383788' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:53.794 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: failed to encode map e543 with expected crc 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3131383788' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:53.795 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:53.808 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T06:43:53.822 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_all_features (14 ms) 2023-10-09T06:43:53.822 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T06:43:53.837 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:43:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:53.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:43:53.895 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:53.942 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:53.972 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:54.021 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:54.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:54.033 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.copyup (212 ms) 2023-10-09T06:43:54.034 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T06:43:54.064 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.sparse_copyup (30 ms) 2023-10-09T06:43:54.064 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T06:43:54.072 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-10-09T06:43:54.072 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T06:43:54.084 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.add_remove_child (13 ms) 2023-10-09T06:43:54.084 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T06:43:54.114 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.directory_methods (29 ms) 2023-10-09T06:43:54.115 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.create 2023-10-09T06:43:54.127 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:43:54.140 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.create (27 ms) 2023-10-09T06:43:54.140 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T06:43:54.142 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T06:43:54.142 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T06:43:54.143 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-09T06:43:54.143 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T06:43:54.144 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-10-09T06:43:54.145 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T06:43:54.146 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-09T06:43:54.146 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T06:43:54.147 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-09T06:43:54.147 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T06:43:54.151 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-10-09T06:43:54.151 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: failed to encode map e543 with expected crc 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: osdmap e543: 8 total, 8 up, 8 in 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: failed to encode map e543 with expected crc 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: failed to encode map e543 with expected crc 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: failed to encode map e543 with expected crc 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: failed to encode map e543 with expected crc 2023-10-09T06:43:54.152 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3131383788' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:54.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T06:43:54.155 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:54.155 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-10-09T06:43:54.155 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T06:43:54.160 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-10-09T06:43:54.160 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T06:43:54.180 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-10-09T06:43:54.180 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T06:43:54.187 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-10-09T06:43:54.188 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T06:43:54.200 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:43:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:54.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:43:54.229 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2023-10-09T06:43:54.229 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T06:43:54.258 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2023-10-09T06:43:54.258 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T06:43:54.279 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-10-09T06:43:54.280 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T06:43:54.284 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-09T06:43:54.285 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T06:43:54.293 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2023-10-09T06:43:54.293 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T06:43:54.295 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-10-09T06:43:54.296 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T06:43:54.301 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2023-10-09T06:43:54.302 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T06:43:54.306 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:54.308 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2023-10-09T06:43:54.308 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T06:43:54.308 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T06:43:54.308 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T06:43:54.317 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_snap_add (8 ms) 2023-10-09T06:43:54.317 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T06:43:54.322 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-10-09T06:43:54.322 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T06:43:54.327 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T06:43:54.327 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T06:43:54.333 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:54.352 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2023-10-09T06:43:54.353 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T06:43:54.359 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2023-10-09T06:43:54.359 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T06:43:54.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:54.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:54.392 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2023-10-09T06:43:54.392 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T06:43:54.407 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-09T06:43:54.407 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T06:43:54.412 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:43:54.481 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:54.508 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:54.553 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:54.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:54.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:54.657 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:54.695 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: pgmap v1687: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.956 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[212750]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: pgmap v1687: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: failed to encode map e544 with expected crc 2023-10-09T06:43:54.957 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:54 smithi107 ceph-mon[220921]: failed to encode map e544 with expected crc 2023-10-09T06:43:55.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: pgmap v1687: 135 pgs: 6 creating+peering, 26 unknown, 103 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:43:55.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi107-639838-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T06:43:55.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: osdmap e544: 8 total, 8 up, 8 in 2023-10-09T06:43:55.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: failed to encode map e544 with expected crc 2023-10-09T06:43:55.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: failed to encode map e544 with expected crc 2023-10-09T06:43:55.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: failed to encode map e544 with expected crc 2023-10-09T06:43:55.156 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:54 smithi150 ceph-mon[194178]: failed to encode map e544 with expected crc 2023-10-09T06:43:55.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:54 smithi107 conmon[212706]: 2023-10-09T06:43:54.991+0000 7f7c794f9700 -1 mon.a@0(leader).osd e545 definitely_dead 0 2023-10-09T06:43:55.723 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:55.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:55.820 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.6. 2023-10-09T06:43:55.828 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:55.855 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:55.899 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:55.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:56.006 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1761069170' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.007 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[212750]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1761069170' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:55 smithi107 ceph-mon[220921]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:43:56.043 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:56.150 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:43:56.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:56.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:43:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: osdmap e545: 8 total, 8 up, 8 in 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: failed to encode map e545 with expected crc 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1761069170' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T06:43:56.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:55 smithi150 ceph-mon[194178]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T06:43:56.254 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:56.281 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:56.326 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:56.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:56.434 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:43:56.460 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:56.504 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:43:56.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:56.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:43:56.610 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:43:56.636 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:56.679 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:43:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:56.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:43:56.784 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:56.822 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: pgmap v1690: 135 pgs: 6 creating+peering, 17 unknown, 112 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail; 197 KiB/s rd, 105 KiB/s wr, 317 op/s 2023-10-09T06:43:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:56.905 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:56 smithi150 ceph-mon[194178]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: pgmap v1690: 135 pgs: 6 creating+peering, 17 unknown, 112 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail; 197 KiB/s rd, 105 KiB/s wr, 317 op/s 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.133 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[212750]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: pgmap v1690: 135 pgs: 6 creating+peering, 17 unknown, 112 active+clean; 582 KiB data, 10 GiB used, 705 GiB / 715 GiB avail; 197 KiB/s rd, 105 KiB/s wr, 317 op/s 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.134 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.135 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.135 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.135 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.135 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.135 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:56 smithi107 ceph-mon[220921]: failed to encode map e546 with expected crc 2023-10-09T06:43:57.289 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:57.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:57.394 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:43:57.428 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:57.687 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:43:57.687 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:43:57.687 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:43:57.687 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:43:57.687 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:43:57.699 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:43:57.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:57.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:43:57.805 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:43:57.840 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:57.852 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror_image_status (3446 ms) 2023-10-09T06:43:57.852 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T06:43:57.887 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:43:57.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:57.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:43:57.950 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror_image_map (97 ms) 2023-10-09T06:43:57.950 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T06:43:57.957 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-10-09T06:43:57.957 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T06:43:57.968 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-10-09T06:43:57.968 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T06:43:57.971 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T06:43:57.971 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T06:43:57.973 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-10-09T06:43:57.973 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T06:43:57.976 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.dir_add_already_existing (2 ms) 2023-10-09T06:43:57.976 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T06:43:57.982 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-10-09T06:43:57.982 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T06:43:57.985 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-10-09T06:43:57.985 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T06:43:57.989 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T06:43:57.989 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T06:43:57.991 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-10-09T06:43:57.991 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T06:43:57.993 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:57.996 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-09T06:43:57.996 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T06:43:57.997 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-10-09T06:43:57.997 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T06:43:58.002 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-09T06:43:58.002 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T06:43:58.004 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-10-09T06:43:58.004 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T06:43:58.008 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T06:43:58.008 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T06:43:58.010 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-10-09T06:43:58.010 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T06:43:58.012 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T06:43:58.012 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T06:43:58.014 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T06:43:58.014 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T06:43:58.017 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-10-09T06:43:58.017 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.023 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.023 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T06:43:58.023 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T06:43:58.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.024 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:57 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.024 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-09T06:43:58.025 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T06:43:58.025 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:58.025 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T06:43:58.026 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T06:43:58.125 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (99 ms) 2023-10-09T06:43:58.125 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T06:43:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T06:43:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: failed to encode map e547 with expected crc 2023-10-09T06:43:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:57 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:58.319 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (195 ms) 2023-10-09T06:43:58.320 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T06:43:58.322 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-10-09T06:43:58.322 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T06:43:58.324 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T06:43:58.324 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T06:43:58.332 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-10-09T06:43:58.332 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T06:43:58.358 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.op_features (26 ms) 2023-10-09T06:43:58.358 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T06:43:58.378 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-10-09T06:43:58.378 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T06:43:58.386 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T06:43:58.386 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T06:43:58.396 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2023-10-09T06:43:58.397 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T06:43:58.402 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-10-09T06:43:58.403 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T06:43:58.412 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.migration (9 ms) 2023-10-09T06:43:58.412 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T06:43:58.425 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-10-09T06:43:58.426 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: pgmap v1693: 135 pgs: 11 peering, 5 creating+peering, 13 unknown, 106 active+clean; 582 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 307 KiB/s rd, 160 KiB/s wr, 494 op/s 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: pgmap v1693: 135 pgs: 11 peering, 5 creating+peering, 13 unknown, 106 active+clean; 582 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 307 KiB/s rd, 160 KiB/s wr, 494 op/s 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:58 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: pgmap v1693: 135 pgs: 11 peering, 5 creating+peering, 13 unknown, 106 active+clean; 582 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 307 KiB/s rd, 160 KiB/s wr, 494 op/s 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: failed to encode map e548 with expected crc 2023-10-09T06:43:59.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:58 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T06:43:59.644 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:59.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:43:59.699 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:43:59.749 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:43:59.776 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:43:59.826 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1400 ms) 2023-10-09T06:43:59.826 INFO:tasks.workunit.client.0.smithi107.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T06:43:59.879 INFO:tasks.workunit.client.0.smithi107.stdout:[ OK ] TestClsRbd.sparsify (53 ms) 2023-10-09T06:43:59.989 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.991 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:59.992 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[212750]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.993 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: failed to encode map e549 with expected crc 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:43:59.994 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:43:59 smithi107 ceph-mon[220921]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:43:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:43:59.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:00.096 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.121 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:00.127 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:00.146 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: failed to encode map e549 with expected crc 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:00.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:43:59 smithi150 ceph-mon[194178]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T06:44:00.178 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:00.183 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:00.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:00.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:00.202 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:00.220 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:00.240 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:00.258 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:00.279 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:00.280 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.280 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.280 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.281 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.281 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.281 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.281 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.281 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:44:00.288 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:00.321 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:00.374 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:00.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:00.463 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:00.480 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:00.507 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:00.549 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:00.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:00.656 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:00.683 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:00.724 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:00.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:00.828 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:00.856 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:00.859 INFO:tasks.workunit.client.0.smithi107.stdout:[----------] 70 tests from TestClsRbd (6071 ms total) 2023-10-09T06:44:00.859 INFO:tasks.workunit.client.0.smithi107.stdout: 2023-10-09T06:44:00.859 INFO:tasks.workunit.client.0.smithi107.stdout:[----------] Global test environment tear-down 2023-10-09T06:44:00.859 INFO:tasks.workunit.client.0.smithi107.stdout:[==========] 70 tests from 1 test suite ran. (8996 ms total) 2023-10-09T06:44:00.859 INFO:tasks.workunit.client.0.smithi107.stdout:[ PASSED ] 70 tests. 2023-10-09T06:44:00.860 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:44:00.862 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:44:00.862 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:44:00.910 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:00.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: pgmap v1696: 135 pgs: 6 remapped+peering, 15 peering, 2 creating+peering, 112 active+clean; 582 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 148 op/s; 81 KiB/s, 2 objects/s recovering 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.931 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3957639242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1592657574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: pgmap v1696: 135 pgs: 6 remapped+peering, 15 peering, 2 creating+peering, 112 active+clean; 582 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 148 op/s; 81 KiB/s, 2 objects/s recovering 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.932 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: failed to encode map e550 with expected crc 2023-10-09T06:44:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3957639242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1592657574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:00.933 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:00.933 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T06:44:00.934 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:44:01.015 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:01.040 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:01.082 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:01.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:01.133 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: pgmap v1696: 135 pgs: 6 remapped+peering, 15 peering, 2 creating+peering, 112 active+clean; 582 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 148 op/s; 81 KiB/s, 2 objects/s recovering 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: failed to encode map e550 with expected crc 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3957639242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1592657574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:01.134 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:01.188 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:01.217 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:01.260 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:01.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:01.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:01.367 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:01.395 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:01.513 DEBUG:teuthology.parallel:result is None 2023-10-09T06:44:01.514 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:01.542 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:01.543 DEBUG:teuthology.orchestra.run.smithi107:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:01.544 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:01.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:01.565 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:01.565 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:44:01.565 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T06:44:01.565 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:44:01.566 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:44:01.566 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:44:01.566 DEBUG:teuthology.orchestra.run.smithi107:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:01.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:01.623 INFO:teuthology.orchestra.run.smithi107.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:44:01.623 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:01.648 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:01.676 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:01.679 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:01.680 DEBUG:teuthology.orchestra.run.smithi107:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:44:01.720 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:01.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:01.736 INFO:tasks.workunit:timeout=3h 2023-10-09T06:44:01.736 INFO:tasks.workunit:cleanup=True 2023-10-09T06:44:01.737 DEBUG:teuthology.orchestra.run.smithi107:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:44:01.794 INFO:tasks.workunit.client.0.smithi107.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T06:44:01.825 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:01.851 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:01.896 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:01.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:01.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.924 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:01.925 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:01.974 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:01.975 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: failed to encode map e551 with expected crc 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:01.976 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:02.004 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:02.034 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:02.080 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:02.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:02.185 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:02.214 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:02.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:02] "GET /metrics HTTP/1.1" 200 34066 "" "Prometheus/2.43.0" 2023-10-09T06:44:02.510 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 7, 3, 5, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:44:02.510 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:44:02.510 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T06:44:02.510 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T06:44:02.510 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:44:02.742 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:02.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:02.847 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:02.872 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:02 smithi150 ceph-mon[194178]: failed to encode map e551 with expected crc 2023-10-09T06:44:02.872 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:02 smithi150 ceph-mon[194178]: pgmap v1699: 103 pgs: 6 remapped+peering, 5 peering, 92 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2023-10-09T06:44:02.874 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:02.918 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:02.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:03.025 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:03.052 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:03.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:02 smithi107 ceph-mon[212750]: failed to encode map e551 with expected crc 2023-10-09T06:44:03.084 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:02 smithi107 ceph-mon[212750]: pgmap v1699: 103 pgs: 6 remapped+peering, 5 peering, 92 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2023-10-09T06:44:03.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:02 smithi107 ceph-mon[220921]: failed to encode map e551 with expected crc 2023-10-09T06:44:03.085 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:02 smithi107 ceph-mon[220921]: pgmap v1699: 103 pgs: 6 remapped+peering, 5 peering, 92 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2023-10-09T06:44:03.095 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:03.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:03.203 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:03.232 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:03.246 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:03.333 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:03.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:03.436 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:03.447 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:44:03.462 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:03.504 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:03.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:03.567 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:03.567 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:03.611 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:03.638 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:03.682 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:03.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:03.787 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:03.813 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:03.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:03 smithi107 ceph-mon[220921]: 1.7 scrub starts 2023-10-09T06:44:03.856 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:03 smithi107 ceph-mon[220921]: 1.7 scrub ok 2023-10-09T06:44:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:03 smithi107 ceph-mon[212750]: 1.7 scrub starts 2023-10-09T06:44:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:03 smithi107 ceph-mon[212750]: 1.7 scrub ok 2023-10-09T06:44:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:03 smithi150 ceph-mon[194178]: 1.7 scrub starts 2023-10-09T06:44:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:03 smithi150 ceph-mon[194178]: 1.7 scrub ok 2023-10-09T06:44:04.772 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:44:04.960 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:44:04.985 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:04 smithi150 ceph-mon[194178]: pgmap v1700: 103 pgs: 4 peering, 99 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T06:44:04.985 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:04 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:44:05.069 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:05.074 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:04 smithi107 ceph-mon[220921]: pgmap v1700: 103 pgs: 4 peering, 99 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T06:44:05.074 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:04 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:44:05.075 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:04 smithi107 ceph-mon[212750]: pgmap v1700: 103 pgs: 4 peering, 99 active+clean; 595 KiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-10-09T06:44:05.075 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:04 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T06:44:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:05.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:05.098 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:05.177 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:05.215 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:05.300 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:05.416 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:05.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:05.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:05.521 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:05.550 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:05.671 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:05.895 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:05.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:06.000 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:06.031 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:06.075 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:06.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:06.178 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:06.207 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:06.252 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:06.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:06.356 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:06.383 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:06.477 DEBUG:teuthology.orchestra.run.smithi107:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T06:44:06.493 INFO:tasks.workunit.client.0.smithi107.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T06:44:06.495 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:44:06.495 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T06:44:06.529 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:06.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:06.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:06.632 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:06.661 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:06.720 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:06.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:06.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:06.733 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T06:44:06.802 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T06:44:06.824 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:06.852 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:06.867 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:44:06.927 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T06:44:06.929 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:44:06.929 INFO:tasks.workunit.client.0.smithi107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T06:44:06.949 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:06.950 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:06 smithi150 ceph-mon[194178]: 1.6 scrub starts 2023-10-09T06:44:06.950 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:06 smithi150 ceph-mon[194178]: 1.6 scrub ok 2023-10-09T06:44:06.950 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:06 smithi150 ceph-mon[194178]: pgmap v1701: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T06:44:06.951 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:06 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:44:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:06.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:07.029 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:07.029 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:07.053 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:07.080 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:07.121 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:07.123 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T06:44:07.123 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T06:44:07.123 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[212750]: pgmap v1701: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T06:44:07.123 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:44:07.123 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[220921]: 1.6 scrub starts 2023-10-09T06:44:07.124 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[220921]: 1.6 scrub ok 2023-10-09T06:44:07.124 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[220921]: pgmap v1701: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-10-09T06:44:07.124 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:06 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:44:07.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:07.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:07.225 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:07.252 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:07.293 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:07.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:07.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:07.424 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:07.451 INFO:tasks.workunit.client.0.smithi107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T06:44:07.455 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2023-10-09T06:44:07.455 DEBUG:teuthology.orchestra.run.smithi107:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T06:44:07.512 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T06:44:07.513 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T06:44:07.513 DEBUG:teuthology.orchestra.run.smithi107:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T06:44:07.573 INFO:tasks.workunit.client.0.smithi107.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T06:44:07.573 INFO:tasks.workunit.client.0.smithi107.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T06:44:07.573 INFO:tasks.workunit.client.0.smithi107.stderr:+ TMPDIR=/tmp/rbd_import_export_644583 2023-10-09T06:44:07.573 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm -rf /tmp/rbd_import_export_644583 2023-10-09T06:44:07.573 INFO:tasks.workunit.client.0.smithi107.stderr:+ mkdir /tmp/rbd_import_export_644583 2023-10-09T06:44:07.574 INFO:tasks.workunit.client.0.smithi107.stderr:+ trap 'rm -rf /tmp/rbd_import_export_644583' INT TERM EXIT 2023-10-09T06:44:07.574 INFO:tasks.workunit.client.0.smithi107.stderr:+ mkdir foo.644583 2023-10-09T06:44:07.649 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:44:07.710 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import foo.644583 foo.dir 2023-10-09T06:44:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:07 smithi150 ceph-mon[194178]: failed to encode map e552 with expected crc 2023-10-09T06:44:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:07 smithi150 ceph-mon[194178]: failed to encode map e552 with expected crc 2023-10-09T06:44:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:07 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:44:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:07 smithi150 ceph-mon[194178]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:44:07.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:07.995 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:08.138 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: cannot import a directory 2023-10-09T06:44:08.139 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 0% complete...failed. 2023-10-09T06:44:08.139 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: import failed: (21) Is a directory 2023-10-09T06:44:08.141 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:08.141 INFO:tasks.workunit.client.0.smithi107.stderr:+ rmdir foo.644583 2023-10-09T06:44:08.142 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_644583/img bs=1k count=1 seek=10 2023-10-09T06:44:08.143 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records in 2023-10-09T06:44:08.143 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records out 2023-10-09T06:44:08.143 INFO:tasks.workunit.client.0.smithi107.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.4022e-05 s, 23.3 MB/s 2023-10-09T06:44:08.143 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_644583/img bs=1k count=10 seek=100 2023-10-09T06:44:08.143 INFO:tasks.workunit.client.0.smithi107.stderr:10+0 records in 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:10+0 records out 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:10240 bytes (10 kB, 10 KiB) copied, 4.7271e-05 s, 217 MB/s 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_644583/img bs=1k count=100 seek=1000 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:70+1 records in 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:70+1 records out 2023-10-09T06:44:08.144 INFO:tasks.workunit.client.0.smithi107.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000181089 s, 398 MB/s 2023-10-09T06:44:08.145 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_644583/img bs=1k seek=10000 2023-10-09T06:44:08.145 INFO:tasks.workunit.client.0.smithi107.stderr:139+1 records in 2023-10-09T06:44:08.145 INFO:tasks.workunit.client.0.smithi107.stderr:139+1 records out 2023-10-09T06:44:08.145 INFO:tasks.workunit.client.0.smithi107.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000293738 s, 488 MB/s 2023-10-09T06:44:08.145 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_644583/img bs=1k seek=100000 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[220921]: failed to encode map e552 with expected crc 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[220921]: failed to encode map e552 with expected crc 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[220921]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[212750]: failed to encode map e552 with expected crc 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[212750]: failed to encode map e552 with expected crc 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2705966966' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[212750]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T06:44:08.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:08.172 INFO:tasks.workunit.client.0.smithi107.stderr:70+1 records in 2023-10-09T06:44:08.172 INFO:tasks.workunit.client.0.smithi107.stderr:70+1 records out 2023-10-09T06:44:08.172 INFO:tasks.workunit.client.0.smithi107.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0259765 s, 2.8 MB/s 2023-10-09T06:44:08.172 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_644583/img bs=1k seek=1000000 2023-10-09T06:44:08.227 INFO:tasks.workunit.client.0.smithi107.stderr:189+1 records in 2023-10-09T06:44:08.227 INFO:tasks.workunit.client.0.smithi107.stderr:189+1 records out 2023-10-09T06:44:08.227 INFO:tasks.workunit.client.0.smithi107.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.0544957 s, 3.6 MB/s 2023-10-09T06:44:08.227 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm testimg 2023-10-09T06:44:08.263 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 0% complete...failed. 2023-10-09T06:44:08.263 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:44:08.266 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:08.266 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:09.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:08 smithi107 ceph-mon[212750]: pgmap v1703: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 8.3 KiB/s, 0 objects/s recovering 2023-10-09T06:44:09.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:08 smithi107 ceph-mon[220921]: pgmap v1703: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 8.3 KiB/s, 0 objects/s recovering 2023-10-09T06:44:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:08 smithi150 ceph-mon[194178]: pgmap v1703: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 8.3 KiB/s, 0 objects/s recovering 2023-10-09T06:44:09.445 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:09.445 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:09.538 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:09.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:09.642 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:09.672 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:10.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[212750]: 1.f scrub starts 2023-10-09T06:44:10.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[212750]: 1.f scrub ok 2023-10-09T06:44:10.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[212750]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:44:10.009 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[220921]: 1.f scrub starts 2023-10-09T06:44:10.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[220921]: 1.f scrub ok 2023-10-09T06:44:10.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[220921]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[220921]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:44:10.010 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:09 smithi107 ceph-mon[220921]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.017 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:10.020 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:10.025 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:44:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:10.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:10.039 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:10.076 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:10.097 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:10.117 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:10.126 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.136 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:09 smithi150 ceph-mon[194178]: 1.f scrub starts 2023-10-09T06:44:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:09 smithi150 ceph-mon[194178]: 1.f scrub ok 2023-10-09T06:44:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:09 smithi150 ceph-mon[194178]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:09 smithi150 ceph-mon[194178]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T06:44:10.153 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:09 smithi150 ceph-mon[194178]: failed to encode map e553 with expected crc 2023-10-09T06:44:10.156 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:10.156 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:10.178 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.179 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:44:10.206 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:10.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:10.314 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:10.340 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:10.394 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:10.435 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:10.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:10.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:10.499 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T06:44:10.500 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:10.503 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg /tmp/rbd_import_export_644583/img2 2023-10-09T06:44:10.541 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:10.609 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:10.896 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:10.899 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg - 2023-10-09T06:44:11.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: pgmap v1705: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:11.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3725222802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: pgmap v1705: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: failed to encode map e553 with expected crc 2023-10-09T06:44:11.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:44:11.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:10 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3725222802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: pgmap v1705: 103 pgs: 103 active+clean; 595 KiB data, 8.5 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: failed to encode map e553 with expected crc 2023-10-09T06:44:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T06:44:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: failed to encode map e554 with expected crc 2023-10-09T06:44:11.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:10 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3725222802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:11.362 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:11.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:11.468 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:11.498 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:11.586 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:11.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:11.691 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:11.726 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:11.730 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:11.734 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm testimg 2023-10-09T06:44:11.825 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:44:11.828 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img2 2023-10-09T06:44:11.834 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:11.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:11.938 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:11.941 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:11.941 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:44:11.941 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:44:11.941 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:44:11.941 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.6 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.79321862400002, 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.4013005, 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:11.942 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:11.943 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.944 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:11.945 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144530+0000", 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144569+0000", 2023-10-09T06:44:11.946 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8152000000000003e-05 2023-10-09T06:44:11.947 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.947 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.947 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:11.947 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343804+0000", 2023-10-09T06:44:11.947 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19923497000000001 2023-10-09T06:44:11.948 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.948 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.948 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343819+0000", 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.579e-05 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.949 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344902+0000", 2023-10-09T06:44:11.950 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010829030000000001 2023-10-09T06:44:11.950 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.950 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.950 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:11.950 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345087+0000", 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000184298 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345267+0000", 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000180574 2023-10-09T06:44:11.951 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.952 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.952 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:11.952 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.355828+0000", 2023-10-09T06:44:11.952 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010561034 2023-10-09T06:44:11.952 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545825+0000", 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.189996794 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.953 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545832+0000", 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.954 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.16 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.59289773799998, 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.90554467699999996, 2023-10-09T06:44:11.955 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:44:11.956 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.957 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:11.957 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.957 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:11.957 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:44:11.957 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.958 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.962 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.962 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:11.962 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344851+0000", 2023-10-09T06:44:11.962 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:11.962 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344911+0000", 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0288999999999998e-05 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.963 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344924+0000", 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3145999999999999e-05 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344930+0000", 2023-10-09T06:44:11.964 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3849999999999999e-06 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545781+0000", 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20085007999999999 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.965 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545788+0000", 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5059999999999998e-06 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.966 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244178+0000", 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69839010700000004 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for degraded object", 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244920+0000", 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00074191700000000003 2023-10-09T06:44:11.967 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250315+0000", 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053952330000000001 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:11.968 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250397+0000", 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2120000000000007e-05 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.969 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98798 13.d 13:b3d56a11:::benchmark_data_smithi107_535588_object98797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.581635092, 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32981040700000003, 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:11.970 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98798 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:11.971 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.972 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356114+0000", 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:11.973 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:44:11.974 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3579000000000003e-05 2023-10-09T06:44:11.974 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.974 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.974 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:11.974 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356182+0000", 2023-10-09T06:44:11.975 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5062000000000001e-05 2023-10-09T06:44:11.975 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.975 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.975 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:11.975 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356191+0000", 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9409999999999996e-06 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356320+0000", 2023-10-09T06:44:11.976 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012913700000000001 2023-10-09T06:44:11.977 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.977 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.977 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:44:11.977 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356328+0000", 2023-10-09T06:44:11.977 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2300000000000002e-06 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578270+0000", 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.221941894 2023-10-09T06:44:11.978 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578431+0000", 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016151199999999999 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.979 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.579090+0000", 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00065903200000000004 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:11.980 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617475+0000", 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.038384553000000002 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617547+0000", 2023-10-09T06:44:11.981 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1931e-05 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685815+0000", 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.068268076999999996 2023-10-09T06:44:11.982 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685868+0000", 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3386e-05 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.983 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685926+0000", 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7726000000000003e-05 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:11.984 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98803 13.15 13:abc5b445:::benchmark_data_smithi107_535588_object98802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.369764073, 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.67498770900000005, 2023-10-09T06:44:11.985 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98803 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.986 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.987 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567984+0000", 2023-10-09T06:44:11.988 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568068+0000", 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4116999999999994e-05 2023-10-09T06:44:11.989 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.990 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.990 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:11.990 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568098+0000", 2023-10-09T06:44:11.990 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9153999999999999e-05 2023-10-09T06:44:11.991 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.991 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.991 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:11.991 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568108+0000", 2023-10-09T06:44:11.991 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0000000000000001e-05 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594900+0000", 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026791966 2023-10-09T06:44:11.992 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594910+0000", 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0601000000000001e-05 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.993 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231876+0000", 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.63696583200000001 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.994 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231989+0000", 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113386 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.232227+0000", 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00023801399999999999 2023-10-09T06:44:11.995 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235621+0000", 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033936639999999998 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:11.996 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235647+0000", 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5706000000000001e-05 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242883+0000", 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072361309999999998 2023-10-09T06:44:11.997 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242933+0000", 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0300000000000003e-05 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:11.998 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242974+0000", 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0917000000000001e-05 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:11.999 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.000 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.000 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98804 13.16 13:6bab783a:::benchmark_data_smithi107_535588_object98803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:12.000 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:44:12.000 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 435.25161661200002, 2023-10-09T06:44:12.000 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57256559900000004, 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:12.001 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98804 2023-10-09T06:44:12.002 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.002 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.002 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.002 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.002 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:44:12.003 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.004 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.004 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.004 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.004 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686133+0000", 2023-10-09T06:44:12.004 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686232+0000", 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9147999999999993e-05 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686245+0000", 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3025e-05 2023-10-09T06:44:12.005 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686253+0000", 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0690000000000002e-06 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686291+0000", 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8095e-05 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.006 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686300+0000", 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1460000000000002e-06 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244979+0000", 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.55867869000000003 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.007 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245112+0000", 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000133514 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245364+0000", 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000251501 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.008 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252198+0000", 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068343839999999998 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252224+0000", 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5789999999999999e-05 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.255637+0000", 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034128890000000001 2023-10-09T06:44:12.009 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258639+0000", 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003001709 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258699+0000", 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0885999999999999e-05 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.010 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:205773 13.6 13:624e0fc4:::benchmark_data_smithi107_535588_object99666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 308.41657092399998, 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31593875100000002, 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:12.011 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 205773 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.012 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521178+0000", 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521184+0000", 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.366e-06 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.013 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521194+0000", 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0015e-05 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521200+0000", 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8919999999999997e-06 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.014 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521246+0000", 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6483000000000002e-05 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521364+0000", 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117922 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521840+0000", 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047612499999999999 2023-10-09T06:44:12.015 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.524383+0000", 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0025425209999999998 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.525153+0000", 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077040099999999999 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.016 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837031+0000", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31187785499999998 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837079+0000", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8609999999999997e-05 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837118+0000", 2023-10-09T06:44:12.017 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8887999999999998e-05 2023-10-09T06:44:12.018 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.018 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.018 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.018 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.018 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.019 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165426 14.11 14:89616edc:::benchmark_data_smithi107_590976_object64653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.019 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:44:12.019 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 97.032761206999993, 2023-10-09T06:44:12.019 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.033732128, 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165426 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.020 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:12.021 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904991+0000", 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0470000000000001e-06 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904996+0000", 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1580000000000004e-06 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.022 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904999+0000", 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6850000000000001e-06 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.905040+0000", 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0907e-05 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.023 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.905046+0000", 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8409999999999996e-06 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.911594+0000", 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0065488229999999 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.024 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.911687+0000", 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2645999999999994e-05 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.912085+0000", 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000397558 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919179+0000", 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0070941600000000004 2023-10-09T06:44:12.025 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938564+0000", 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019385083000000001 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938629+0000", 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5233000000000004e-05 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.026 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938682+0000", 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2992999999999999e-05 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938721+0000", 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9322000000000001e-05 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.027 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165446 14.11 14:8f96ca76:::benchmark_data_smithi107_590976_object64673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 97.009017292999999, 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0102008440000001, 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165446 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.028 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928732+0000", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928738+0000", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6089999999999997e-06 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928756+0000", 2023-10-09T06:44:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7442999999999999e-05 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928763+0000", 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1840000000000002e-06 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928844+0000", 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1043000000000003e-05 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.031 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928856+0000", 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1476e-05 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914241+0000", 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98538567899999996 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.032 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914346+0000", 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105036 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914586+0000", 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00023933799999999999 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919184+0000", 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045983209999999998 2023-10-09T06:44:12.033 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938791+0000", 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019606734000000001 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938843+0000", 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2163999999999997e-05 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.034 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938894+0000", 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0708999999999998e-05 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938934+0000", 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0425e-05 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.035 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165473 14.11 14:8e8a7a16:::benchmark_data_smithi107_590976_object64700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.970594238000004, 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97204445699999997, 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.036 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165473 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:44:12.037 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967155+0000", 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967161+0000", 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.327e-06 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.038 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967173+0000", 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2575000000000001e-05 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967178+0000", 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1460000000000004e-06 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.039 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967226+0000", 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7862999999999999e-05 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967236+0000", 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0349e-05 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916014+0000", 2023-10-09T06:44:12.040 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94877750699999996 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916115+0000", 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101289 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916309+0000", 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000193933 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.041 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922232+0000", 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0059224799999999999 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938972+0000", 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016740718000000002 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.042 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939114+0000", 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014144799999999999 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939161+0000", 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7512000000000002e-05 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939201+0000", 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9348999999999999e-05 2023-10-09T06:44:12.043 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165478 14.11 14:8e82682f:::benchmark_data_smithi107_590976_object64705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.970554505999999, 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97228993600000002, 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.044 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165478 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.045 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967198+0000", 2023-10-09T06:44:12.046 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4700000000000001e-06 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967206+0000", 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6839999999999996e-06 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967210+0000", 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.578e-06 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.047 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967278+0000", 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7393999999999999e-05 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967286+0000", 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4999999999999999e-06 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.048 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917507+0000", 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95022087 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917613+0000", 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010586699999999999 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917829+0000", 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000215978 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.049 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.924990+0000", 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071616329999999997 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939247+0000", 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014256863 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.050 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939406+0000", 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000159006 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939450+0000", 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3582999999999999e-05 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.051 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939486+0000", 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5973000000000003e-05 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165589 14.11 14:8ce7d8ce:::benchmark_data_smithi107_590976_object64816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.828882410999995, 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.83075238100000004, 2023-10-09T06:44:12.052 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165589 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.053 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.054 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108871+0000", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.534e-06 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108881+0000", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0372e-05 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108887+0000", 2023-10-09T06:44:12.055 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0599999999999996e-06 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108950+0000", 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2433000000000003e-05 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108960+0000", 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.04e-05 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.056 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919215+0000", 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.81025469800000005 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919288+0000", 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3063000000000002e-05 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.057 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919501+0000", 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00021323199999999999 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.925000+0000", 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0054988820000000001 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939289+0000", 2023-10-09T06:44:12.058 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0142892 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939550+0000", 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00026099400000000002 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939588+0000", 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7834999999999999e-05 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.059 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939620+0000", 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2251e-05 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165615 14.11 14:8ee05e4b:::benchmark_data_smithi107_590976_object64842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.802946175000002, 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.80491372500000002, 2023-10-09T06:44:12.060 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165615 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.061 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134803+0000", 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.062 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134809+0000", 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.614e-06 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134819+0000", 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9410000000000002e-06 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.063 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134826+0000", 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.686e-06 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134882+0000", 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6459999999999998e-05 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134890+0000", 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1890000000000007e-06 2023-10-09T06:44:12.064 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.920872+0000", 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.78598192099999997 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.920951+0000", 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8248e-05 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.065 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.921135+0000", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018456700000000001 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929219+0000", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080839090000000002 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939522+0000", 2023-10-09T06:44:12.066 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010303321000000001 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939652+0000", 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012991899999999999 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939688+0000", 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5663000000000001e-05 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.067 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939718+0000", 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0117999999999999e-05 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165695 14.11 14:8d5362c1:::benchmark_data_smithi107_590976_object64922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.716222043000002, 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71835888999999997, 2023-10-09T06:44:12.068 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165695 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.069 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221527+0000", 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.070 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221533+0000", 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8719999999999999e-06 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221546+0000", 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2943e-05 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.071 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221553+0000", 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6579999999999996e-06 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221623+0000", 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0397999999999999e-05 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221636+0000", 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3148e-05 2023-10-09T06:44:12.072 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922267+0000", 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.7006308 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922368+0000", 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010130500000000001 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.073 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922574+0000", 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020514700000000001 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929228+0000", 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006654755 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939749+0000", 2023-10-09T06:44:12.074 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010520659 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939819+0000", 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0282000000000002e-05 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939856+0000", 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7147999999999997e-05 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.075 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939887+0000", 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0889000000000002e-05 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165730 14.11 14:8a95b677:::benchmark_data_smithi107_590976_object64957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.664475135000004, 2023-10-09T06:44:12.076 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.666711689, 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165730 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.077 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273274+0000", 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.078 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273280+0000", 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8220000000000003e-06 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273293+0000", 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3315999999999999e-05 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.079 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273300+0000", 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6499999999999999e-06 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273360+0000", 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9812999999999998e-05 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.080 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273369+0000", 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8089999999999999e-06 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923653+0000", 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.65028459800000005 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923733+0000", 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9450999999999995e-05 2023-10-09T06:44:12.081 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923954+0000", 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022080600000000001 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933196+0000", 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0092422059999999993 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.082 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939791+0000", 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0065949349999999997 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939920+0000", 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012873800000000001 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.083 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939955+0000", 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5553000000000003e-05 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939987+0000", 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1987999999999999e-05 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.084 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165764 14.11 14:8ec3c7ab:::benchmark_data_smithi107_590976_object64991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.616648733999995, 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.61904860500000003, 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165764 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.085 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.086 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321101+0000", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321105+0000", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2169999999999996e-06 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321117+0000", 2023-10-09T06:44:12.087 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1663e-05 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321121+0000", 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5360000000000003e-06 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321166+0000", 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4274999999999999e-05 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.088 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321173+0000", 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6350000000000006e-06 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.925047+0000", 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.60387425900000002 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.089 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.926164+0000", 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011165070000000001 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.926373+0000", 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020852800000000001 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933203+0000", 2023-10-09T06:44:12.090 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006830806 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940022+0000", 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068189089999999997 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940079+0000", 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6690000000000001e-05 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.091 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940117+0000", 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8528000000000001e-05 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940150+0000", 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2891000000000001e-05 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.092 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165786 14.11 14:88b78780:::benchmark_data_smithi107_590976_object65013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.585149353000006, 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.58767717600000002, 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165786 2023-10-09T06:44:12.093 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.094 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352600+0000", 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352606+0000", 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9499999999999998e-06 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.095 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352619+0000", 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2561000000000001e-05 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352624+0000", 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1420000000000001e-06 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352683+0000", 2023-10-09T06:44:12.096 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9027000000000003e-05 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352693+0000", 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0054999999999999e-05 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.927842+0000", 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57514921399999996 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.097 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.927915+0000", 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2760000000000001e-05 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.928102+0000", 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000187524 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.098 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938023+0000", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0099206620000000002 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940052+0000", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020286380000000001 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940210+0000", 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000157882 2023-10-09T06:44:12.099 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940247+0000", 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7533000000000001e-05 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940278+0000", 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1272000000000001e-05 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.100 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165798 14.11 14:8d92ee1f:::benchmark_data_smithi107_590976_object65025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.571257404999997, 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57388538, 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.101 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165798 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.102 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366492+0000", 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366498+0000", 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.103 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366509+0000", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1209999999999999e-05 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366513+0000", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4410000000000003e-06 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366582+0000", 2023-10-09T06:44:12.104 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8691999999999999e-05 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366592+0000", 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6490000000000003e-06 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929266+0000", 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.562674919 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.105 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929342+0000", 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5185999999999995e-05 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929544+0000", 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020193000000000001 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.106 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938033+0000", 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0084898899999999999 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940183+0000", 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002149523 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940310+0000", 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012735900000000001 2023-10-09T06:44:12.107 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940347+0000", 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6547000000000001e-05 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940378+0000", 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1557000000000003e-05 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.108 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165900 14.11 14:8e1dafdb:::benchmark_data_smithi107_590976_object65127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.451054581999998, 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.45380924900000003, 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.109 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165900 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.110 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486695+0000", 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486701+0000", 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9829999999999999e-06 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.111 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486718+0000", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6121e-05 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486725+0000", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5519999999999999e-06 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486784+0000", 2023-10-09T06:44:12.112 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8655e-05 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486790+0000", 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6320000000000002e-06 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931232+0000", 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.44444197699999999 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.113 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931347+0000", 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000114487 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931582+0000", 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000235212 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.114 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938171+0000", 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0065892980000000004 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940410+0000", 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022390679999999999 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940438+0000", 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7515999999999999e-05 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.115 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940474+0000", 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6340000000000001e-05 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940505+0000", 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0802000000000001e-05 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.116 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165928 14.11 14:88753501:::benchmark_data_smithi107_590976_object65155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.418367528000005, 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.42130255999999999, 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165928 2023-10-09T06:44:12.117 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.118 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519382+0000", 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519388+0000", 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0050000000000001e-06 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.119 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519401+0000", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3086e-05 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519406+0000", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8119999999999998e-06 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519450+0000", 2023-10-09T06:44:12.120 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4354000000000003e-05 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519456+0000", 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1999999999999999e-06 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933244+0000", 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.41378736399999999 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.121 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933371+0000", 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012719200000000001 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933614+0000", 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000243494 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.122 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938178+0000", 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045638270000000003 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940536+0000", 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023584399999999998 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940617+0000", 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0304999999999998e-05 2023-10-09T06:44:12.123 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940655+0000", 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7996e-05 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940686+0000", 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0746e-05 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.124 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.125 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.125 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165988 14.11 14:88a9b60e:::benchmark_data_smithi107_590976_object65215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:44:12.126 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:44:12.126 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 96.332098205999998, 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.33515009800000001, 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165988 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:12.127 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:12.128 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605651+0000", 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605657+0000", 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3140000000000003e-06 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605669+0000", 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2645e-05 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.129 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605674+0000", 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0119999999999996e-06 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605727+0000", 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3069999999999998e-05 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:12.130 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605736+0000", 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3219999999999993e-06 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.934794+0000", 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.329058146 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.934909+0000", 2023-10-09T06:44:12.131 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115077 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.935148+0000", 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000239423 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938185+0000", 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030366360000000001 2023-10-09T06:44:12.132 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940564+0000", 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023789010000000001 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940717+0000", 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015333600000000001 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.133 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940757+0000", 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9394000000000003e-05 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940802+0000", 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5697e-05 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.134 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:12.135 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:12.135 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:11 smithi107 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:11 smithi107 ceph-mon[212750]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:11 smithi150 ceph-mon[194178]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:11 smithi150 ceph-mon[194178]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:11 smithi107 ceph-mon[220921]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.136 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:11 smithi107 ceph-mon[220921]: failed to encode map e553 with expected crc 2023-10-09T06:44:12.172 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:12.220 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:12.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:12.324 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:12.353 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:12] "GET /metrics HTTP/1.1" 200 34066 "" "Prometheus/2.43.0" 2023-10-09T06:44:12.564 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:12.565 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img3 2023-10-09T06:44:12.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:12.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:12.623 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:44:12.668 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:12.696 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:12.745 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:12.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:12.849 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:12.876 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:12.910 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:44:12.926 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:12.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:12.947 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:12 smithi107 ceph-mon[212750]: pgmap v1707: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 6.0 KiB/s wr, 0 op/s 2023-10-09T06:44:12.948 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:12.948 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:12 smithi107 ceph-mon[220921]: pgmap v1707: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 6.0 KiB/s wr, 0 op/s 2023-10-09T06:44:12.948 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:12.968 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:13.030 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:13.043 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img2 /tmp/rbd_import_export_644583/img3 2023-10-09T06:44:13.057 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:12 smithi150 ceph-mon[194178]: pgmap v1707: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 6.0 KiB/s wr, 0 op/s 2023-10-09T06:44:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:13.155 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:13.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:13.259 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:13.285 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:13.297 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format - testimg 2023-10-09T06:44:13.328 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:13.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:13.430 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:44:13.434 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:13.462 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:13.657 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:13.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:13.760 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:13.796 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:13.848 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:13.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:13.882 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:13.954 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:13.985 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:14.079 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:14.080 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:14.083 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg /tmp/rbd_import_export_644583/img2 2023-10-09T06:44:14.083 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:14.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:14.087 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:44:14.088 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.089 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:13 smithi107 ceph-mon[220921]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:44:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:44:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:44:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/667722120' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T06:44:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:13 smithi150 ceph-mon[194178]: failed to encode map e555 with expected crc 2023-10-09T06:44:14.188 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:14.222 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:14.326 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:14.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:14.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:14.380 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:14.383 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg - 2023-10-09T06:44:14.432 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:14.462 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:14.488 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 7, 3, 5, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:44:14.488 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:44:14.489 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 2, 4, 7, 3, 5, 0] 2023-10-09T06:44:14.489 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T06:44:14.507 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:14.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:14.564 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:14.564 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:44:14.564 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:44:14.564 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:14.611 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:14.639 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:14.693 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:14.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:14.796 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:14.825 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:14 smithi150 ceph-mon[194178]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:14 smithi150 ceph-mon[194178]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:14 smithi150 ceph-mon[194178]: pgmap v1709: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 90 KiB/s rd, 32 KiB/s wr, 42 op/s 2023-10-09T06:44:15.157 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:44:15.168 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:15.171 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm testimg 2023-10-09T06:44:15.252 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T06:44:15.255 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img2 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[212750]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[212750]: pgmap v1709: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 90 KiB/s rd, 32 KiB/s wr, 42 op/s 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[220921]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[220921]: failed to encode map e555 with expected crc 2023-10-09T06:44:15.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:14 smithi107 ceph-mon[220921]: pgmap v1709: 103 pgs: 103 active+clean; 1.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 90 KiB/s rd, 32 KiB/s wr, 42 op/s 2023-10-09T06:44:15.506 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:15.823 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T06:44:15.902 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img3 2023-10-09T06:44:16.367 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img2 /tmp/rbd_import_export_644583/img3 2023-10-09T06:44:16.832 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:16.833 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:44:16.833 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:44:16.833 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:44:16.833 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:16 smithi150 ceph-mon[194178]: pgmap v1710: 103 pgs: 103 active+clean; 4.1 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 168 KiB/s rd, 106 KiB/s wr, 42 op/s 2023-10-09T06:44:17.233 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:17.234 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[212750]: pgmap v1710: 103 pgs: 103 active+clean; 4.1 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 168 KiB/s rd, 106 KiB/s wr, 42 op/s 2023-10-09T06:44:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:17.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:17.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[220921]: pgmap v1710: 103 pgs: 103 active+clean; 4.1 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 168 KiB/s rd, 106 KiB/s wr, 42 op/s 2023-10-09T06:44:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:17.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:17.433 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:44:17.786 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]: dispatch 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]: dispatch 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]: dispatch 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]: dispatch 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]': finished 2023-10-09T06:44:18.185 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]': finished 2023-10-09T06:44:18.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]': finished 2023-10-09T06:44:18.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]': finished 2023-10-09T06:44:18.186 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:17 smithi150 ceph-mon[194178]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]: dispatch 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]: dispatch 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]: dispatch 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]: dispatch 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]': finished 2023-10-09T06:44:18.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]': finished 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]': finished 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]': finished 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[212750]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]: dispatch 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]: dispatch 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]: dispatch 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]: dispatch 2023-10-09T06:44:18.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.10", "id": [5, 2]}]': finished 2023-10-09T06:44:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.19", "id": [7, 1]}]': finished 2023-10-09T06:44:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.1a", "id": [4, 2]}]': finished 2023-10-09T06:44:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [3, 2]}]': finished 2023-10-09T06:44:18.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:17 smithi107 ceph-mon[220921]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T06:44:19.013 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:18 smithi150 ceph-mon[194178]: pgmap v1711: 103 pgs: 103 active+clean; 3.0 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 256 KiB/s rd, 122 KiB/s wr, 96 op/s 2023-10-09T06:44:19.013 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:18 smithi150 ceph-mon[194178]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.014 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:18 smithi150 ceph-mon[194178]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.014 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:18 smithi150 ceph-mon[194178]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.014 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:18 smithi150 ceph-mon[194178]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:44:19.216 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:19.216 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:44:19.216 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197860 12.14 12.69738a74 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.48850422500004, 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93658382699999998, 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.217 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197860 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:44:19.218 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725514+0000", 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725517+0000", 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2739999999999999e-06 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.219 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725526+0000", 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7979999999999995e-06 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725530+0000", 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4449999999999998e-06 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.220 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725576+0000", 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6060999999999998e-05 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.725583+0000", 2023-10-09T06:44:19.221 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5089999999999999e-06 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662088+0000", 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93650507699999996 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662098+0000", 2023-10-09T06:44:19.223 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0195e-05 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197873 12.14 12.b12e1f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.47664022900005, 2023-10-09T06:44:19.224 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92474845500000002, 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197873 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.225 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737378+0000", 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737377+0000", 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.226 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737383+0000", 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6949999999999997e-06 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737394+0000", 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0998e-05 2023-10-09T06:44:19.227 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737401+0000", 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9929999999999998e-06 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737462+0000", 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1781999999999999e-05 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.228 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.737471+0000", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0180000000000007e-06 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662119+0000", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.92464809999999997 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662127+0000", 2023-10-09T06:44:19.229 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0849999999999999e-06 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:197996 12.14 12.b91ea374 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.32393552300005, 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77207506199999998, 2023-10-09T06:44:19.230 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197996 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.231 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890083+0000", 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890081+0000", 2023-10-09T06:44:19.232 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.233 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890088+0000", 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0820000000000001e-06 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890106+0000", 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8141000000000001e-05 2023-10-09T06:44:19.234 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890112+0000", 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0440000000000001e-06 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890164+0000", 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1412000000000002e-05 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.235 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.890172+0000", 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0730000000000005e-06 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662151+0000", 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.77197900699999999 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.236 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.237 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662158+0000", 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9009999999999997e-06 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.238 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198013 12.14 12.e99f1f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.30867235300002, 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75683585900000006, 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198013 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.239 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905346+0000", 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.240 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905344+0000", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905352+0000", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8949999999999998e-06 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905369+0000", 2023-10-09T06:44:19.241 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6959000000000001e-05 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905378+0000", 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1300000000000007e-06 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905442+0000", 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4079999999999996e-05 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.242 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.905453+0000", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1319e-05 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662175+0000", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75672163800000003 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662182+0000", 2023-10-09T06:44:19.243 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6850000000000004e-06 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198020 12.14 12.481c23d4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.303714614, 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75190084000000001, 2023-10-09T06:44:19.244 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198020 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.245 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910304+0000", 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910302+0000", 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.246 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910309+0000", 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4549999999999998e-06 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910325+0000", 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5356999999999999e-05 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910332+0000", 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3660000000000003e-06 2023-10-09T06:44:19.247 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910385+0000", 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2348000000000001e-05 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.910392+0000", 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1810000000000004e-06 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.248 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662199+0000", 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.75180681100000002 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662205+0000", 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0079999999999999e-06 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:198050 12.14 12.2fad4614 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T06:44:19.249 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 555.26467011099999, 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71287966800000002, 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198050 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.250 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949348+0000", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949346+0000", 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:19.251 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949357+0000", 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0203999999999999e-05 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949379+0000", 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2098000000000001e-05 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.252 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949388+0000", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7950000000000002e-06 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949444+0000", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5989999999999998e-05 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:03.949454+0000", 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2890000000000003e-06 2023-10-09T06:44:19.253 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662221+0000", 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71276758399999995 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:35:04.662228+0000", 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5769999999999999e-06 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.254 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 443.07104312899997, 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.50336221199999998, 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:19.255 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.256 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143027+0000", 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1832000000000001e-05 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.257 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143033+0000", 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5590000000000002e-06 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143038+0000", 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7360000000000001e-06 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144824+0000", 2023-10-09T06:44:19.258 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017859289999999999 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144963+0000", 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000139577 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145090+0000", 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012642299999999999 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.259 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148188+0000", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030985750000000001 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.149512+0000", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013238119999999999 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646328+0000", 2023-10-09T06:44:19.260 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49681566199999999 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646337+0000", 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5510000000000005e-06 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98785 13.18 13:1808f816:::benchmark_data_smithi107_535588_object98784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:19.261 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 442.87310438200001, 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89694675400000001, 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98785 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.262 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340912+0000", 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:19.263 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340963+0000", 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0998999999999997e-05 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340979+0000", 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5761999999999999e-05 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.264 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340986+0000", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0770000000000002e-06 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545392+0000", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20440555299999999 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545401+0000", 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0270000000000001e-06 2023-10-09T06:44:19.265 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237781+0000", 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69237996599999996 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237861+0000", 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0128000000000003e-05 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.266 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 442.87301864, 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.87606833799999995, 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:44:19.267 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.268 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340999+0000", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341044+0000", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4756999999999999e-05 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341054+0000", 2023-10-09T06:44:19.269 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7010000000000008e-06 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341058+0000", 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.459e-06 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341171+0000", 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011276 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.270 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341177+0000", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0839999999999997e-06 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647137+0000", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30595975600000003 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647149+0000", 2023-10-09T06:44:19.271 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2629000000000001e-05 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.216997+0000", 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.56984790500000004 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.217068+0000", 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0773999999999997e-05 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.272 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178397 14.7 14:e0fa40a8:::benchmark_data_smithi107_590976_object77624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.813105708999998, 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10589730999999999, 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.273 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178397 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.274 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400911+0000", 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400918+0000", 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3579999999999997e-06 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.275 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400934+0000", 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5434000000000002e-05 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400939+0000", 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3739999999999997e-06 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400983+0000", 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3353000000000003e-05 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.276 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401109+0000", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012683599999999999 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401585+0000", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047565699999999999 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.404448+0000", 2023-10-09T06:44:19.277 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0028627769999999999 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.407749+0000", 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003300716 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506743+0000", 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.098994487000000006 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.278 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506784+0000", 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1288000000000002e-05 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506810+0000", 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5507000000000002e-05 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.279 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178401 14.17 14:ece702fc:::benchmark_data_smithi107_590976_object77628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.813069537999993, 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10584073200000001, 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178401 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.280 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400948+0000", 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400951+0000", 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0419999999999998e-06 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400956+0000", 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8930000000000004e-06 2023-10-09T06:44:19.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400959+0000", 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5969999999999999e-06 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400993+0000", 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4440999999999998e-05 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401093+0000", 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9690000000000003e-05 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401500+0000", 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00040725000000000001 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.404443+0000", 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0029427580000000002 2023-10-09T06:44:19.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.407560+0000", 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031173659999999999 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506722+0000", 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.099162103000000001 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506757+0000", 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4103000000000002e-05 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506790+0000", 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2941000000000002e-05 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.286 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178419 14.17 14:ebc07a88:::benchmark_data_smithi107_590976_object77646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.789522376999997, 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.082372165999999997, 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178419 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.287 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424495+0000", 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424501+0000", 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8969999999999996e-06 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424513+0000", 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2408999999999999e-05 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424519+0000", 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424555+0000", 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6338999999999999e-05 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424666+0000", 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011116900000000001 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.425113+0000", 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000446771 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427285+0000", 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021724420000000001 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.431654+0000", 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043690209999999998 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506811+0000", 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.075156195999999995 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506842+0000", 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0868000000000003e-05 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506868+0000", 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.658e-05 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178421 14.7 14:e5df54a1:::benchmark_data_smithi107_590976_object77648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.293 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.786251476000004, 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.082669139000000003, 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178421 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.294 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427765+0000", 2023-10-09T06:44:19.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427772+0000", 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9040000000000003e-06 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427793+0000", 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0621000000000001e-05 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427800+0000", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.956e-06 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427837+0000", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6757000000000001e-05 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427932+0000", 2023-10-09T06:44:19.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5500000000000004e-05 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.428346+0000", 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041347500000000001 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.430654+0000", 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023079559999999999 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.431882+0000", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00122803 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510384+0000", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.078502646999999995 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510412+0000", 2023-10-09T06:44:19.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7922000000000001e-05 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510436+0000", 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3734e-05 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178436 14.17 14:ec20489c:::benchmark_data_smithi107_590976_object77663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.300 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.695540567999998, 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.141909865, 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178436 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518477+0000", 2023-10-09T06:44:19.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518483+0000", 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7509999999999997e-06 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518494+0000", 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1367e-05 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518500+0000", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3770000000000001e-06 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518540+0000", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0073000000000001e-05 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518660+0000", 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119566 2023-10-09T06:44:19.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.519125+0000", 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00046468099999999999 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.523888+0000", 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047633709999999998 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.526865+0000", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002976682 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660288+0000", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13342374700000001 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660352+0000", 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3979000000000005e-05 2023-10-09T06:44:19.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660388+0000", 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5169000000000002e-05 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178444 14.7 14:e1dee3ca:::benchmark_data_smithi107_590976_object77671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:44:19.307 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 82.685691200999997, 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13265004399999999, 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178444 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:44:19.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528326+0000", 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528331+0000", 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3589999999999999e-06 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528344+0000", 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2918999999999999e-05 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528350+0000", 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8139999999999997e-06 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528419+0000", 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8691999999999999e-05 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528548+0000", 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012942800000000001 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.529066+0000", 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00051763099999999995 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.532147+0000", 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030808070000000001 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.532871+0000", 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072465400000000003 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660886+0000", 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.128015035 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660937+0000", 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0776999999999999e-05 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660977+0000", 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9959000000000001e-05 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184443 14.7 14:e06ab74b:::benchmark_data_smithi107_590976_object83670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:44:19.314 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 74.280839381999996, 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.084506680000000001, 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184443 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933177+0000", 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933183+0000", 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.049e-06 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933196+0000", 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605999999999999e-05 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933202+0000", 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5500000000000002e-06 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933237+0000", 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5765000000000001e-05 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933341+0000", 2023-10-09T06:44:19.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103628 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933801+0000", 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00046017999999999997 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.937489+0000", 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036878739999999998 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.939261+0000", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001771931 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017589+0000", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.078328362999999998 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017643+0000", 2023-10-09T06:44:19.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3695e-05 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017686+0000", 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2534000000000003e-05 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184742 14.7 14:e5ca66eb:::benchmark_data_smithi107_590976_object83969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:44:19.321 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 73.892694664999993, 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14423997699999999, 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184742 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321323+0000", 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:19.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321328+0000", 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1789999999999999e-06 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321340+0000", 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2048000000000001e-05 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321345+0000", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1329999999999998e-06 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321383+0000", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7926999999999998e-05 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321493+0000", 2023-10-09T06:44:19.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110463 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.322025+0000", 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00053155800000000001 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.324589+0000", 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0025637350000000001 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.334123+0000", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.009534486 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465475+0000", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.131351775 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465525+0000", 2023-10-09T06:44:19.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0402999999999998e-05 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465564+0000", 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8377999999999997e-05 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184743 14.17 14:ef0552c5:::benchmark_data_smithi107_590976_object83970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:44:19.328 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 73.892663538999997, 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14719207600000001, 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184743 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321354+0000", 2023-10-09T06:44:19.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321357+0000", 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6019999999999998e-06 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321363+0000", 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9950000000000002e-06 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321367+0000", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1180000000000002e-06 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321415+0000", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.833e-05 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321540+0000", 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012452399999999999 2023-10-09T06:44:19.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.322031+0000", 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00049152899999999999 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.324715+0000", 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026839500000000001 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.331930+0000", 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072146340000000002 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468466+0000", 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.136535939 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468511+0000", 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5204000000000003e-05 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468547+0000", 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5644e-05 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:197475 14.7 14:e72db5f8:::benchmark_data_smithi107_590976_object96702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 55.887915616999997, 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13116150100000001, 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197475 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.336 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326101+0000", 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326108+0000", 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8090000000000004e-06 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326123+0000", 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5167999999999999e-05 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326130+0000", 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3879999999999997e-06 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.328431+0000", 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002301127 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.328566+0000", 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013549100000000001 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.329085+0000", 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00051890300000000003 2023-10-09T06:44:19.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.334068+0000", 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049826990000000002 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.334082+0000", 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4457000000000001e-05 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457186+0000", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.123103543 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457224+0000", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8163e-05 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457264+0000", 2023-10-09T06:44:19.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0188000000000002e-05 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:198497 14.17 14:e9f032a0:::benchmark_data_smithi107_590976_object97724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 54.353866932999999, 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.18453463000000001, 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:19.343 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:19.344 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198497 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:19.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860150+0000", 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:19.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860160+0000", 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0236e-05 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860182+0000", 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1721000000000001e-05 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:19.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860191+0000", 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7790000000000007e-06 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860241+0000", 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9258999999999997e-05 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860345+0000", 2023-10-09T06:44:19.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104053 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860870+0000", 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00052574000000000002 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.863319+0000", 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024490380000000002 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.863763+0000", 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044367000000000001 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044608+0000", 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.180844857 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:19.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044649+0000", 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0763000000000003e-05 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044686+0000", 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7264000000000001e-05 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:19.351 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[212750]: pgmap v1711: 103 pgs: 103 active+clean; 3.0 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 256 KiB/s rd, 122 KiB/s wr, 96 op/s 2023-10-09T06:44:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[212750]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[212750]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:44:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[220921]: pgmap v1711: 103 pgs: 103 active+clean; 3.0 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 256 KiB/s rd, 122 KiB/s wr, 96 op/s 2023-10-09T06:44:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[220921]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[220921]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[220921]: failed to encode map e556 with expected crc 2023-10-09T06:44:19.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:18 smithi107 ceph-mon[220921]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T06:44:19.807 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:44:20.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:19 smithi150 ceph-mon[194178]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:19 smithi150 ceph-mon[194178]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:19 smithi150 ceph-mon[194178]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:19 smithi150 ceph-mon[194178]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:19 smithi150 ceph-mon[194178]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:44:20.155 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[212750]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:44:20.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[220921]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[220921]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[220921]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[220921]: failed to encode map e557 with expected crc 2023-10-09T06:44:20.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:19 smithi107 ceph-mon[220921]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T06:44:20.641 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd help export 2023-10-09T06:44:20.642 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -q export-format 2023-10-09T06:44:20.661 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_644583/img bs=1k count=10 seek=100 2023-10-09T06:44:20.862 INFO:tasks.workunit.client.0.smithi107.stderr:10+0 records in 2023-10-09T06:44:20.863 INFO:tasks.workunit.client.0.smithi107.stderr:10+0 records out 2023-10-09T06:44:20.863 INFO:tasks.workunit.client.0.smithi107.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2693e-05 s, 141 MB/s 2023-10-09T06:44:20.863 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:20.930 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:20.930 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:20.933 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap create testimg@snap 2023-10-09T06:44:21.087 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:21.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:21.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: failed to encode map e557 with expected crc 2023-10-09T06:44:21.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: pgmap v1714: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 290 KiB/s rd, 117 KiB/s wr, 78 op/s 2023-10-09T06:44:21.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.115 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:20 smithi150 ceph-mon[194178]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:44:21.190 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: failed to encode map e557 with expected crc 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: pgmap v1714: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 290 KiB/s rd, 117 KiB/s wr, 78 op/s 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.221 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[212750]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: failed to encode map e557 with expected crc 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: pgmap v1714: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 290 KiB/s rd, 117 KiB/s wr, 78 op/s 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: failed to encode map e558 with expected crc 2023-10-09T06:44:21.222 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:20 smithi107 ceph-mon[220921]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T06:44:21.224 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:21.235 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:21.343 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:21.431 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:21.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:21.535 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:21.564 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:21.577 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:21.577 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:44:21.578 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:44:21.578 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:21.693 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:21.718 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:21.745 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:21.745 INFO:tasks.workunit.client.0.smithi107.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:44:21.752 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T06:44:21.765 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:21.794 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:21.817 INFO:tasks.workunit.client.0.smithi107.stderr:++ rbd image-meta list testimg 2023-10-09T06:44:21.817 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:21.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:21.825 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:21.844 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:21.856 INFO:tasks.workunit.client.0.smithi107.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T06:44:21.856 INFO:tasks.workunit.client.0.smithi107.stderr: 2023-10-09T06:44:21.856 INFO:tasks.workunit.client.0.smithi107.stderr:Key Value 2023-10-09T06:44:21.857 INFO:tasks.workunit.client.0.smithi107.stderr:key1 value1' 2023-10-09T06:44:21.857 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:21.864 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:21.866 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.866 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.866 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.866 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.866 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.867 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.867 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.867 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:44:21.921 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:21.922 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:21.925 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_644583/img_v2 testimg_import 2023-10-09T06:44:21.954 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:22] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/801894943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[212750]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/801894943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:21 smithi107 ceph-mon[220921]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: failed to encode map e559 with expected crc 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/801894943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:21 smithi150 ceph-mon[194178]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T06:44:22.223 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:44:22.566 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:22.951 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:44:22.954 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout:rbd image 'testimg_import': 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: size 110 KiB in 1 objects 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: order 22 (4 MiB objects) 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: snapshot_count: 1 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: id: 926d5b287b91 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: block_name_prefix: rbd_data.926d5b287b91 2023-10-09T06:44:22.986 INFO:tasks.workunit.client.0.smithi107.stdout: format: 2 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: op_features: 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: flags: 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: create_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: access_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:22.987 INFO:tasks.workunit.client.0.smithi107.stdout: modify_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:22.989 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import@snap 2023-10-09T06:44:23.021 INFO:tasks.workunit.client.0.smithi107.stdout:rbd image 'testimg_import': 2023-10-09T06:44:23.021 INFO:tasks.workunit.client.0.smithi107.stdout: size 110 KiB in 1 objects 2023-10-09T06:44:23.021 INFO:tasks.workunit.client.0.smithi107.stdout: order 22 (4 MiB objects) 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: snapshot_count: 1 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: id: 926d5b287b91 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: block_name_prefix: rbd_data.926d5b287b91 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: format: 2 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: op_features: 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: flags: 2023-10-09T06:44:23.022 INFO:tasks.workunit.client.0.smithi107.stdout: create_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:23.023 INFO:tasks.workunit.client.0.smithi107.stdout: access_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:23.023 INFO:tasks.workunit.client.0.smithi107.stdout: modify_timestamp: Mon Oct 9 06:44:21 2023 2023-10-09T06:44:23.023 INFO:tasks.workunit.client.0.smithi107.stdout: protected: False 2023-10-09T06:44:23.024 INFO:tasks.workunit.client.0.smithi107.stderr:++ rbd image-meta list testimg_import 2023-10-09T06:44:23.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T06:44:23.063 INFO:tasks.workunit.client.0.smithi107.stderr: 2023-10-09T06:44:23.063 INFO:tasks.workunit.client.0.smithi107.stderr:Key Value 2023-10-09T06:44:23.063 INFO:tasks.workunit.client.0.smithi107.stderr:key1 value1' 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr: 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:Key Value 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr: 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:Key Value 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:key1 value1' ']' 2023-10-09T06:44:23.064 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg_import /tmp/rbd_import_export_644583/img_import 2023-10-09T06:44:23.096 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete...done. 2023-10-09T06:44:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:22 smithi150 ceph-mon[194178]: pgmap v1717: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 KiB/s rd, 767 B/s wr, 4 op/s 2023-10-09T06:44:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:22 smithi150 ceph-mon[194178]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:22 smithi150 ceph-mon[194178]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.099 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:22 smithi150 ceph-mon[194178]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.099 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img_import 2023-10-09T06:44:23.100 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/img /tmp/rbd_import_export_644583/img_import 2023-10-09T06:44:23.100 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/img --format %b 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[212750]: pgmap v1717: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 KiB/s rd, 767 B/s wr, 4 op/s 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[220921]: pgmap v1717: 103 pgs: 1 remapped+peering, 1 peering, 101 active+clean; 582 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 115 KiB/s rd, 767 B/s wr, 4 op/s 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[220921]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[220921]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:22 smithi107 ceph-mon[220921]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.164 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=24 2023-10-09T06:44:23.164 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/img_import --format %b 2023-10-09T06:44:23.165 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=224 2023-10-09T06:44:23.165 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=200 2023-10-09T06:44:23.165 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=200 2023-10-09T06:44:23.165 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T06:44:23.165 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_644583/img_snap 2023-10-09T06:44:23.199 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete...done. 2023-10-09T06:44:23.202 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_644583/img_snap_import 2023-10-09T06:44:23.215 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:23.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:23.235 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete...done. 2023-10-09T06:44:23.238 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/img_snap /tmp/rbd_import_export_644583/img_snap_import 2023-10-09T06:44:23.238 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/img_snap /tmp/rbd_import_export_644583/img_snap_import 2023-10-09T06:44:23.239 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/img_snap --format %b 2023-10-09T06:44:23.240 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=224 2023-10-09T06:44:23.240 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/img_snap_import --format %b 2023-10-09T06:44:23.241 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=224 2023-10-09T06:44:23.241 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:23.241 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:23.241 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:44:23.242 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:23.242 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_import 2023-10-09T06:44:23.242 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_snap 2023-10-09T06:44:23.243 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_snap_import 2023-10-09T06:44:23.243 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T06:44:23.318 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:23.352 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:23.565 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:23.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:23.669 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:23.697 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:23.766 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:23.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:23.869 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:23.899 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:23.947 INFO:tasks.workunit.client.0.smithi107.stderr: Removing snap: 100% complete...done. 2023-10-09T06:44:23.955 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg_import 2023-10-09T06:44:23.994 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:23.996 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: 1.1a scrub starts 2023-10-09T06:44:23.996 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: 1.1a scrub ok 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: failed to encode map e560 with expected crc 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: failed to encode map e561 with expected crc 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: failed to encode map e561 with expected crc 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:44:23.997 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:23 smithi150 ceph-mon[194178]: failed to encode map e561 with expected crc 2023-10-09T06:44:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:23.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:24.053 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:24.057 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap rm testimg@snap 2023-10-09T06:44:24.079 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:24.079 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:24.099 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:24.127 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: 1.1a scrub starts 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: 1.1a scrub ok 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: failed to encode map e560 with expected crc 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: failed to encode map e560 with expected crc 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:44:24.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[220921]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: 1.1a scrub ok 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: failed to encode map e560 with expected crc 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T06:44:24.299 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:23 smithi107 ceph-mon[212750]: failed to encode map e561 with expected crc 2023-10-09T06:44:24.376 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:24.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:24.480 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:24.507 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:24.651 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 conmon[212706]: 2023-10-09T06:44:24.474+0000 7f7c794f9700 -1 mon.a@0(leader).osd e562 definitely_dead 0 2023-10-09T06:44:24.711 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:24.713 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:44:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:24.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:24.817 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:24.848 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:24.892 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:24.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:24.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:24.965 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is set 2023-10-09T06:44:24.970 INFO:teuthology.orchestra.run.smithi107.stderr:marked out osd.0. 2023-10-09T06:44:24.996 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:25.021 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: pgmap v1720: 103 pgs: 1 remapped+peering, 102 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 41 KiB/s rd, 4.5 KiB/s wr, 51 op/s 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3287822697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:24 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.024 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:25.070 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:25.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:25.085 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:44:25.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: pgmap v1720: 103 pgs: 1 remapped+peering, 102 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 41 KiB/s rd, 4.5 KiB/s wr, 51 op/s 2023-10-09T06:44:25.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.111 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:44:25.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:44:25.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:44:25.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3287822697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.112 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.112 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: pgmap v1720: 103 pgs: 1 remapped+peering, 102 active+clean; 692 KiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 41 KiB/s rd, 4.5 KiB/s wr, 51 op/s 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: failed to encode map e562 with expected crc 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3287822697' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.113 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:24 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T06:44:25.176 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:25.202 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:25.244 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:25.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:25.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:25.348 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:25.375 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:25.720 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:25.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:25.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:25.823 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:25.853 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:25.988 INFO:tasks.workunit.client.0.smithi107.stderr: Removing snap: 100% complete...done. 2023-10-09T06:44:26.005 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm testimg 2023-10-09T06:44:26.099 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:26.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:26.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:26.102 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:26.105 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --order 20 /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[212750]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:25 smithi107 ceph-mon[220921]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.161 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:26.162 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:26.164 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:26.202 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:26.202 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:26.205 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_644583/img_v2 testimg_import 2023-10-09T06:44:26.226 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2131584105' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: osdmap e563: 8 total, 8 up, 6 in 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.227 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:25 smithi150 ceph-mon[194178]: failed to encode map e563 with expected crc 2023-10-09T06:44:26.230 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:26.264 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:44:26.268 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import 2023-10-09T06:44:26.269 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep order 2023-10-09T06:44:26.269 INFO:tasks.workunit.client.0.smithi107.stderr:+ awk '{print $2}' 2023-10-09T06:44:26.269 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 20 2023-10-09T06:44:26.305 INFO:tasks.workunit.client.0.smithi107.stdout:20 2023-10-09T06:44:26.305 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:26.306 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg_import 2023-10-09T06:44:26.307 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:26.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:26.388 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:26.412 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:26.418 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:26.421 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg 2023-10-09T06:44:26.442 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:26.522 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:26.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:26.530 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:26.533 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:26.581 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:26.585 INFO:tasks.workunit.client.0.smithi107.stderr:++ rbd info testimg 2023-10-09T06:44:26.585 INFO:tasks.workunit.client.0.smithi107.stderr:++ grep features 2023-10-09T06:44:26.616 INFO:teuthology.orchestra.run.smithi150.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:26.616 INFO:teuthology.orchestra.run.smithi150.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:26.624 INFO:tasks.workunit.client.0.smithi107.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T06:44:26.624 INFO:tasks.workunit.client.0.smithi107.stderr: op_features: ' 2023-10-09T06:44:26.624 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:26.626 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:26.655 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:26.661 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:26.664 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_644583/img_v2 testimg_import 2023-10-09T06:44:26.715 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:44:26.719 INFO:tasks.workunit.client.0.smithi107.stderr:++ rbd info testimg_import 2023-10-09T06:44:26.719 INFO:tasks.workunit.client.0.smithi107.stderr:++ grep features 2023-10-09T06:44:26.757 INFO:tasks.workunit.client.0.smithi107.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T06:44:26.757 INFO:tasks.workunit.client.0.smithi107.stderr: op_features: ' 2023-10-09T06:44:26.758 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' ' features: layering 2023-10-09T06:44:26.758 INFO:tasks.workunit.client.0.smithi107.stderr: op_features: ' '!=' ' features: layering 2023-10-09T06:44:26.758 INFO:tasks.workunit.client.0.smithi107.stderr: op_features: ' ']' 2023-10-09T06:44:26.758 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:26.758 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg_import 2023-10-09T06:44:26.802 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:26.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:26.824 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:26.827 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg 2023-10-09T06:44:26.893 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:26.896 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:26.906 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:26.932 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:26.982 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:26.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:27.086 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:27.114 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:27.157 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:27.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:27.241 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:27.244 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:27.260 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:27.282 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: pgmap v1723: 103 pgs: 103 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 136 KiB/s rd, 9.7 KiB/s wr, 176 op/s 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.286 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.287 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_644583/img_v2 testimg_import 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: pgmap v1723: 103 pgs: 103 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 136 KiB/s rd, 9.7 KiB/s wr, 176 op/s 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:26 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.288 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: pgmap v1723: 103 pgs: 103 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 136 KiB/s rd, 9.7 KiB/s wr, 176 op/s 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: osdmap e564: 8 total, 8 up, 6 in 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.300 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.301 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.301 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:26 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:27.313 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:44:27.350 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:44:27.354 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import 2023-10-09T06:44:27.354 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 'stripe unit' 2023-10-09T06:44:27.354 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T06:44:27.388 INFO:tasks.workunit.client.0.smithi107.stdout: stripe unit: 4 KiB 2023-10-09T06:44:27.389 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import 2023-10-09T06:44:27.389 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 'stripe count' 2023-10-09T06:44:27.389 INFO:tasks.workunit.client.0.smithi107.stderr:+ awk '{print $3}' 2023-10-09T06:44:27.389 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 1000 2023-10-09T06:44:27.424 INFO:tasks.workunit.client.0.smithi107.stdout:1000 2023-10-09T06:44:27.424 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/img_v2 2023-10-09T06:44:27.425 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg_import 2023-10-09T06:44:27.486 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:27.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:27.506 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:44:27.509 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg 2023-10-09T06:44:27.590 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:27.599 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T06:44:27.604 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_644583/img testimg 2023-10-09T06:44:27.617 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:27.665 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:27.668 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap create testimg@snap1 2023-10-09T06:44:27.671 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:27.713 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:27.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:27.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:27.741 INFO:tasks.workunit.client.0.smithi107.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:44:27.747 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap create testimg@snap2 2023-10-09T06:44:27.818 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:27.844 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:27.888 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:27.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:27.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:27.992 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:28.021 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:28.068 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:28.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:28.079 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:28.097 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:28.116 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:28.135 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e564 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[220921]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e564 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.153 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:27 smithi107 ceph-mon[212750]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e564 with expected crc 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: osdmap e565: 8 total, 8 up, 6 in 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: failed to encode map e565 with expected crc 2023-10-09T06:44:28.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:27 smithi150 ceph-mon[194178]: osdmap e566: 8 total, 8 up, 6 in 2023-10-09T06:44:28.155 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:28.172 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:28.173 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:28.192 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:28.201 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:28.210 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:28.211 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.211 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.211 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.212 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:44:28.295 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:28.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:28.342 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T06:44:28.398 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:28.425 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:28.467 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:28.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:28.571 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:28.597 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:28.759 INFO:tasks.workunit.client.0.smithi107.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T06:44:28.773 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap protect testimg@snap2 2023-10-09T06:44:28.818 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_644583/snap_protect 2023-10-09T06:44:28.861 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:28.865 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_644583/snap_protect testimg_import 2023-10-09T06:44:28.954 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:28.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:29.033 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: pgmap v1726: 103 pgs: 1 unknown, 9 peering, 93 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 109 KiB/s rd, 8.7 KiB/s wr, 142 op/s 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.034 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.035 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:29.035 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/939076313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:29.035 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:28 smithi150 ceph-mon[194178]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T06:44:29.058 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:29.085 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:29.131 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:29.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:29.192 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:29.237 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:29.252 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:29.252 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:44:29.253 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:44:29.253 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: pgmap v1726: 103 pgs: 1 unknown, 9 peering, 93 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 109 KiB/s rd, 8.7 KiB/s wr, 142 op/s 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/939076313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[220921]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: pgmap v1726: 103 pgs: 1 unknown, 9 peering, 93 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 109 KiB/s rd, 8.7 KiB/s wr, 142 op/s 2023-10-09T06:44:29.262 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: failed to encode map e566 with expected crc 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:29.263 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/939076313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:29.264 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:28 smithi107 ceph-mon[212750]: osdmap e567: 8 total, 8 up, 6 in 2023-10-09T06:44:29.264 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:29.336 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:29.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:29.440 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.470 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:29.515 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:29.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:29.619 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:29.648 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:29.796 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:29.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:29.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:29.900 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:29.929 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:29.942 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:44:29.971 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:29.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:30.076 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:30.103 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:30.206 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:30.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.260 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[212750]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.261 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.262 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:29 smithi107 ceph-mon[220921]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T06:44:30.308 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:30.310 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:30.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.336 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: failed to encode map e567 with expected crc 2023-10-09T06:44:30.337 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:29 smithi150 ceph-mon[194178]: osdmap e568: 8 total, 8 up, 6 in 2023-10-09T06:44:30.339 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:30.423 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:30.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:30.527 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:30.555 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:30.598 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:30.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:30.702 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:30.735 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:30.757 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T06:44:30.761 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import@snap1 2023-10-09T06:44:30.761 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 'protected: False' 2023-10-09T06:44:30.785 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:30.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:30.795 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 7, 3, 5] out_osds: [6, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T06:44:30.795 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T06:44:30.796 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T06:44:30.796 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:44:30.801 INFO:tasks.workunit.client.0.smithi107.stdout: protected: False 2023-10-09T06:44:30.802 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd info testimg_import@snap2 2023-10-09T06:44:30.802 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep 'protected: True' 2023-10-09T06:44:30.840 INFO:tasks.workunit.client.0.smithi107.stdout: protected: True 2023-10-09T06:44:30.841 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/snap_protect 2023-10-09T06:44:30.841 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T06:44:30.887 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T06:44:30.890 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:30.922 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:30.935 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap purge testimg 2023-10-09T06:44:30.976 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:30.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:31.080 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:31.107 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:31.110 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[212750]: pgmap v1729: 103 pgs: 4 active+undersized+remapped, 1 unknown, 11 peering, 87 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 226 KiB/s rd, 28 KiB/s wr, 296 op/s; 83 KiB/s, 2 keys/s, 4 objects/s recovering 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[212750]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[212750]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[220921]: pgmap v1729: 103 pgs: 4 active+undersized+remapped, 1 unknown, 11 peering, 87 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 226 KiB/s rd, 28 KiB/s wr, 296 op/s; 83 KiB/s, 2 keys/s, 4 objects/s recovering 2023-10-09T06:44:31.138 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[220921]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[220921]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.139 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:30 smithi107 ceph-mon[220921]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T06:44:31.179 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:31.293 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:31.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:30 smithi150 ceph-mon[194178]: pgmap v1729: 103 pgs: 4 active+undersized+remapped, 1 unknown, 11 peering, 87 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 226 KiB/s rd, 28 KiB/s wr, 296 op/s; 83 KiB/s, 2 keys/s, 4 objects/s recovering 2023-10-09T06:44:31.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:30 smithi150 ceph-mon[194178]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:30 smithi150 ceph-mon[194178]: failed to encode map e568 with expected crc 2023-10-09T06:44:31.295 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:30 smithi150 ceph-mon[194178]: osdmap e569: 8 total, 8 up, 6 in 2023-10-09T06:44:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:31.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:31.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:31.428 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:31.474 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:31.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:31.578 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:31.610 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:31.762 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:31.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:31.785 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:31.786 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:44:31.786 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:44:31.786 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:44:31.786 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:31.865 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:31.892 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:31.987 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is set 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:32] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[212750]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:32.279 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:32.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[220921]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[220921]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:32.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:32.280 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:32.312 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:32.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:32.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:31 smithi150 ceph-mon[194178]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:31 smithi150 ceph-mon[194178]: failed to encode map e569 with expected crc 2023-10-09T06:44:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:32.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:32.417 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:32.446 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:32.493 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:44:32.494 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:32.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:32.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:32.592 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:32.599 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:32.633 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:32.742 INFO:tasks.workunit.client.0.smithi107.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:44:32.755 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd snap purge testimg_import 2023-10-09T06:44:32.788 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:32.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:32.841 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:44:32.892 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:32.919 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:33.068 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:33.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:33.172 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:33.199 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:33.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: pgmap v1732: 103 pgs: 4 active+undersized+remapped, 2 peering, 97 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 384 KiB/s rd, 45 KiB/s wr, 511 op/s; 83 KiB/s, 3 keys/s, 6 objects/s recovering 2023-10-09T06:44:33.260 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T06:44:33.261 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:32 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:33.293 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:33.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: pgmap v1732: 103 pgs: 4 active+undersized+remapped, 2 peering, 97 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 384 KiB/s rd, 45 KiB/s wr, 511 op/s; 83 KiB/s, 3 keys/s, 6 objects/s recovering 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:33.297 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: pgmap v1732: 103 pgs: 4 active+undersized+remapped, 2 peering, 97 active+clean; 692 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 384 KiB/s rd, 45 KiB/s wr, 511 op/s; 83 KiB/s, 3 keys/s, 6 objects/s recovering 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/94792956' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:33.298 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: osdmap e570: 8 total, 8 up, 6 in 2023-10-09T06:44:33.299 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: failed to encode map e570 with expected crc 2023-10-09T06:44:33.299 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T06:44:33.299 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: osdmap e571: 8 total, 8 up, 6 in 2023-10-09T06:44:33.299 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:32 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:33.397 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T06:44:33.427 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:33.517 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:44:33.519 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:44:33.523 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T06:44:33.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:33.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T06:44:33.630 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T06:44:33.658 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:33.827 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T06:44:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:33.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T06:44:33.930 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:33.957 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:34.003 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:34.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:34.108 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:34.133 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[212750]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[212750]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[220921]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.209 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[220921]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.210 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[220921]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.210 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:33 smithi107 ceph-mon[220921]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T06:44:34.350 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:44:34.350 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:44:34.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:44:34.350 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:44:34.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180927 12.d 12:b6ff457f:::benchmark_data_smithi107_479504_object81316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 595.59902724300002, 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.35203043499999997, 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180927 2023-10-09T06:44:34.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748412+0000", 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748411+0000", 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748415+0000", 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6779999999999999e-06 2023-10-09T06:44:34.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748424+0000", 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0650000000000005e-06 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748429+0000", 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3270000000000002e-06 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748471+0000", 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2400000000000001e-05 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748563+0000", 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2088000000000003e-05 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:34.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.748643+0000", 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9326000000000006e-05 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.750567+0000", 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0019239070000000001 2023-10-09T06:44:34.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.751954+0000", 2023-10-09T06:44:34.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013873869999999999 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100346+0000", 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.34839218799999999 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100401+0000", 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4484999999999998e-05 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.100442+0000", 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1430000000000001e-05 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.359 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46340.0:180945 12.d 12:b29197fc:::benchmark_data_smithi107_479504_object81334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 595.56807869199997, 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32532718199999999, 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46340", 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 180945 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779360+0000", 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779359+0000", 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779366+0000", 2023-10-09T06:44:34.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8349999999999998e-06 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779380+0000", 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3978999999999999e-05 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779386+0000", 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2960000000000004e-06 2023-10-09T06:44:34.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779426+0000", 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0438000000000001e-05 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779510+0000", 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3070000000000003e-05 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.779582+0000", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2076999999999994e-05 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.782438+0000", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002856408 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:38.786572+0000", 2023-10-09T06:44:34.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0041337539999999999 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104626+0000", 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31805433399999999 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104660+0000", 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3751000000000003e-05 2023-10-09T06:44:34.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:34:39.104687+0000", 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7487999999999998e-05 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98783 13.e 13:7e44f156:::benchmark_data_smithi107_535588_object98782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:44:34.367 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.208529822, 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23612880999999999, 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98783 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138907+0000", 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:34.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138985+0000", 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8372000000000001e-05 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138998+0000", 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.254e-05 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139004+0000", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1290000000000001e-06 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139052+0000", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8241000000000001e-05 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139170+0000", 2023-10-09T06:44:34.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117379 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139302+0000", 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013261800000000001 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145027+0000", 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057249759999999997 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148343+0000", 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033158129999999999 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374933+0000", 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22659017100000001 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374992+0000", 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8931000000000002e-05 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.375038+0000", 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5528000000000002e-05 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.374 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.20841570599998, 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30779935400000002, 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139054+0000", 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000001e-05 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139062+0000", 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139066+0000", 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140814+0000", 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001748793 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140929+0000", 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.141055+0000", 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126563 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.146811+0000", 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005755418 2023-10-09T06:44:34.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446811+0000", 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29999972899999999 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446822+0000", 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1856e-05 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 458.09297338099998, 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93419914299999995, 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:44:34.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254463+0000", 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254519+0000", 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6208999999999997e-05 2023-10-09T06:44:34.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254547+0000", 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8109000000000001e-05 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254562+0000", 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5073e-05 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254724+0000", 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161974 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254729+0000", 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497899+0000", 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24316990899999999 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497910+0000", 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0874e-05 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188593+0000", 2023-10-09T06:44:34.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69068296699999998 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188665+0000", 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1802000000000001e-05 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98796 13.7 13:e44dc813:::benchmark_data_smithi107_535588_object98795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:34.387 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 457.99132377400002, 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84800129700000004, 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98796 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356113+0000", 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4719000000000002e-05 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356184+0000", 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6277e-05 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356192+0000", 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9789999999999993e-06 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498051+0000", 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14185953700000001 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498058+0000", 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2799999999999998e-06 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188683+0000", 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.690624601 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188792+0000", 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108497 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188939+0000", 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014728699999999999 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.191074+0000", 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021353840000000002 2023-10-09T06:44:34.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.203068+0000", 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011993634 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204019+0000", 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000951625 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204076+0000", 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6569e-05 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204116+0000", 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0346999999999998e-05 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98805 13.0 13:054831b9:::benchmark_data_smithi107_535588_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 457.66121982999999, 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.528813278, 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98805 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.396 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686214+0000", 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999952 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686248+0000", 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4001000000000002e-05 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686269+0000", 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0539000000000001e-05 2023-10-09T06:44:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686278+0000", 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2469999999999997e-06 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686339+0000", 2023-10-09T06:44:34.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0735e-05 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686348+0000", 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181836+0000", 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49548772400000002 2023-10-09T06:44:34.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181965+0000", 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000129448 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.182343+0000", 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037732500000000002 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.184651+0000", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023081030000000002 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.199956+0000", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015305375 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214937+0000", 2023-10-09T06:44:34.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014981213 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214993+0000", 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5884999999999998e-05 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.215032+0000", 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9060999999999998e-05 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.403 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207733 13.0 13:0665e03e:::benchmark_data_smithi107_535588_object101626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 327.63183738800001, 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26422807999999998, 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.404 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207733 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:44:34.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715600+0000", 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715606+0000", 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715622+0000", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6038000000000001e-05 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715630+0000", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2320000000000004e-06 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715686+0000", 2023-10-09T06:44:34.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6475999999999999e-05 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715825+0000", 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013932800000000001 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716377+0000", 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055127400000000003 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.722227+0000", 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0058503299999999999 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.964701+0000", 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24247376800000001 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979740+0000", 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015039132 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979785+0000", 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5225999999999997e-05 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979830+0000", 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4431000000000002e-05 2023-10-09T06:44:34.410 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211410 13.7 13:e70c994c:::benchmark_data_smithi107_535588_object105303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 322.677992614, 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267478086, 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211410 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669444+0000", 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669454+0000", 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8539999999999992e-06 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669466+0000", 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1838000000000001e-05 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669472+0000", 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0059999999999998e-06 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669508+0000", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669585+0000", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7124000000000002e-05 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670141+0000", 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055599200000000005 2023-10-09T06:44:34.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.674991+0000", 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048495869999999998 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936590+0000", 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159882800000001 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936837+0000", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024747099999999998 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936885+0000", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8071000000000001e-05 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936924+0000", 2023-10-09T06:44:34.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8995000000000002e-05 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211418 13.17 13:e9af5116:::benchmark_data_smithi107_535588_object105311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 322.67503363999998, 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.264762895, 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.418 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211418 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672404+0000", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672414+0000", 2023-10-09T06:44:34.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0522e-05 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672428+0000", 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3747999999999999e-05 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672435+0000", 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.878e-06 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672490+0000", 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5414999999999997e-05 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672574+0000", 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4127000000000003e-05 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:44:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673020+0000", 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044588000000000001 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676720+0000", 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036994749999999998 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936560+0000", 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25984011699999998 2023-10-09T06:44:34.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937086+0000", 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000526558 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937128+0000", 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1659999999999998e-05 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937168+0000", 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9950999999999997e-05 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211420 13.17 13:ef772928:::benchmark_data_smithi107_535588_object105313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 322.67419770100003, 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265498495, 2023-10-09T06:44:34.425 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211420 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673244+0000", 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7500000000000001e-06 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673253+0000", 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2239999999999995e-06 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673257+0000", 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4170000000000002e-06 2023-10-09T06:44:34.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675010+0000", 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017534130000000001 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675083+0000", 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2701999999999996e-05 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675377+0000", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029354000000000001 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676887+0000", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015100999999999999 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936617+0000", 2023-10-09T06:44:34.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.259729969 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938661+0000", 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020442260000000001 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938704+0000", 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3467000000000002e-05 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938740+0000", 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5256999999999998e-05 2023-10-09T06:44:34.432 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:6728 14.f 14:fd78970a:::benchmark_data_smithi107_590976_object6727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 286.49557722899999, 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24803666199999999, 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.433 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6728 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851861+0000", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851941+0000", 2023-10-09T06:44:34.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0459999999999999e-05 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851954+0000", 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605999999999999e-05 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851959+0000", 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8440000000000003e-06 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852021+0000", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1607999999999997e-05 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852156+0000", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135234 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852317+0000", 2023-10-09T06:44:34.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016078500000000001 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852928+0000", 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00061110299999999999 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.854096+0000", 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011680460000000001 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099820+0000", 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24572355200000001 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099856+0000", 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6050999999999998e-05 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099898+0000", 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2388000000000001e-05 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:53446 14.f 14:fdbc8938:::benchmark_data_smithi107_590976_object53445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 253.70552909899999, 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24989693599999999, 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 53446 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641908+0000", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642000+0000", 2023-10-09T06:44:34.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2066999999999998e-05 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642019+0000", 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8488999999999999e-05 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642027+0000", 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2989999999999997e-06 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642077+0000", 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0349999999999997e-05 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642204+0000", 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126422 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642347+0000", 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142903 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.644067+0000", 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017200099999999999 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.889696+0000", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245629034 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891721+0000", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020251079999999999 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891762+0000", 2023-10-09T06:44:34.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1298000000000003e-05 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891807+0000", 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.46e-05 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93215 14.f 14:fff2cbd4:::benchmark_data_smithi107_590976_object93214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:44:34.447 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 208.30095980300001, 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247730006, 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93215 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046477+0000", 2023-10-09T06:44:34.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046588+0000", 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110818 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046609+0000", 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0880999999999999e-05 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046617+0000", 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9980000000000003e-06 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046668+0000", 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0525000000000002e-05 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.451 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046819+0000", 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015156699999999999 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046943+0000", 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000123709 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050044+0000", 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031011939999999998 2023-10-09T06:44:34.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.291715+0000", 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24167045500000001 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294145+0000", 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024305669999999998 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294182+0000", 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6606999999999997e-05 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294209+0000", 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7245000000000001e-05 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93221 14.f 14:f8a7a189:::benchmark_data_smithi107_590976_object93220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 208.29696664299999, 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245878864, 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93221 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.455 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050471+0000", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050529+0000", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7805e-05 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050544+0000", 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5617999999999999e-05 2023-10-09T06:44:34.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050550+0000", 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8390000000000003e-06 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050596+0000", 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5843000000000001e-05 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050742+0000", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014604700000000001 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050897+0000", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015521900000000001 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.053948+0000", 2023-10-09T06:44:34.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030502860000000001 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.293048+0000", 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.239100441 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296266+0000", 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032179029999999998 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296321+0000", 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5438e-05 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296351+0000", 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9584999999999999e-05 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114305 14.10 e511/504 14:0e814216:::benchmark_data_smithi107_590976_object13532:head v 511'5874, mlcod=495'2568)", 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 184.256013336, 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26106436700000002, 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:44:34.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091424+0000", 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091437+0000", 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3426e-05 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091458+0000", 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.039e-05 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091468+0000", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0360000000000001e-05 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091572+0000", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103453 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091599+0000", 2023-10-09T06:44:34.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7172999999999999e-05 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352452+0000", 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26085362600000001 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352490+0000", 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7518999999999998e-05 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.466 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114306 14.10 e511/504 14:0c753e0c:::benchmark_data_smithi107_590976_object13533:head v 511'5875, mlcod=495'2568)", 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 184.25593928699999, 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26101210699999999, 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091499+0000", 2023-10-09T06:44:34.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091507+0000", 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5369999999999997e-06 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091524+0000", 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6308999999999999e-05 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091533+0000", 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4569999999999994e-06 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345321+0000", 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25378754399999998 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345349+0000", 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7840000000000001e-05 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352493+0000", 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071447630000000002 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352512+0000", 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8321e-05 2023-10-09T06:44:34.471 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:114323 14.f 14:f062eb52:::benchmark_data_smithi107_590976_object13550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 184.23390330999999, 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24122175000000001, 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.472 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 114323 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113533+0000", 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113543+0000", 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2560000000000008e-06 2023-10-09T06:44:34.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.342992+0000", 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22944947199999999 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343004+0000", 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1386000000000001e-05 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.475 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343062+0000", 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8409000000000003e-05 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343203+0000", 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014109999999999999 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343349+0000", 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000146308 2023-10-09T06:44:34.476 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.348700+0000", 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053506509999999997 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.351340+0000", 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026400450000000002 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354664+0000", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033237610000000002 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354715+0000", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1208000000000002e-05 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354757+0000", 2023-10-09T06:44:34.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2179999999999999e-05 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:147218 14.f 14:f23d7e87:::benchmark_data_smithi107_590976_object46445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 143.01706117200001, 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27446428099999998, 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.479 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 147218 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330376+0000", 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330384+0000", 2023-10-09T06:44:34.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2069999999999995e-06 2023-10-09T06:44:34.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330400+0000", 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5277999999999998e-05 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330406+0000", 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6150000000000002e-06 2023-10-09T06:44:34.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330460+0000", 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3185e-05 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330559+0000", 2023-10-09T06:44:34.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9193999999999999e-05 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330638+0000", 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9406999999999994e-05 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.337385+0000", 2023-10-09T06:44:34.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0067469380000000001 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.337410+0000", 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4589000000000001e-05 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604752+0000", 2023-10-09T06:44:34.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26734247799999999 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604800+0000", 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8211000000000003e-05 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604842+0000", 2023-10-09T06:44:34.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1588000000000002e-05 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:166259 14.f 14:fe8f635d:::benchmark_data_smithi107_590976_object65486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 118.109778546, 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22879331999999999, 2023-10-09T06:44:34.488 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 166259 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:44:34.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.490 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237664+0000", 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9009999999999996e-06 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237670+0000", 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8870000000000003e-06 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237675+0000", 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7380000000000002e-06 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241213+0000", 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0035377899999999999 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241367+0000", 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000153726 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241498+0000", 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013085899999999999 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.244495+0000", 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002997132 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.247740+0000", 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032456429999999999 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466399+0000", 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.218658507 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:44:34.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466430+0000", 2023-10-09T06:44:34.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1588000000000003e-05 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466454+0000", 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3125999999999998e-05 2023-10-09T06:44:34.497 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.498 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.498 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.498 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:44:34.498 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:44:34.498 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:44:34.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:33 smithi150 ceph-mon[194178]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:33 smithi150 ceph-mon[194178]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:33 smithi150 ceph-mon[194178]: failed to encode map e571 with expected crc 2023-10-09T06:44:34.499 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:33 smithi150 ceph-mon[194178]: osdmap e572: 8 total, 8 up, 6 in 2023-10-09T06:44:34.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:34.500 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:34.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:34.501 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1734,"stamp":"2023-10-09T06:44:32.725361+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":821413,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":3477,"num_read_kb":4927,"num_write":553,"num_write_kb":1329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":681238,"num_keys_recovered":23,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5613,"ondisk_log_size":5613,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7663572,"kb_used_data":38040,"kb_used_omap":14,"kb_used_meta":7625393,"kb_avail":554807340,"statfs":{"total":575970213888,"available":568122716160,"internally_reserved":0,"allocated":38952960,"data_stored":28580230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15340,"internal_metadata":7808402452},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":112831,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":1363,"num_read_kb":1138,"num_write":176,"num_write_kb":136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":327715,"num_keys_recovered":2,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.994914"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":864,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737643+0000","last_change":"2023-10-09T06:39:15.093370+0000","last_active":"2023-10-09T06:44:30.737643+0000","last_peered":"2023-10-09T06:44:30.737643+0000","last_clean":"2023-10-09T06:44:30.737643+0000","last_became_active":"2023-10-09T06:37:08.252074+0000","last_became_peered":"2023-10-09T06:37:08.252074+0000","last_unstale":"2023-10-09T06:44:30.737643+0000","last_undegraded":"2023-10-09T06:44:30.737643+0000","last_fullsized":"2023-10-09T06:44:30.737643+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:15.093299+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:15.093299+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:19.030571+0000","scrub_duration":0.0018021689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":545,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964391+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:44:26.964391+0000","last_peered":"2023-10-09T06:44:26.964391+0000","last_clean":"2023-10-09T06:44:26.964391+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:44:26.964391+0000","last_undegraded":"2023-10-09T06:44:26.964391+0000","last_fullsized":"2023-10-09T06:44:26.964391+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":270,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.730855+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:44:27.730855+0000","last_peered":"2023-10-09T06:44:27.730855+0000","last_clean":"2023-10-09T06:44:27.730855+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:44:27.730855+0000","last_undegraded":"2023-10-09T06:44:27.730855+0000","last_fullsized":"2023-10-09T06:44:27.730855+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":833,"reported_epoch":566,"state":"peering","last_fresh":"2023-10-09T06:44:27.734725+0000","last_change":"2023-10-09T06:44:27.734933+0000","last_active":"2023-10-09T06:44:27.734705+0000","last_peered":"2023-10-09T06:44:26.976805+0000","last_clean":"2023-10-09T06:44:24.022512+0000","last_became_active":"2023-10-09T06:44:26.976558+0000","last_became_peered":"2023-10-09T06:44:26.976558+0000","last_unstale":"2023-10-09T06:44:27.734725+0000","last_undegraded":"2023-10-09T06:44:27.734725+0000","last_fullsized":"2023-10-09T06:44:27.734725+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:22:00.560448+0000","scrub_duration":0.0015879990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":567,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.742429+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:44:30.742429+0000","last_peered":"2023-10-09T06:44:30.742429+0000","last_clean":"2023-10-09T06:44:30.742429+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:44:30.742429+0000","last_undegraded":"2023-10-09T06:44:30.742429+0000","last_fullsized":"2023-10-09T06:44:30.742429+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":345,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.740009+0000","last_change":"2023-10-09T06:44:25.975180+0000","last_active":"2023-10-09T06:44:30.740009+0000","last_peered":"2023-10-09T06:44:30.740009+0000","last_clean":"2023-10-09T06:44:30.740009+0000","last_became_active":"2023-10-09T06:44:25.974863+0000","last_became_peered":"2023-10-09T06:44:25.974863+0000","last_unstale":"2023-10-09T06:44:30.740009+0000","last_undegraded":"2023-10-09T06:44:30.740009+0000","last_fullsized":"2023-10-09T06:44:30.740009+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:56.368743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":656,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737839+0000","last_change":"2023-10-09T06:44:22.448668+0000","last_active":"2023-10-09T06:44:30.737839+0000","last_peered":"2023-10-09T06:44:30.737839+0000","last_clean":"2023-10-09T06:44:30.737839+0000","last_became_active":"2023-10-09T06:44:21.741619+0000","last_became_peered":"2023-10-09T06:44:21.741619+0000","last_unstale":"2023-10-09T06:44:30.737839+0000","last_undegraded":"2023-10-09T06:44:30.737839+0000","last_fullsized":"2023-10-09T06:44:30.737839+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:22.448591+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:44:22.448591+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:29.967960+0000","scrub_duration":0.001903751,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":297,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.726474+0000","last_change":"2023-10-09T06:44:25.976383+0000","last_active":"2023-10-09T06:44:27.726474+0000","last_peered":"2023-10-09T06:44:27.726474+0000","last_clean":"2023-10-09T06:44:27.726474+0000","last_became_active":"2023-10-09T06:44:25.975323+0000","last_became_peered":"2023-10-09T06:44:25.975323+0000","last_unstale":"2023-10-09T06:44:27.726474+0000","last_undegraded":"2023-10-09T06:44:27.726474+0000","last_fullsized":"2023-10-09T06:44:27.726474+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:01:47.342770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":345,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.734340+0000","last_change":"2023-10-09T06:44:25.977280+0000","last_active":"2023-10-09T06:44:29.734340+0000","last_peered":"2023-10-09T06:44:29.734340+0000","last_clean":"2023-10-09T06:44:29.734340+0000","last_became_active":"2023-10-09T06:44:25.976975+0000","last_became_peered":"2023-10-09T06:44:25.976975+0000","last_unstale":"2023-10-09T06:44:29.734340+0000","last_undegraded":"2023-10-09T06:44:29.734340+0000","last_fullsized":"2023-10-09T06:44:29.734340+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:47.748942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":678,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738006+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:44:30.738006+0000","last_peered":"2023-10-09T06:44:30.738006+0000","last_clean":"2023-10-09T06:44:30.738006+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:44:30.738006+0000","last_undegraded":"2023-10-09T06:44:30.738006+0000","last_fullsized":"2023-10-09T06:44:30.738006+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":338,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.216450+0000","last_change":"2023-10-09T06:44:25.978227+0000","last_active":"2023-10-09T06:44:31.216450+0000","last_peered":"2023-10-09T06:44:31.216450+0000","last_clean":"2023-10-09T06:44:31.216450+0000","last_became_active":"2023-10-09T06:44:25.977894+0000","last_became_peered":"2023-10-09T06:44:25.977894+0000","last_unstale":"2023-10-09T06:44:31.216450+0000","last_undegraded":"2023-10-09T06:44:31.216450+0000","last_fullsized":"2023-10-09T06:44:31.216450+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:38.208258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":280,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.726758+0000","last_change":"2023-10-09T06:43:56.840302+0000","last_active":"2023-10-09T06:44:27.726758+0000","last_peered":"2023-10-09T06:44:27.726758+0000","last_clean":"2023-10-09T06:44:27.726758+0000","last_became_active":"2023-10-09T06:43:56.839988+0000","last_became_peered":"2023-10-09T06:43:56.839988+0000","last_unstale":"2023-10-09T06:44:27.726758+0000","last_undegraded":"2023-10-09T06:44:27.726758+0000","last_fullsized":"2023-10-09T06:44:27.726758+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:51:19.439096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":715,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738173+0000","last_change":"2023-10-09T06:32:12.825271+0000","last_active":"2023-10-09T06:44:30.738173+0000","last_peered":"2023-10-09T06:44:30.738173+0000","last_clean":"2023-10-09T06:44:30.738173+0000","last_became_active":"2023-10-09T06:32:03.461124+0000","last_became_peered":"2023-10-09T06:32:03.461124+0000","last_unstale":"2023-10-09T06:44:30.738173+0000","last_undegraded":"2023-10-09T06:44:30.738173+0000","last_fullsized":"2023-10-09T06:44:30.738173+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.825210+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.825210+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:42.464145+0000","scrub_duration":0.0021468540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.216211+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:44:31.216211+0000","last_peered":"2023-10-09T06:44:31.216211+0000","last_clean":"2023-10-09T06:44:31.216211+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:44:31.216211+0000","last_undegraded":"2023-10-09T06:44:31.216211+0000","last_fullsized":"2023-10-09T06:44:31.216211+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":280,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.733299+0000","last_change":"2023-10-09T06:43:56.839479+0000","last_active":"2023-10-09T06:44:27.733299+0000","last_peered":"2023-10-09T06:44:27.733299+0000","last_clean":"2023-10-09T06:44:27.733299+0000","last_became_active":"2023-10-09T06:43:56.838836+0000","last_became_peered":"2023-10-09T06:43:56.838836+0000","last_unstale":"2023-10-09T06:44:27.733299+0000","last_undegraded":"2023-10-09T06:44:27.733299+0000","last_fullsized":"2023-10-09T06:44:27.733299+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:03.428925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":707,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738317+0000","last_change":"2023-10-09T06:44:28.743130+0000","last_active":"2023-10-09T06:44:30.738317+0000","last_peered":"2023-10-09T06:44:30.738317+0000","last_clean":"2023-10-09T06:44:30.738317+0000","last_became_active":"2023-10-09T06:44:28.742803+0000","last_became_peered":"2023-10-09T06:44:28.742803+0000","last_unstale":"2023-10-09T06:44:30.738317+0000","last_undegraded":"2023-10-09T06:44:30.738317+0000","last_fullsized":"2023-10-09T06:44:30.738317+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:09.415456+0000","scrub_duration":0.001716845,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":451,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738536+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:44:30.738536+0000","last_peered":"2023-10-09T06:44:30.738536+0000","last_clean":"2023-10-09T06:44:30.738536+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:44:30.738536+0000","last_undegraded":"2023-10-09T06:44:30.738536+0000","last_fullsized":"2023-10-09T06:44:30.738536+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":526,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742654+0000","last_change":"2023-10-09T06:44:25.970430+0000","last_active":"2023-10-09T06:44:29.742654+0000","last_peered":"2023-10-09T06:44:29.742654+0000","last_clean":"2023-10-09T06:44:29.742654+0000","last_became_active":"2023-10-09T06:44:25.970017+0000","last_became_peered":"2023-10-09T06:44:25.970017+0000","last_unstale":"2023-10-09T06:44:29.742654+0000","last_undegraded":"2023-10-09T06:44:29.742654+0000","last_fullsized":"2023-10-09T06:44:29.742654+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:11:37.035533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":697,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742769+0000","last_change":"2023-10-09T06:44:28.740565+0000","last_active":"2023-10-09T06:44:29.742769+0000","last_peered":"2023-10-09T06:44:29.742769+0000","last_clean":"2023-10-09T06:44:29.742769+0000","last_became_active":"2023-10-09T06:44:28.739923+0000","last_became_peered":"2023-10-09T06:44:28.739923+0000","last_unstale":"2023-10-09T06:44:29.742769+0000","last_undegraded":"2023-10-09T06:44:29.742769+0000","last_fullsized":"2023-10-09T06:44:29.742769+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:18.179045+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":592,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978372+0000","last_change":"2023-10-09T06:43:56.875245+0000","last_active":"2023-10-09T06:44:26.978372+0000","last_peered":"2023-10-09T06:44:26.978372+0000","last_clean":"2023-10-09T06:44:26.978372+0000","last_became_active":"2023-10-09T06:43:56.834913+0000","last_became_peered":"2023-10-09T06:43:56.834913+0000","last_unstale":"2023-10-09T06:44:26.978372+0000","last_undegraded":"2023-10-09T06:44:26.978372+0000","last_fullsized":"2023-10-09T06:44:26.978372+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:52.397327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":546,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978157+0000","last_change":"2023-10-09T06:43:56.849031+0000","last_active":"2023-10-09T06:44:26.978157+0000","last_peered":"2023-10-09T06:44:26.978157+0000","last_clean":"2023-10-09T06:44:26.978157+0000","last_became_active":"2023-10-09T06:43:56.848339+0000","last_became_peered":"2023-10-09T06:43:56.848339+0000","last_unstale":"2023-10-09T06:44:26.978157+0000","last_undegraded":"2023-10-09T06:44:26.978157+0000","last_fullsized":"2023-10-09T06:44:26.978157+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:49.247759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":737,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.215824+0000","last_change":"2023-10-09T06:44:28.746265+0000","last_active":"2023-10-09T06:44:31.215824+0000","last_peered":"2023-10-09T06:44:31.215824+0000","last_clean":"2023-10-09T06:44:31.215824+0000","last_became_active":"2023-10-09T06:44:28.745985+0000","last_became_peered":"2023-10-09T06:44:28.745985+0000","last_unstale":"2023-10-09T06:44:31.215824+0000","last_undegraded":"2023-10-09T06:44:31.215824+0000","last_fullsized":"2023-10-09T06:44:31.215824+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:08.037325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:08.037325+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015898069999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":419,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738077+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:44:30.738077+0000","last_peered":"2023-10-09T06:44:30.738077+0000","last_clean":"2023-10-09T06:44:30.738077+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:44:30.738077+0000","last_undegraded":"2023-10-09T06:44:30.738077+0000","last_fullsized":"2023-10-09T06:44:30.738077+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":271,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.732079+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:44:27.732079+0000","last_peered":"2023-10-09T06:44:27.732079+0000","last_clean":"2023-10-09T06:44:27.732079+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:44:27.732079+0000","last_undegraded":"2023-10-09T06:44:27.732079+0000","last_fullsized":"2023-10-09T06:44:27.732079+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":882,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737639+0000","last_change":"2023-10-09T06:44:21.739983+0000","last_active":"2023-10-09T06:44:30.737639+0000","last_peered":"2023-10-09T06:44:30.737639+0000","last_clean":"2023-10-09T06:44:30.737639+0000","last_became_active":"2023-10-09T06:44:21.739627+0000","last_became_peered":"2023-10-09T06:44:21.739627+0000","last_unstale":"2023-10-09T06:44:30.737639+0000","last_undegraded":"2023-10-09T06:44:30.737639+0000","last_fullsized":"2023-10-09T06:44:30.737639+0000","mapping_epoch":559,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:01.130448+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":343,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738072+0000","last_change":"2023-10-09T06:44:25.973951+0000","last_active":"2023-10-09T06:44:30.738072+0000","last_peered":"2023-10-09T06:44:30.738072+0000","last_clean":"2023-10-09T06:44:30.738072+0000","last_became_active":"2023-10-09T06:44:25.973631+0000","last_became_peered":"2023-10-09T06:44:25.973631+0000","last_unstale":"2023-10-09T06:44:30.738072+0000","last_undegraded":"2023-10-09T06:44:30.738072+0000","last_fullsized":"2023-10-09T06:44:30.738072+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:21:49.412037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":390,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738336+0000","last_change":"2023-10-09T06:37:08.349694+0000","last_active":"2023-10-09T06:44:30.738336+0000","last_peered":"2023-10-09T06:44:30.738336+0000","last_clean":"2023-10-09T06:44:30.738336+0000","last_became_active":"2023-10-09T06:37:08.349397+0000","last_became_peered":"2023-10-09T06:37:08.349397+0000","last_unstale":"2023-10-09T06:44:30.738336+0000","last_undegraded":"2023-10-09T06:44:30.738336+0000","last_fullsized":"2023-10-09T06:44:30.738336+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:48.543349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":898,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742040+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:44:29.742040+0000","last_peered":"2023-10-09T06:44:29.742040+0000","last_clean":"2023-10-09T06:44:29.742040+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:44:29.742040+0000","last_undegraded":"2023-10-09T06:44:29.742040+0000","last_fullsized":"2023-10-09T06:44:29.742040+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":433,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738481+0000","last_change":"2023-10-09T06:44:25.973649+0000","last_active":"2023-10-09T06:44:30.738481+0000","last_peered":"2023-10-09T06:44:30.738481+0000","last_clean":"2023-10-09T06:44:30.738481+0000","last_became_active":"2023-10-09T06:44:25.973345+0000","last_became_peered":"2023-10-09T06:44:25.973345+0000","last_unstale":"2023-10-09T06:44:30.738481+0000","last_undegraded":"2023-10-09T06:44:30.738481+0000","last_fullsized":"2023-10-09T06:44:30.738481+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:08.952462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":324,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.734828+0000","last_change":"2023-10-09T06:44:25.972265+0000","last_active":"2023-10-09T06:44:28.734828+0000","last_peered":"2023-10-09T06:44:28.734828+0000","last_clean":"2023-10-09T06:44:28.734828+0000","last_became_active":"2023-10-09T06:44:25.971890+0000","last_became_peered":"2023-10-09T06:44:25.971890+0000","last_unstale":"2023-10-09T06:44:28.734828+0000","last_undegraded":"2023-10-09T06:44:28.734828+0000","last_fullsized":"2023-10-09T06:44:28.734828+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:20.481725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":704,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.216092+0000","last_change":"2023-10-09T06:43:59.838366+0000","last_active":"2023-10-09T06:44:31.216092+0000","last_peered":"2023-10-09T06:44:31.216092+0000","last_clean":"2023-10-09T06:44:31.216092+0000","last_became_active":"2023-10-09T06:43:59.838006+0000","last_became_peered":"2023-10-09T06:43:59.838006+0000","last_unstale":"2023-10-09T06:44:31.216092+0000","last_undegraded":"2023-10-09T06:44:31.216092+0000","last_fullsized":"2023-10-09T06:44:31.216092+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:16.500766+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":39,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.735615+0000","last_change":"2023-10-09T06:44:26.001182+0000","last_active":"2023-10-09T06:44:28.735615+0000","last_peered":"2023-10-09T06:44:28.735615+0000","last_clean":"2023-10-09T06:44:28.735615+0000","last_became_active":"2023-10-09T06:44:25.976522+0000","last_became_peered":"2023-10-09T06:44:25.976522+0000","last_unstale":"2023-10-09T06:44:28.735615+0000","last_undegraded":"2023-10-09T06:44:28.735615+0000","last_fullsized":"2023-10-09T06:44:28.735615+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:05.038744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":270,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.730634+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:44:27.730634+0000","last_peered":"2023-10-09T06:44:27.730634+0000","last_clean":"2023-10-09T06:44:27.730634+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:44:27.730634+0000","last_undegraded":"2023-10-09T06:44:27.730634+0000","last_fullsized":"2023-10-09T06:44:27.730634+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":697,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738095+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:44:30.738095+0000","last_peered":"2023-10-09T06:44:30.738095+0000","last_clean":"2023-10-09T06:44:30.738095+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:44:30.738095+0000","last_undegraded":"2023-10-09T06:44:30.738095+0000","last_fullsized":"2023-10-09T06:44:30.738095+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":469,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738230+0000","last_change":"2023-10-09T06:44:25.981587+0000","last_active":"2023-10-09T06:44:30.738230+0000","last_peered":"2023-10-09T06:44:30.738230+0000","last_clean":"2023-10-09T06:44:30.738230+0000","last_became_active":"2023-10-09T06:44:25.968087+0000","last_became_peered":"2023-10-09T06:44:25.968087+0000","last_unstale":"2023-10-09T06:44:30.738230+0000","last_undegraded":"2023-10-09T06:44:30.738230+0000","last_fullsized":"2023-10-09T06:44:30.738230+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:59.483832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":363,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737965+0000","last_change":"2023-10-09T06:43:56.832869+0000","last_active":"2023-10-09T06:44:30.737965+0000","last_peered":"2023-10-09T06:44:30.737965+0000","last_clean":"2023-10-09T06:44:30.737965+0000","last_became_active":"2023-10-09T06:43:56.832590+0000","last_became_peered":"2023-10-09T06:43:56.832590+0000","last_unstale":"2023-10-09T06:44:30.737965+0000","last_undegraded":"2023-10-09T06:44:30.737965+0000","last_fullsized":"2023-10-09T06:44:30.737965+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:54.863602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":997,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.742659+0000","last_change":"2023-10-09T06:44:28.742659+0000","last_active":"2023-10-09T06:44:28.742659+0000","last_peered":"2023-10-09T06:44:28.742659+0000","last_clean":"2023-10-09T06:44:28.742659+0000","last_became_active":"2023-10-09T06:44:28.742256+0000","last_became_peered":"2023-10-09T06:44:28.742256+0000","last_unstale":"2023-10-09T06:44:28.742659+0000","last_undegraded":"2023-10-09T06:44:28.742659+0000","last_fullsized":"2023-10-09T06:44:28.742659+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:43.147137+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":417,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.739936+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:44:30.739936+0000","last_peered":"2023-10-09T06:44:30.739936+0000","last_clean":"2023-10-09T06:44:30.739936+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:44:30.739936+0000","last_undegraded":"2023-10-09T06:44:30.739936+0000","last_fullsized":"2023-10-09T06:44:30.739936+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":573,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964386+0000","last_change":"2023-10-09T06:44:18.735683+0000","last_active":"2023-10-09T06:44:26.964386+0000","last_peered":"2023-10-09T06:44:26.964386+0000","last_clean":"2023-10-09T06:44:26.964386+0000","last_became_active":"2023-10-09T06:44:18.735354+0000","last_became_peered":"2023-10-09T06:44:18.735354+0000","last_unstale":"2023-10-09T06:44:26.964386+0000","last_undegraded":"2023-10-09T06:44:26.964386+0000","last_fullsized":"2023-10-09T06:44:26.964386+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.979649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":703,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978757+0000","last_change":"2023-10-09T06:44:08.762525+0000","last_active":"2023-10-09T06:44:26.978757+0000","last_peered":"2023-10-09T06:44:26.978757+0000","last_clean":"2023-10-09T06:44:26.978757+0000","last_became_active":"2023-10-09T06:43:59.841689+0000","last_became_peered":"2023-10-09T06:43:59.841689+0000","last_unstale":"2023-10-09T06:44:26.978757+0000","last_undegraded":"2023-10-09T06:44:26.978757+0000","last_fullsized":"2023-10-09T06:44:26.978757+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:08.762458+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:08.762458+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:21.916836+0000","scrub_duration":0.0013657389999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":487,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738250+0000","last_change":"2023-10-09T06:43:56.845809+0000","last_active":"2023-10-09T06:44:30.738250+0000","last_peered":"2023-10-09T06:44:30.738250+0000","last_clean":"2023-10-09T06:44:30.738250+0000","last_became_active":"2023-10-09T06:43:56.837327+0000","last_became_peered":"2023-10-09T06:43:56.837327+0000","last_unstale":"2023-10-09T06:44:30.738250+0000","last_undegraded":"2023-10-09T06:44:30.738250+0000","last_fullsized":"2023-10-09T06:44:30.738250+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:13:09.633037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":171,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742499+0000","last_change":"2023-10-09T06:43:56.834084+0000","last_active":"2023-10-09T06:44:29.742499+0000","last_peered":"2023-10-09T06:44:29.742499+0000","last_clean":"2023-10-09T06:44:29.742499+0000","last_became_active":"2023-10-09T06:43:56.833920+0000","last_became_peered":"2023-10-09T06:43:56.833920+0000","last_unstale":"2023-10-09T06:44:29.742499+0000","last_undegraded":"2023-10-09T06:44:29.742499+0000","last_fullsized":"2023-10-09T06:44:29.742499+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:07.548789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":690,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978424+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:44:26.978424+0000","last_peered":"2023-10-09T06:44:26.978424+0000","last_clean":"2023-10-09T06:44:26.978424+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:44:26.978424+0000","last_undegraded":"2023-10-09T06:44:26.978424+0000","last_fullsized":"2023-10-09T06:44:26.978424+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":148,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.732126+0000","last_change":"2023-10-09T06:43:56.835968+0000","last_active":"2023-10-09T06:44:27.732126+0000","last_peered":"2023-10-09T06:44:27.732126+0000","last_clean":"2023-10-09T06:44:27.732126+0000","last_became_active":"2023-10-09T06:43:56.835521+0000","last_became_peered":"2023-10-09T06:43:56.835521+0000","last_unstale":"2023-10-09T06:44:27.732126+0000","last_undegraded":"2023-10-09T06:44:27.732126+0000","last_fullsized":"2023-10-09T06:44:27.732126+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:39.815960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":306,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742275+0000","last_change":"2023-10-09T06:44:25.965645+0000","last_active":"2023-10-09T06:44:29.742275+0000","last_peered":"2023-10-09T06:44:29.742275+0000","last_clean":"2023-10-09T06:44:29.742275+0000","last_became_active":"2023-10-09T06:44:25.965051+0000","last_became_peered":"2023-10-09T06:44:25.965051+0000","last_unstale":"2023-10-09T06:44:29.742275+0000","last_undegraded":"2023-10-09T06:44:29.742275+0000","last_fullsized":"2023-10-09T06:44:29.742275+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:53.429386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":922,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737818+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:44:30.737818+0000","last_peered":"2023-10-09T06:44:30.737818+0000","last_clean":"2023-10-09T06:44:30.737818+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:44:30.737818+0000","last_undegraded":"2023-10-09T06:44:30.737818+0000","last_fullsized":"2023-10-09T06:44:30.737818+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":310,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.727007+0000","last_change":"2023-10-09T06:44:25.992472+0000","last_active":"2023-10-09T06:44:27.727007+0000","last_peered":"2023-10-09T06:44:27.727007+0000","last_clean":"2023-10-09T06:44:27.727007+0000","last_became_active":"2023-10-09T06:44:25.972946+0000","last_became_peered":"2023-10-09T06:44:25.972946+0000","last_unstale":"2023-10-09T06:44:27.727007+0000","last_undegraded":"2023-10-09T06:44:27.727007+0000","last_fullsized":"2023-10-09T06:44:27.727007+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:32:20.598002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":556,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978281+0000","last_change":"2023-10-09T06:43:56.838171+0000","last_active":"2023-10-09T06:44:26.978281+0000","last_peered":"2023-10-09T06:44:26.978281+0000","last_clean":"2023-10-09T06:44:26.978281+0000","last_became_active":"2023-10-09T06:43:56.837220+0000","last_became_peered":"2023-10-09T06:43:56.837220+0000","last_unstale":"2023-10-09T06:44:26.978281+0000","last_undegraded":"2023-10-09T06:44:26.978281+0000","last_fullsized":"2023-10-09T06:44:26.978281+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:27:12.146925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":603,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.726843+0000","last_change":"2023-10-09T06:39:26.873753+0000","last_active":"2023-10-09T06:44:27.726843+0000","last_peered":"2023-10-09T06:44:27.726843+0000","last_clean":"2023-10-09T06:44:27.726843+0000","last_became_active":"2023-10-09T06:37:08.345592+0000","last_became_peered":"2023-10-09T06:37:08.345592+0000","last_unstale":"2023-10-09T06:44:27.726843+0000","last_undegraded":"2023-10-09T06:44:27.726843+0000","last_fullsized":"2023-10-09T06:44:27.726843+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:39:26.873632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:20:10.335911+0000","scrub_duration":0.0020716710000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":94,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.974475+0000","last_change":"2023-10-09T06:43:56.838485+0000","last_active":"2023-10-09T06:44:26.974475+0000","last_peered":"2023-10-09T06:44:26.974475+0000","last_clean":"2023-10-09T06:44:26.974475+0000","last_became_active":"2023-10-09T06:43:56.838074+0000","last_became_peered":"2023-10-09T06:43:56.838074+0000","last_unstale":"2023-10-09T06:44:26.974475+0000","last_undegraded":"2023-10-09T06:44:26.974475+0000","last_fullsized":"2023-10-09T06:44:26.974475+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:26.366015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":543,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978645+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:44:26.978645+0000","last_peered":"2023-10-09T06:44:26.978645+0000","last_clean":"2023-10-09T06:44:26.978645+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:44:26.978645+0000","last_undegraded":"2023-10-09T06:44:26.978645+0000","last_fullsized":"2023-10-09T06:44:26.978645+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":924,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742122+0000","last_change":"2023-10-09T06:44:28.742866+0000","last_active":"2023-10-09T06:44:29.742122+0000","last_peered":"2023-10-09T06:44:29.742122+0000","last_clean":"2023-10-09T06:44:29.742122+0000","last_became_active":"2023-10-09T06:44:28.742474+0000","last_became_peered":"2023-10-09T06:44:28.742474+0000","last_unstale":"2023-10-09T06:44:29.742122+0000","last_undegraded":"2023-10-09T06:44:29.742122+0000","last_fullsized":"2023-10-09T06:44:29.742122+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:44:43.234538+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":547,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964250+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:44:26.964250+0000","last_peered":"2023-10-09T06:44:26.964250+0000","last_clean":"2023-10-09T06:44:26.964250+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:44:26.964250+0000","last_undegraded":"2023-10-09T06:44:26.964250+0000","last_fullsized":"2023-10-09T06:44:26.964250+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":536,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978610+0000","last_change":"2023-10-09T06:43:56.837589+0000","last_active":"2023-10-09T06:44:26.978610+0000","last_peered":"2023-10-09T06:44:26.978610+0000","last_clean":"2023-10-09T06:44:26.978610+0000","last_became_active":"2023-10-09T06:43:56.837269+0000","last_became_peered":"2023-10-09T06:43:56.837269+0000","last_unstale":"2023-10-09T06:44:26.978610+0000","last_undegraded":"2023-10-09T06:44:26.978610+0000","last_fullsized":"2023-10-09T06:44:26.978610+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:50:03.903213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":650,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964302+0000","last_change":"2023-10-09T06:44:12.819203+0000","last_active":"2023-10-09T06:44:26.964302+0000","last_peered":"2023-10-09T06:44:26.964302+0000","last_clean":"2023-10-09T06:44:26.964302+0000","last_became_active":"2023-10-09T06:43:59.839953+0000","last_became_peered":"2023-10-09T06:43:59.839953+0000","last_unstale":"2023-10-09T06:44:26.964302+0000","last_undegraded":"2023-10-09T06:44:26.964302+0000","last_fullsized":"2023-10-09T06:44:26.964302+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:12.819100+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:12.819100+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:01.244271+0000","scrub_duration":0.001698457,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":271,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.726684+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:44:27.726684+0000","last_peered":"2023-10-09T06:44:27.726684+0000","last_clean":"2023-10-09T06:44:27.726684+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:44:27.726684+0000","last_undegraded":"2023-10-09T06:44:27.726684+0000","last_fullsized":"2023-10-09T06:44:27.726684+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":526,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742370+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:44:29.742370+0000","last_peered":"2023-10-09T06:44:29.742370+0000","last_clean":"2023-10-09T06:44:29.742370+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:44:29.742370+0000","last_undegraded":"2023-10-09T06:44:29.742370+0000","last_fullsized":"2023-10-09T06:44:29.742370+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":612,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.732340+0000","last_change":"2023-10-09T06:44:03.276198+0000","last_active":"2023-10-09T06:44:27.732340+0000","last_peered":"2023-10-09T06:44:27.732340+0000","last_clean":"2023-10-09T06:44:27.732340+0000","last_became_active":"2023-10-09T06:43:59.839574+0000","last_became_peered":"2023-10-09T06:43:59.839574+0000","last_unstale":"2023-10-09T06:44:27.732340+0000","last_undegraded":"2023-10-09T06:44:27.732340+0000","last_fullsized":"2023-10-09T06:44:27.732340+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:03.276039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:03.276039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:52:48.574253+0000","scrub_duration":0.0023411270000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"567'78","reported_seq":827,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742359+0000","last_change":"2023-10-09T06:44:25.987529+0000","last_active":"2023-10-09T06:44:29.742359+0000","last_peered":"2023-10-09T06:44:29.742359+0000","last_clean":"2023-10-09T06:44:29.742359+0000","last_became_active":"2023-10-09T06:44:25.967053+0000","last_became_peered":"2023-10-09T06:44:25.967053+0000","last_unstale":"2023-10-09T06:44:29.742359+0000","last_undegraded":"2023-10-09T06:44:29.742359+0000","last_fullsized":"2023-10-09T06:44:29.742359+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":78,"log_dups_size":0,"ondisk_log_size":78,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:38.440053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000345001,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":338,"num_read_kb":772,"num_write":82,"num_write_kb":288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":2,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.5","version":"0'0","reported_seq":139,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.732300+0000","last_change":"2023-10-09T06:37:08.342927+0000","last_active":"2023-10-09T06:44:27.732300+0000","last_peered":"2023-10-09T06:44:27.732300+0000","last_clean":"2023-10-09T06:44:27.732300+0000","last_became_active":"2023-10-09T06:37:08.342675+0000","last_became_peered":"2023-10-09T06:37:08.342675+0000","last_unstale":"2023-10-09T06:44:27.732300+0000","last_undegraded":"2023-10-09T06:44:27.732300+0000","last_fullsized":"2023-10-09T06:44:27.732300+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:58.062801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":513,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742456+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:44:29.742456+0000","last_peered":"2023-10-09T06:44:29.742456+0000","last_clean":"2023-10-09T06:44:29.742456+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:44:29.742456+0000","last_undegraded":"2023-10-09T06:44:29.742456+0000","last_fullsized":"2023-10-09T06:44:29.742456+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":749,"reported_epoch":566,"state":"peering","last_fresh":"2023-10-09T06:44:27.733194+0000","last_change":"2023-10-09T06:44:27.733194+0000","last_active":"2023-10-09T06:44:25.955398+0000","last_peered":"2023-10-09T06:44:23.941010+0000","last_clean":"2023-10-09T06:44:23.941010+0000","last_became_active":"2023-10-09T06:37:08.296427+0000","last_became_peered":"2023-10-09T06:37:08.296427+0000","last_unstale":"2023-10-09T06:44:27.733194+0000","last_undegraded":"2023-10-09T06:44:27.733194+0000","last_fullsized":"2023-10-09T06:44:27.733194+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:31.102149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:31.102149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:18:27.122969+0000","scrub_duration":0.0015293380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"565'69","reported_seq":620,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.796334+0000","last_change":"2023-10-09T06:44:26.007555+0000","last_active":"2023-10-09T06:44:30.796334+0000","last_peered":"2023-10-09T06:44:30.796334+0000","last_clean":"2023-10-09T06:44:30.796334+0000","last_became_active":"2023-10-09T06:44:25.971925+0000","last_became_peered":"2023-10-09T06:44:25.971925+0000","last_unstale":"2023-10-09T06:44:30.796334+0000","last_undegraded":"2023-10-09T06:44:30.796334+0000","last_fullsized":"2023-10-09T06:44:30.796334+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:19.383868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002105979,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":416,"num_read_kb":533,"num_write":72,"num_write_kb":128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.3","version":"0'0","reported_seq":567,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978099+0000","last_change":"2023-10-09T06:44:25.970465+0000","last_active":"2023-10-09T06:44:26.978099+0000","last_peered":"2023-10-09T06:44:26.978099+0000","last_clean":"2023-10-09T06:44:26.978099+0000","last_became_active":"2023-10-09T06:44:25.970140+0000","last_became_peered":"2023-10-09T06:44:25.970140+0000","last_unstale":"2023-10-09T06:44:26.978099+0000","last_undegraded":"2023-10-09T06:44:26.978099+0000","last_fullsized":"2023-10-09T06:44:26.978099+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:50.761333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":543,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742265+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:44:29.742265+0000","last_peered":"2023-10-09T06:44:29.742265+0000","last_clean":"2023-10-09T06:44:29.742265+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:44:29.742265+0000","last_undegraded":"2023-10-09T06:44:29.742265+0000","last_fullsized":"2023-10-09T06:44:29.742265+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":932,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.737989+0000","last_change":"2023-10-09T06:44:28.743002+0000","last_active":"2023-10-09T06:44:30.737989+0000","last_peered":"2023-10-09T06:44:30.737989+0000","last_clean":"2023-10-09T06:44:30.737989+0000","last_became_active":"2023-10-09T06:44:28.742596+0000","last_became_peered":"2023-10-09T06:44:28.742596+0000","last_unstale":"2023-10-09T06:44:30.737989+0000","last_undegraded":"2023-10-09T06:44:30.737989+0000","last_fullsized":"2023-10-09T06:44:30.737989+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:49:00.447596+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"567'102","reported_seq":978,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.852570+0000","last_change":"2023-10-09T06:44:26.022096+0000","last_active":"2023-10-09T06:44:28.852570+0000","last_peered":"2023-10-09T06:44:28.852570+0000","last_clean":"2023-10-09T06:44:28.852570+0000","last_became_active":"2023-10-09T06:44:25.972571+0000","last_became_peered":"2023-10-09T06:44:25.972571+0000","last_unstale":"2023-10-09T06:44:28.852570+0000","last_undegraded":"2023-10-09T06:44:28.852570+0000","last_fullsized":"2023-10-09T06:44:28.852570+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:45.877535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000327022,"stat_sum":{"num_bytes":51,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":721,"num_read_kb":1616,"num_write":90,"num_write_kb":574,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":16,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"3.0","version":"0'0","reported_seq":569,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.741768+0000","last_change":"2023-10-09T06:44:25.970123+0000","last_active":"2023-10-09T06:44:29.741768+0000","last_peered":"2023-10-09T06:44:29.741768+0000","last_clean":"2023-10-09T06:44:29.741768+0000","last_became_active":"2023-10-09T06:44:25.969812+0000","last_became_peered":"2023-10-09T06:44:25.969812+0000","last_unstale":"2023-10-09T06:44:29.741768+0000","last_undegraded":"2023-10-09T06:44:29.741768+0000","last_fullsized":"2023-10-09T06:44:29.741768+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:28.643887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":582,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742447+0000","last_change":"2023-10-09T06:44:25.970135+0000","last_active":"2023-10-09T06:44:29.742447+0000","last_peered":"2023-10-09T06:44:29.742447+0000","last_clean":"2023-10-09T06:44:29.742447+0000","last_became_active":"2023-10-09T06:44:25.969783+0000","last_became_peered":"2023-10-09T06:44:25.969783+0000","last_unstale":"2023-10-09T06:44:29.742447+0000","last_undegraded":"2023-10-09T06:44:29.742447+0000","last_fullsized":"2023-10-09T06:44:29.742447+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:23.772755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"567'59","reported_seq":505,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.997274+0000","last_change":"2023-10-09T06:44:32.121524+0000","last_active":"2023-10-09T06:44:31.997274+0000","last_peered":"2023-10-09T06:44:31.997274+0000","last_clean":"2023-10-09T06:44:31.997274+0000","last_became_active":"2023-10-09T06:37:08.250992+0000","last_became_peered":"2023-10-09T06:37:08.250992+0000","last_unstale":"2023-10-09T06:44:31.997274+0000","last_undegraded":"2023-10-09T06:44:31.997274+0000","last_fullsized":"2023-10-09T06:44:31.997274+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":59,"log_dups_size":0,"ondisk_log_size":59,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:40.239525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.124371709,"stat_sum":{"num_bytes":225280,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":104,"num_read_kb":378,"num_write":61,"num_write_kb":198,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"1.0","version":"77'161","reported_seq":933,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738046+0000","last_change":"2023-10-09T06:44:28.743476+0000","last_active":"2023-10-09T06:44:30.738046+0000","last_peered":"2023-10-09T06:44:30.738046+0000","last_clean":"2023-10-09T06:44:30.738046+0000","last_became_active":"2023-10-09T06:44:28.743113+0000","last_became_peered":"2023-10-09T06:44:28.743113+0000","last_unstale":"2023-10-09T06:44:30.738046+0000","last_undegraded":"2023-10-09T06:44:30.738046+0000","last_fullsized":"2023-10-09T06:44:30.738046+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:01:59.139882+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":290,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.733144+0000","last_change":"2023-10-09T06:43:56.837415+0000","last_active":"2023-10-09T06:44:27.733144+0000","last_peered":"2023-10-09T06:44:27.733144+0000","last_clean":"2023-10-09T06:44:27.733144+0000","last_became_active":"2023-10-09T06:43:56.837016+0000","last_became_peered":"2023-10-09T06:43:56.837016+0000","last_unstale":"2023-10-09T06:44:27.733144+0000","last_undegraded":"2023-10-09T06:44:27.733144+0000","last_fullsized":"2023-10-09T06:44:27.733144+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:53:29.177761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978892+0000","last_change":"2023-10-09T06:44:25.973584+0000","last_active":"2023-10-09T06:44:26.978892+0000","last_peered":"2023-10-09T06:44:26.978892+0000","last_clean":"2023-10-09T06:44:26.978892+0000","last_became_active":"2023-10-09T06:44:25.973262+0000","last_became_peered":"2023-10-09T06:44:25.973262+0000","last_unstale":"2023-10-09T06:44:26.978892+0000","last_undegraded":"2023-10-09T06:44:26.978892+0000","last_fullsized":"2023-10-09T06:44:26.978892+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:06.789277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":899,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742200+0000","last_change":"2023-10-09T06:43:59.838208+0000","last_active":"2023-10-09T06:44:29.742200+0000","last_peered":"2023-10-09T06:44:29.742200+0000","last_clean":"2023-10-09T06:44:29.742200+0000","last_became_active":"2023-10-09T06:43:59.837800+0000","last_became_peered":"2023-10-09T06:43:59.837800+0000","last_unstale":"2023-10-09T06:44:29.742200+0000","last_undegraded":"2023-10-09T06:44:29.742200+0000","last_fullsized":"2023-10-09T06:44:29.742200+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:33.324556+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"569'20","reported_seq":569,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.997138+0000","last_change":"2023-10-09T06:44:32.121431+0000","last_active":"2023-10-09T06:44:31.997138+0000","last_peered":"2023-10-09T06:44:31.997138+0000","last_clean":"2023-10-09T06:44:31.997138+0000","last_became_active":"2023-10-09T06:44:25.977581+0000","last_became_peered":"2023-10-09T06:44:25.977581+0000","last_unstale":"2023-10-09T06:44:31.997138+0000","last_undegraded":"2023-10-09T06:44:31.997138+0000","last_fullsized":"2023-10-09T06:44:31.997138+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":20,"log_dups_size":0,"ondisk_log_size":20,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:41.177456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.124406616,"stat_sum":{"num_bytes":70,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":274,"num_read_kb":244,"num_write":17,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":10240,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"3.1","version":"0'0","reported_seq":308,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964239+0000","last_change":"2023-10-09T06:44:25.973593+0000","last_active":"2023-10-09T06:44:26.964239+0000","last_peered":"2023-10-09T06:44:26.964239+0000","last_clean":"2023-10-09T06:44:26.964239+0000","last_became_active":"2023-10-09T06:44:25.973296+0000","last_became_peered":"2023-10-09T06:44:25.973296+0000","last_unstale":"2023-10-09T06:44:26.964239+0000","last_undegraded":"2023-10-09T06:44:26.964239+0000","last_fullsized":"2023-10-09T06:44:26.964239+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:11.877898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":329,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742829+0000","last_change":"2023-10-09T06:44:25.967295+0000","last_active":"2023-10-09T06:44:29.742829+0000","last_peered":"2023-10-09T06:44:29.742829+0000","last_clean":"2023-10-09T06:44:29.742829+0000","last_became_active":"2023-10-09T06:44:25.966933+0000","last_became_peered":"2023-10-09T06:44:25.966933+0000","last_unstale":"2023-10-09T06:44:29.742829+0000","last_undegraded":"2023-10-09T06:44:29.742829+0000","last_fullsized":"2023-10-09T06:44:29.742829+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:28.788819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":583,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.741944+0000","last_change":"2023-10-09T06:44:28.741587+0000","last_active":"2023-10-09T06:44:28.741944+0000","last_peered":"2023-10-09T06:44:28.741944+0000","last_clean":"2023-10-09T06:44:28.741944+0000","last_became_active":"2023-10-09T06:44:28.741199+0000","last_became_peered":"2023-10-09T06:44:28.741199+0000","last_unstale":"2023-10-09T06:44:28.741944+0000","last_undegraded":"2023-10-09T06:44:28.741944+0000","last_fullsized":"2023-10-09T06:44:28.741944+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:05.537703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:05.537703+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.013646335000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"570'115","reported_seq":817,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T06:44:32.122713+0000","last_change":"2023-10-09T06:44:32.122713+0000","last_active":"2023-10-09T06:44:32.122713+0000","last_peered":"2023-10-09T06:44:32.122713+0000","last_clean":"2023-10-09T06:44:32.122713+0000","last_became_active":"2023-10-09T06:44:25.971156+0000","last_became_peered":"2023-10-09T06:44:25.971156+0000","last_unstale":"2023-10-09T06:44:32.122713+0000","last_undegraded":"2023-10-09T06:44:32.122713+0000","last_fullsized":"2023-10-09T06:44:32.122713+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":115,"log_dups_size":0,"ondisk_log_size":115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:38:17.626761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011652699999999999,"stat_sum":{"num_bytes":70,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":643,"num_read_kb":556,"num_write":102,"num_write_kb":51,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":35,"num_keys_recovered":11,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":569,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742192+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:44:29.742192+0000","last_peered":"2023-10-09T06:44:29.742192+0000","last_clean":"2023-10-09T06:44:29.742192+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:44:29.742192+0000","last_undegraded":"2023-10-09T06:44:29.742192+0000","last_fullsized":"2023-10-09T06:44:29.742192+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":329,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:31.215942+0000","last_change":"2023-10-09T06:44:25.969904+0000","last_active":"2023-10-09T06:44:31.215942+0000","last_peered":"2023-10-09T06:44:31.215942+0000","last_clean":"2023-10-09T06:44:31.215942+0000","last_became_active":"2023-10-09T06:44:25.969566+0000","last_became_peered":"2023-10-09T06:44:25.969566+0000","last_unstale":"2023-10-09T06:44:31.215942+0000","last_undegraded":"2023-10-09T06:44:31.215942+0000","last_fullsized":"2023-10-09T06:44:31.215942+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:44:11.562430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":909,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978502+0000","last_change":"2023-10-09T06:43:59.840490+0000","last_active":"2023-10-09T06:44:26.978502+0000","last_peered":"2023-10-09T06:44:26.978502+0000","last_clean":"2023-10-09T06:44:26.978502+0000","last_became_active":"2023-10-09T06:43:59.840116+0000","last_became_peered":"2023-10-09T06:43:59.840116+0000","last_unstale":"2023-10-09T06:44:26.978502+0000","last_undegraded":"2023-10-09T06:44:26.978502+0000","last_fullsized":"2023-10-09T06:44:26.978502+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:46.397166+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":314,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.730775+0000","last_change":"2023-10-09T06:44:25.994546+0000","last_active":"2023-10-09T06:44:27.730775+0000","last_peered":"2023-10-09T06:44:27.730775+0000","last_clean":"2023-10-09T06:44:27.730775+0000","last_became_active":"2023-10-09T06:44:25.976204+0000","last_became_peered":"2023-10-09T06:44:25.976204+0000","last_unstale":"2023-10-09T06:44:27.730775+0000","last_undegraded":"2023-10-09T06:44:27.730775+0000","last_fullsized":"2023-10-09T06:44:27.730775+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:24:37.270476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"567'70","reported_seq":857,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.830532+0000","last_change":"2023-10-09T06:44:24.963876+0000","last_active":"2023-10-09T06:44:30.830532+0000","last_peered":"2023-10-09T06:44:30.830532+0000","last_clean":"2023-10-09T06:44:30.830532+0000","last_became_active":"2023-10-09T06:43:56.839639+0000","last_became_peered":"2023-10-09T06:43:56.839639+0000","last_unstale":"2023-10-09T06:44:30.830532+0000","last_undegraded":"2023-10-09T06:44:30.830532+0000","last_fullsized":"2023-10-09T06:44:30.830532+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":70,"log_dups_size":0,"ondisk_log_size":70,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:20.238060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010324080000000001,"stat_sum":{"num_bytes":16,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":541,"num_read_kb":440,"num_write":42,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"4","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":299,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.726552+0000","last_change":"2023-10-09T06:44:25.976163+0000","last_active":"2023-10-09T06:44:27.726552+0000","last_peered":"2023-10-09T06:44:27.726552+0000","last_clean":"2023-10-09T06:44:27.726552+0000","last_became_active":"2023-10-09T06:44:25.972968+0000","last_became_peered":"2023-10-09T06:44:25.972968+0000","last_unstale":"2023-10-09T06:44:27.726552+0000","last_undegraded":"2023-10-09T06:44:27.726552+0000","last_fullsized":"2023-10-09T06:44:27.726552+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:14.667615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":924,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.732170+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:44:27.732170+0000","last_peered":"2023-10-09T06:44:27.732170+0000","last_clean":"2023-10-09T06:44:27.732170+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:44:27.732170+0000","last_undegraded":"2023-10-09T06:44:27.732170+0000","last_fullsized":"2023-10-09T06:44:27.732170+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"565'54","reported_seq":438,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T06:44:32.122554+0000","last_change":"2023-10-09T06:44:32.122726+0000","last_active":"2023-10-09T06:44:32.122554+0000","last_peered":"2023-10-09T06:44:32.122554+0000","last_clean":"2023-10-09T06:44:32.122554+0000","last_became_active":"2023-10-09T06:43:56.843089+0000","last_became_peered":"2023-10-09T06:43:56.843089+0000","last_unstale":"2023-10-09T06:44:32.122554+0000","last_undegraded":"2023-10-09T06:44:32.122554+0000","last_fullsized":"2023-10-09T06:44:32.122554+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:07.216017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063770600000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":440,"num_read_kb":388,"num_write":26,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"3.6","version":"68'1","reported_seq":61,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.743019+0000","last_change":"2023-10-09T06:44:25.972701+0000","last_active":"2023-10-09T06:44:29.743019+0000","last_peered":"2023-10-09T06:44:29.743019+0000","last_clean":"2023-10-09T06:44:29.743019+0000","last_became_active":"2023-10-09T06:44:25.967963+0000","last_became_peered":"2023-10-09T06:44:25.967963+0000","last_unstale":"2023-10-09T06:44:29.743019+0000","last_undegraded":"2023-10-09T06:44:29.743019+0000","last_fullsized":"2023-10-09T06:44:29.743019+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:13.336436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":549,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964309+0000","last_change":"2023-10-09T06:44:25.970224+0000","last_active":"2023-10-09T06:44:26.964309+0000","last_peered":"2023-10-09T06:44:26.964309+0000","last_clean":"2023-10-09T06:44:26.964309+0000","last_became_active":"2023-10-09T06:44:25.969961+0000","last_became_peered":"2023-10-09T06:44:25.969961+0000","last_unstale":"2023-10-09T06:44:26.964309+0000","last_undegraded":"2023-10-09T06:44:26.964309+0000","last_fullsized":"2023-10-09T06:44:26.964309+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:57:50.091498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":901,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T06:44:28.739663+0000","last_change":"2023-10-09T06:44:28.739663+0000","last_active":"2023-10-09T06:44:28.739663+0000","last_peered":"2023-10-09T06:44:28.739663+0000","last_clean":"2023-10-09T06:44:28.739663+0000","last_became_active":"2023-10-09T06:44:28.739332+0000","last_became_peered":"2023-10-09T06:44:28.739332+0000","last_unstale":"2023-10-09T06:44:28.739663+0000","last_undegraded":"2023-10-09T06:44:28.739663+0000","last_fullsized":"2023-10-09T06:44:28.739663+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:34.586120+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.977991+0000","last_change":"2023-10-09T06:44:25.968069+0000","last_active":"2023-10-09T06:44:26.977991+0000","last_peered":"2023-10-09T06:44:26.977991+0000","last_clean":"2023-10-09T06:44:26.977991+0000","last_became_active":"2023-10-09T06:44:25.967636+0000","last_became_peered":"2023-10-09T06:44:25.967636+0000","last_unstale":"2023-10-09T06:44:26.977991+0000","last_undegraded":"2023-10-09T06:44:26.977991+0000","last_fullsized":"2023-10-09T06:44:26.977991+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:57:36.741799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":561,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742121+0000","last_change":"2023-10-09T06:28:49.644543+0000","last_active":"2023-10-09T06:44:29.742121+0000","last_peered":"2023-10-09T06:44:29.742121+0000","last_clean":"2023-10-09T06:44:29.742121+0000","last_became_active":"2023-10-09T06:28:49.644053+0000","last_became_peered":"2023-10-09T06:28:49.644053+0000","last_unstale":"2023-10-09T06:44:29.742121+0000","last_undegraded":"2023-10-09T06:44:29.742121+0000","last_fullsized":"2023-10-09T06:44:29.742121+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:15.961482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":798,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.733377+0000","last_change":"2023-10-09T06:43:59.841408+0000","last_active":"2023-10-09T06:44:27.733377+0000","last_peered":"2023-10-09T06:44:27.733377+0000","last_clean":"2023-10-09T06:44:27.733377+0000","last_became_active":"2023-10-09T06:43:59.841036+0000","last_became_peered":"2023-10-09T06:43:59.841036+0000","last_unstale":"2023-10-09T06:44:27.733377+0000","last_undegraded":"2023-10-09T06:44:27.733377+0000","last_fullsized":"2023-10-09T06:44:27.733377+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:27.958153+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":244,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978322+0000","last_change":"2023-10-09T06:37:08.251856+0000","last_active":"2023-10-09T06:44:26.978322+0000","last_peered":"2023-10-09T06:44:26.978322+0000","last_clean":"2023-10-09T06:44:26.978322+0000","last_became_active":"2023-10-09T06:37:08.251490+0000","last_became_peered":"2023-10-09T06:37:08.251490+0000","last_unstale":"2023-10-09T06:44:26.978322+0000","last_undegraded":"2023-10-09T06:44:26.978322+0000","last_fullsized":"2023-10-09T06:44:26.978322+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:48.779792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":573,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978033+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:44:26.978033+0000","last_peered":"2023-10-09T06:44:26.978033+0000","last_clean":"2023-10-09T06:44:26.978033+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:44:26.978033+0000","last_undegraded":"2023-10-09T06:44:26.978033+0000","last_fullsized":"2023-10-09T06:44:26.978033+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":258,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978734+0000","last_change":"2023-10-09T06:43:56.894480+0000","last_active":"2023-10-09T06:44:26.978734+0000","last_peered":"2023-10-09T06:44:26.978734+0000","last_clean":"2023-10-09T06:44:26.978734+0000","last_became_active":"2023-10-09T06:43:56.836013+0000","last_became_peered":"2023-10-09T06:43:56.836013+0000","last_unstale":"2023-10-09T06:44:26.978734+0000","last_undegraded":"2023-10-09T06:44:26.978734+0000","last_fullsized":"2023-10-09T06:44:26.978734+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:27.402384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":593,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964386+0000","last_change":"2023-10-09T06:44:25.973648+0000","last_active":"2023-10-09T06:44:26.964386+0000","last_peered":"2023-10-09T06:44:26.964386+0000","last_clean":"2023-10-09T06:44:26.964386+0000","last_became_active":"2023-10-09T06:44:25.973315+0000","last_became_peered":"2023-10-09T06:44:25.973315+0000","last_unstale":"2023-10-09T06:44:26.964386+0000","last_undegraded":"2023-10-09T06:44:26.964386+0000","last_fullsized":"2023-10-09T06:44:26.964386+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:24.983267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":946,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T06:44:30.738259+0000","last_change":"2023-10-09T06:43:59.843534+0000","last_active":"2023-10-09T06:44:30.738259+0000","last_peered":"2023-10-09T06:44:30.738259+0000","last_clean":"2023-10-09T06:44:30.738259+0000","last_became_active":"2023-10-09T06:43:59.843152+0000","last_became_peered":"2023-10-09T06:43:59.843152+0000","last_unstale":"2023-10-09T06:44:30.738259+0000","last_undegraded":"2023-10-09T06:44:30.738259+0000","last_fullsized":"2023-10-09T06:44:30.738259+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:52:21.087078+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":281,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.733580+0000","last_change":"2023-10-09T06:43:56.838100+0000","last_active":"2023-10-09T06:44:27.733580+0000","last_peered":"2023-10-09T06:44:27.733580+0000","last_clean":"2023-10-09T06:44:27.733580+0000","last_became_active":"2023-10-09T06:43:56.837814+0000","last_became_peered":"2023-10-09T06:43:56.837814+0000","last_unstale":"2023-10-09T06:44:27.733580+0000","last_undegraded":"2023-10-09T06:44:27.733580+0000","last_fullsized":"2023-10-09T06:44:27.733580+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:44:21.445491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":297,"reported_epoch":566,"state":"active+clean","last_fresh":"2023-10-09T06:44:27.733217+0000","last_change":"2023-10-09T06:44:25.976051+0000","last_active":"2023-10-09T06:44:27.733217+0000","last_peered":"2023-10-09T06:44:27.733217+0000","last_clean":"2023-10-09T06:44:27.733217+0000","last_became_active":"2023-10-09T06:44:25.972168+0000","last_became_peered":"2023-10-09T06:44:25.972168+0000","last_unstale":"2023-10-09T06:44:27.733217+0000","last_undegraded":"2023-10-09T06:44:27.733217+0000","last_fullsized":"2023-10-09T06:44:27.733217+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:56.953311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":693,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T06:44:29.742170+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:44:29.742170+0000","last_peered":"2023-10-09T06:44:29.742170+0000","last_clean":"2023-10-09T06:44:29.742170+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:44:29.742170+0000","last_undegraded":"2023-10-09T06:44:29.742170+0000","last_fullsized":"2023-10-09T06:44:29.742170+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.978435+0000","last_change":"2023-10-09T06:44:25.976552+0000","last_active":"2023-10-09T06:44:26.978435+0000","last_peered":"2023-10-09T06:44:26.978435+0000","last_clean":"2023-10-09T06:44:26.978435+0000","last_became_active":"2023-10-09T06:44:25.970107+0000","last_became_peered":"2023-10-09T06:44:25.970107+0000","last_unstale":"2023-10-09T06:44:26.978435+0000","last_undegraded":"2023-10-09T06:44:26.978435+0000","last_fullsized":"2023-10-09T06:44:26.978435+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:17:45.134967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":536,"reported_epoch":565,"state":"active+clean","last_fresh":"2023-10-09T06:44:26.964192+0000","last_change":"2023-10-09T06:43:56.845479+0000","last_active":"2023-10-09T06:44:26.964192+0000","last_peered":"2023-10-09T06:44:26.964192+0000","last_clean":"2023-10-09T06:44:26.964192+0000","last_became_active":"2023-10-09T06:43:56.845179+0000","last_became_peered":"2023-10-09T06:43:56.845179+0000","last_unstale":"2023-10-09T06:44:26.964192+0000","last_undegraded":"2023-10-09T06:44:26.964192+0000","last_fullsized":"2023-10-09T06:44:26.964192+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:22.370875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":15568,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":225541,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":3477,"num_read_kb":4927,"num_write":492,"num_write_kb":1300,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":10310,"num_keys_recovered":13,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":72342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":567,"ondisk_log_size":567,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234567,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1514880,"kb_used_data":5688,"kb_used_omap":0,"kb_used_meta":1509184,"kb_avail":92230272,"statfs":{"total":95995035648,"available":94443798528,"internally_reserved":0,"allocated":5824512,"data_stored":4680045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545404416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:44:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":3.0270000000000001,"15min":3.4169999999999998},"min":{"1min":0.23999999999999999,"5min":0.216,"15min":0.216},"max":{"1min":2.2210000000000001,"5min":76.727999999999994,"15min":196.28899999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":3.0870000000000002,"15min":3.496},"min":{"1min":0.30099999999999999,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":1.623,"5min":76.712999999999994,"15min":196.333},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 06:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":3.3799999999999999,"15min":4.9509999999999996},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.1219999999999999,"5min":169.51900000000001,"15min":208.626},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.0649999999999999,"5min":3.323,"15min":4.117},"min":{"1min":0.313,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":10.191000000000001,"5min":169.101,"15min":181.24199999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.6120000000000001,"15min":4.8689999999999998},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4900000000000002,"5min":30.117000000000001,"15min":228.24000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.929,"15min":5.0209999999999999},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.29999999999999999},"max":{"1min":5.8109999999999999,"5min":30.245999999999999,"15min":228.291},"last":0.56999999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:43:45 2023","interfaces":[{"interface":"back","average":{"1min":4.7930000000000001,"5min":3.2909999999999999,"15min":5.0609999999999999},"min":{"1min":0.34000000000000002,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":59.847000000000001,"5min":59.847000000000001,"15min":187.34200000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":5.0869999999999997,"5min":3.2200000000000002,"15min":5.0709999999999997},"min":{"1min":0.432,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":59.698999999999998,"5min":59.698999999999998,"15min":187.08199999999999},"last":0.58199999999999996}]},{"osd":4,"last update":"Mon Oct 9 06:43:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":5.0999999999999996,"15min":2.927},"min":{"1min":0.28399999999999997,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":2.7650000000000001,"5min":225.041,"15min":225.041},"last":0.442},{"interface":"front","average":{"1min":0.627,"5min":4.9379999999999997,"15min":2.738},"min":{"1min":0.32400000000000001,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":2.6440000000000001,"5min":225.244,"15min":225.244},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Oct 9 06:43:45 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":2.7370000000000001,"15min":2.395},"min":{"1min":0.32200000000000001,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":4.8520000000000003,"5min":219.923,"15min":219.923},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.1359999999999999,"5min":2.8300000000000001,"15min":2.395},"min":{"1min":0.36099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.8050000000000002,"5min":219.86000000000001,"15min":219.86000000000001},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:45 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":4.9640000000000004,"15min":3.2320000000000002},"min":{"1min":0.34599999999999997,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":2.9279999999999999,"5min":222.179,"15min":222.179},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.022,"5min":5.0099999999999998,"15min":3.0680000000000001},"min":{"1min":0.38100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.722,"5min":223.261,"15min":223.261},"last":0.66100000000000003}]}]},{"osd":6,"up_from":0,"seq":197568496202,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":110,"seq":472446402940,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155640,"kb_used_data":6820,"kb_used_omap":4,"kb_used_meta":1148795,"kb_avail":92589512,"statfs":{"total":95995035648,"available":94811660288,"internally_reserved":0,"allocated":6983680,"data_stored":4545888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1176366990},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":4.5999999999999996,"15min":4.4450000000000003},"min":{"1min":0.215,"5min":0.20100000000000001,"15min":0.182},"max":{"1min":10.369,"5min":114.97799999999999,"15min":143.465},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.633,"5min":5.2320000000000002,"15min":4.7960000000000003},"min":{"1min":0.23300000000000001,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":8.8689999999999998,"5min":114.934,"15min":143.66399999999999},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.2370000000000001,"15min":2.1920000000000002},"min":{"1min":0.217,"5min":0.193,"15min":0.184},"max":{"1min":4.149,"5min":8.0129999999999999,"15min":138.452},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.7030000000000001,"5min":2.5859999999999999,"15min":2.9580000000000002},"min":{"1min":0.22900000000000001,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":8.8940000000000001,"5min":20.085000000000001,"15min":138.47499999999999},"last":0.307}]},{"osd":3,"last update":"Mon Oct 9 06:43:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.7610000000000001,"15min":2.7429999999999999},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":9.8580000000000005,"5min":27.655999999999999,"15min":27.655999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.383,"5min":2.0880000000000001,"15min":1.9770000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":10.332000000000001,"5min":27.745999999999999,"15min":27.745999999999999},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.5070000000000001,"15min":3.1760000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":6.4260000000000002,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.458,"5min":3.3069999999999999,"15min":2.948},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":8.9239999999999995,"5min":180.70599999999999,"15min":180.70599999999999},"last":0.48699999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.2070000000000001,"15min":2.5600000000000001},"min":{"1min":0.35699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":2.9279999999999999,"5min":7.9420000000000002,"15min":119.408},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.478,"15min":2.8069999999999999},"min":{"1min":0.32200000000000001,"5min":0.312,"15min":0.312},"max":{"1min":9.6820000000000004,"5min":9.6820000000000004,"15min":119.483},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":4.8440000000000003,"15min":3.931},"min":{"1min":0.35599999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.4760000000000009,"5min":183.12100000000001,"15min":183.12100000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.012,"5min":5.1399999999999997,"15min":4.226},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.316},"max":{"1min":6.4649999999999999,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":3.2429999999999999,"15min":3.9140000000000001},"min":{"1min":0.41399999999999998,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":6.4950000000000001,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.628},{"interface":"front","average":{"1min":0.80000000000000004,"5min":2.6739999999999999,"15min":3.339},"min":{"1min":0.44400000000000001,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":2.9750000000000001,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.61599999999999999}]}]},{"osd":0,"up_from":0,"seq":390842024373,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":128,"seq":549755814242,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1074360,"kb_used_data":7012,"kb_used_omap":7,"kb_used_meta":1067320,"kb_avail":92670792,"statfs":{"total":95995035648,"available":94894891008,"internally_reserved":0,"allocated":7180288,"data_stored":4830921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1092935761},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":7.194,"15min":6.6079999999999997},"min":{"1min":0.21199999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":19.562000000000001,"5min":214.595,"15min":214.595},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.0489999999999999,"5min":6.8499999999999996,"15min":6.25},"min":{"1min":0.27600000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":22.814,"5min":214.94300000000001,"15min":214.94300000000001},"last":0.621}]},{"osd":1,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":6.8559999999999999,"15min":5.6189999999999998},"min":{"1min":0.25900000000000001,"5min":0.20499999999999999,"15min":0.20200000000000001},"max":{"1min":5.7370000000000001,"5min":299.988,"15min":299.988},"last":0.503},{"interface":"front","average":{"1min":0.76500000000000001,"5min":6.9699999999999998,"15min":5.8739999999999997},"min":{"1min":0.251,"5min":0.21299999999999999,"15min":0.183},"max":{"1min":5.6959999999999997,"5min":300.30500000000001,"15min":300.30500000000001},"last":0.57999999999999996}]},{"osd":3,"last update":"Mon Oct 9 06:43:40 2023","interfaces":[{"interface":"back","average":{"1min":7.4189999999999996,"5min":3.923,"15min":4.2000000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":74.317999999999998,"5min":74.317999999999998,"15min":91.644000000000005},"last":1.994},{"interface":"front","average":{"1min":6.9880000000000004,"5min":3.7240000000000002,"15min":4.3120000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":74.355999999999995,"5min":74.355999999999995,"15min":91.685000000000002},"last":0.75600000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.877,"15min":2.1760000000000002},"min":{"1min":0.313,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.4740000000000002,"5min":9.5679999999999996,"15min":79.643000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.82799999999999996,"5min":1.1140000000000001,"15min":2.4430000000000001},"min":{"1min":0.39900000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.609,"5min":7.0910000000000002,"15min":82.872},"last":0.56200000000000006}]},{"osd":5,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.90700000000000003,"15min":2.23},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.5169999999999999,"5min":9.7040000000000006,"15min":82.834999999999994},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.159,"15min":2.427},"min":{"1min":0.40699999999999997,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":3.6099999999999999,"5min":7.1369999999999996,"15min":82.921999999999997},"last":0.60099999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.2250000000000001,"15min":3.0950000000000002},"min":{"1min":0.378,"5min":0.33800000000000002,"15min":0.313},"max":{"1min":2.915,"5min":7.1669999999999998,"15min":123.47799999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.2230000000000001,"15min":3.0819999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.313},"max":{"1min":2.839,"5min":9.3740000000000006,"15min":122.517},"last":1.089}]},{"osd":7,"last update":"Mon Oct 9 06:44:17 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.6600000000000001,"15min":3.1850000000000001},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":2.8799999999999999,"5min":132.06399999999999,"15min":132.06399999999999},"last":1.1439999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":2.5329999999999999,"15min":2.8370000000000002},"min":{"1min":0.38800000000000001,"5min":0.35899999999999999,"15min":0.35499999999999998},"max":{"1min":3.9990000000000001,"5min":132.196,"15min":132.196},"last":0.94399999999999995}]}]},{"osd":3,"up_from":149,"seq":639950127439,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":947984,"kb_used_data":6884,"kb_used_omap":2,"kb_used_meta":941053,"kb_avail":92797168,"statfs":{"total":95995035648,"available":95024300032,"internally_reserved":0,"allocated":7049216,"data_stored":4833546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":963638325},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":4.0570000000000004,"15min":4.5380000000000003},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19},"max":{"1min":28.236999999999998,"5min":95.281999999999996,"15min":161.821},"last":0.46899999999999997},{"interface":"front","average":{"1min":4.532,"5min":4.6449999999999996,"15min":4.6239999999999997},"min":{"1min":0.24099999999999999,"5min":0.192,"15min":0.17799999999999999},"max":{"1min":28.954000000000001,"5min":98.572000000000003,"15min":163.52000000000001},"last":0.41799999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":3.5019999999999998,"5min":4.008,"15min":4.5839999999999996},"min":{"1min":0.223,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":15.773999999999999,"5min":102.468,"15min":184.012},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.637,"5min":4.0860000000000003,"15min":5.1660000000000004},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.7030000000000003,"5min":102.58,"15min":184.703},"last":0.54600000000000004}]},{"osd":2,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":3.387,"5min":3.1699999999999999,"15min":4.1180000000000003},"min":{"1min":0.28100000000000003,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":15.807,"5min":118.93899999999999,"15min":182.44900000000001},"last":0.499},{"interface":"front","average":{"1min":1.8400000000000001,"5min":2.8969999999999998,"15min":4.3150000000000004},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":8.7899999999999991,"5min":118.97199999999999,"15min":182.56399999999999},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":2.7589999999999999,"15min":3.3450000000000002},"min":{"1min":0.28299999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.8490000000000002,"5min":133.80099999999999,"15min":178.22200000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.407,"5min":2.9900000000000002,"15min":3.4129999999999998},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.5659999999999998,"5min":133.648,"15min":180.43600000000001},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":2.9649999999999999,"15min":3.6709999999999998},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.34000000000000002},"max":{"1min":5.6059999999999999,"5min":128.083,"15min":177.56299999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.407,"5min":3.1259999999999999,"15min":3.177},"min":{"1min":0.32000000000000001,"5min":0.312,"15min":0.312},"max":{"1min":15.743,"5min":128.11799999999999,"15min":177.59399999999999},"last":0.66100000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.7450000000000001,"15min":2.782},"min":{"1min":0.36399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":5.6360000000000001,"5min":8.5779999999999994,"15min":130.39599999999999},"last":0.629},{"interface":"front","average":{"1min":2.7370000000000001,"5min":1.768,"15min":2.5379999999999998},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":15.69,"5min":15.69,"15min":130.67699999999999},"last":0.61799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:43:46 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.2320000000000002,"15min":3.032},"min":{"1min":0.371,"5min":0.314,"15min":0.314},"max":{"1min":5.3949999999999996,"5min":99.248000000000005,"15min":121.613},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.9490000000000001,"5min":2.4319999999999999,"15min":3.4860000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":5.6639999999999997,"5min":99.457999999999998,"15min":121.559},"last":0.67400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019475,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485240,"kb_used_data":5664,"kb_used_omap":0,"kb_used_meta":1479552,"kb_avail":92259912,"statfs":{"total":95995035648,"available":94474149888,"internally_reserved":0,"allocated":5799936,"data_stored":4686875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1515061248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":7.3730000000000002,"5min":4.1280000000000001,"15min":6.9119999999999999},"min":{"1min":0.27000000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":68.421999999999997,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":7.258,"5min":4.3010000000000002,"15min":7.1159999999999997},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.23100000000000001},"max":{"1min":68.433000000000007,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.65400000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":3.9990000000000001,"15min":3.9929999999999999},"min":{"1min":0.25800000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.1589999999999998,"5min":163.392,"15min":163.392},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.304,"5min":4.1420000000000003,"15min":4.1159999999999997},"min":{"1min":0.34599999999999997,"5min":0.251,"15min":0.251},"max":{"1min":11.569000000000001,"5min":163.42400000000001,"15min":163.42400000000001},"last":0.61799999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.1150000000000002,"15min":4.1929999999999996},"min":{"1min":0.375,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":3.6850000000000001,"5min":175.97900000000001,"15min":175.97900000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.4259999999999999,"5min":3.2959999999999998,"15min":4.2119999999999997},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.9119999999999999,"5min":175.875,"15min":175.875},"last":0.54100000000000004}]},{"osd":3,"last update":"Mon Oct 9 06:43:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":3.2170000000000001,"15min":6.0979999999999999},"min":{"1min":0.34799999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8239999999999998,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.627},{"interface":"front","average":{"1min":1.984,"5min":3.0659999999999998,"15min":5.8730000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":7.3630000000000004,"5min":112.21899999999999,"15min":161.934},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.61299999999999999,"15min":0.95999999999999996},"min":{"1min":0.307,"5min":0.253,"15min":0.252},"max":{"1min":1.881,"5min":6.7869999999999999,"15min":9.3979999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.74299999999999999,"15min":1.127},"min":{"1min":0.27900000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.4700000000000002,"5min":5.9820000000000002,"15min":9.5009999999999994},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:43:27 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.651,"15min":2.2959999999999998},"min":{"1min":0.34599999999999997,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":5.3819999999999997,"5min":80.549000000000007,"15min":163.36699999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.087,"5min":1.708,"15min":2.4969999999999999},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":6.734,"5min":83.218000000000004,"15min":163.381},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.5900000000000001,"15min":1.9419999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.7619999999999996,"5min":76.081000000000003,"15min":106.801},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.552,"15min":1.9950000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.3529999999999998,"5min":76.283000000000001,"15min":106.827},"last":0.51900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757841,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485468,"kb_used_data":5972,"kb_used_omap":0,"kb_used_meta":1479488,"kb_avail":92259684,"statfs":{"total":95995035648,"available":94473916416,"internally_reserved":0,"allocated":6115328,"data_stored":5002955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514995712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":3.2829999999999999,"15min":5.444},"min":{"1min":0.26000000000000001,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":4.9800000000000004,"5min":66.799000000000007,"15min":294.89699999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":3.5030000000000001,"15min":5.79},"min":{"1min":0.316,"5min":0.25900000000000001,"15min":0.215},"max":{"1min":4.8209999999999997,"5min":66.614000000000004,"15min":294.30399999999997},"last":0.39000000000000001}]},{"osd":1,"last update":"Mon Oct 9 06:44:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.7610000000000001,"15min":6.8559999999999999},"min":{"1min":0.315,"5min":0.253,"15min":0.22},"max":{"1min":4.5759999999999996,"5min":69.287999999999997,"15min":188.34100000000001},"last":0.379},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.9279999999999999,"15min":6.8860000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":5.0170000000000003,"5min":67.245999999999995,"15min":188.839},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:44:08 2023","interfaces":[{"interface":"back","average":{"1min":4.9459999999999997,"5min":4.4409999999999998,"15min":4.9199999999999999},"min":{"1min":0.34100000000000003,"5min":0.29299999999999998,"15min":0.27000000000000002},"max":{"1min":74.049999999999997,"5min":145.20400000000001,"15min":145.20400000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":5.1929999999999996,"5min":4.5119999999999996,"15min":4.7930000000000001},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":74.263000000000005,"5min":145.71899999999999,"15min":145.71899999999999},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Oct 9 06:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":2.9119999999999999,"15min":5.532},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.28299999999999997},"max":{"1min":4.6539999999999999,"5min":63.578000000000003,"15min":220.41800000000001},"last":0.44},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.8460000000000001,"15min":5.2409999999999997},"min":{"1min":0.42099999999999999,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.0039999999999996,"5min":62.113999999999997,"15min":220.45500000000001},"last":0.45000000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:43:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.667,"15min":1.7330000000000001},"min":{"1min":0.27300000000000002,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":5.5270000000000001,"5min":74.725999999999999,"15min":79.043000000000006},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.728,"15min":1.734},"min":{"1min":0.32600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":3.923,"5min":74.777000000000001,"15min":79.165999999999997},"last":0.25700000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.8560000000000001,"15min":1.8100000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.251},"max":{"1min":2.2759999999999998,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":1.778,"15min":1.8999999999999999},"min":{"1min":0.36499999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.753,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.36799999999999999}]},{"osd":7,"last update":"Mon Oct 9 06:43:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.85899999999999999,"15min":1.6020000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":5.4729999999999999,"5min":5.4729999999999999,"15min":78.890000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.872,"15min":1.415},"min":{"1min":0.33700000000000002,"5min":0.29699999999999999,"15min":0.28599999999999998},"max":{"1min":3.0550000000000002,"5min":3.944,"15min":79.138999999999996},"last":0.317}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":10310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":20531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":86,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":35,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":20499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":20620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:44:34.508 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T06:44:34.508 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-10-09T06:44:34.601 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:34.631 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:34.674 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:34.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:34.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:34.778 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:34.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:44:34.797 INFO:tasks.workunit.client.0.smithi107.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T06:44:34.804 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg 2023-10-09T06:44:34.808 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:34.882 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:34.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:34.898 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:34.910 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:34.913 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd remove testimg_import 2023-10-09T06:44:34.922 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:34.946 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T06:44:34.948 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:34.978 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:34.985 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:34.995 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:35.008 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:35.013 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:35.027 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:35.031 INFO:tasks.workunit.client.0.smithi107.stderr:+ tiered=0 2023-10-09T06:44:35.031 INFO:tasks.workunit.client.0.smithi107.stderr:+ ceph osd dump 2023-10-09T06:44:35.032 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep ''\''rbd'\''' 2023-10-09T06:44:35.032 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep tier 2023-10-09T06:44:35.033 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep '^pool' 2023-10-09T06:44:35.038 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:35.056 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:35.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:35.072 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:35.092 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:35.093 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.093 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.093 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.093 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.094 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: from='client.37764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: pgmap v1735: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 238 KiB/s rd, 27 KiB/s wr, 317 op/s; 4 B/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T06:44:35.126 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/142816428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: from='client.37764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: pgmap v1735: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 238 KiB/s rd, 27 KiB/s wr, 317 op/s; 4 B/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T06:44:35.127 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/142816428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: from='client.37764 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:44:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: pgmap v1735: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 238 KiB/s rd, 27 KiB/s wr, 317 op/s; 4 B/s, 0 keys/s, 2 objects/s recovering 2023-10-09T06:44:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: failed to encode map e572 with expected crc 2023-10-09T06:44:35.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: osdmap e573: 8 total, 8 up, 6 in 2023-10-09T06:44:35.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/142816428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:35.164 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:35.190 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:35.231 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:35.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:35.338 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:35.365 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:35.407 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:35.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:35.515 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T06:44:35.542 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:35.586 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T06:44:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:35.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T06:44:35.668 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_644583/sparse1 2023-10-09T06:44:35.674 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records in 2023-10-09T06:44:35.674 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records out 2023-10-09T06:44:35.674 INFO:tasks.workunit.client.0.smithi107.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00498425 s, 210 MB/s 2023-10-09T06:44:35.674 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_644583/sparse2 2023-10-09T06:44:35.680 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records in 2023-10-09T06:44:35.680 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records out 2023-10-09T06:44:35.680 INFO:tasks.workunit.client.0.smithi107.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00492292 s, 213 MB/s 2023-10-09T06:44:35.680 INFO:tasks.workunit.client.0.smithi107.stderr:+ truncate /tmp/rbd_import_export_644583/sparse2 -s 2M 2023-10-09T06:44:35.691 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:35.717 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:35.796 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse1 2023-10-09T06:44:35.831 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 0% complete...failed. 2023-10-09T06:44:35.831 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:44:35.834 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:35.834 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_644583/sparse1 2023-10-09T06:44:35.892 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:44:35.892 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:35.892 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:35.895 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd ls -l 2023-10-09T06:44:35.895 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep sparse1 2023-10-09T06:44:35.895 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:44:35.934 INFO:tasks.workunit.client.0.smithi107.stdout:sparse1 2 MiB 2 2023-10-09T06:44:35.935 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse1 2023-10-09T06:44:35.935 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse1 2023-10-09T06:44:35.935 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse1 2023-10-09T06:44:35.935 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:35.935 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:35.972 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bab3664d8597 2023-10-09T06:44:35.973 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bab3664d8597 2023-10-09T06:44:35.973 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:44:35.973 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse1 2023-10-09T06:44:35.973 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bab3664d8597\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:35.974 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:35.974 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse1 2023-10-09T06:44:35.974 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:35.974 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:36.010 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:36.010 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:36.010 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:36.010 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:36.010 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:36.089 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects=1 2023-10-09T06:44:36.089 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 1 2023-10-09T06:44:36.090 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:44:36.090 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:36.126 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:36.130 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:36.130 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:36.132 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse1 --format %b 2023-10-09T06:44:36.133 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=2048 2023-10-09T06:44:36.133 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse1.out --format %b 2023-10-09T06:44:36.134 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=2048 2023-10-09T06:44:36.134 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:36.134 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:36.135 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:44:36.135 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:36.135 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse1 2023-10-09T06:44:36.217 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:44:36.220 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse2 2023-10-09T06:44:36.255 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 0% complete...failed. 2023-10-09T06:44:36.256 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T06:44:36.258 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:36.259 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_644583/sparse2 2023-10-09T06:44:36.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[212750]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.324 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[220921]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[220921]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.325 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:35 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:35 smithi150 ceph-mon[194178]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:35 smithi150 ceph-mon[194178]: failed to encode map e573 with expected crc 2023-10-09T06:44:36.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:35 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/549428042' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T06:44:36.418 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:36.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:36.453 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T06:44:36.454 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:36.454 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:36.456 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd ls -l 2023-10-09T06:44:36.456 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep sparse2 2023-10-09T06:44:36.456 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T06:44:36.491 INFO:tasks.workunit.client.0.smithi107.stdout:sparse2 2 MiB 2 2023-10-09T06:44:36.491 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse2 2023-10-09T06:44:36.491 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse2 2023-10-09T06:44:36.495 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse2 2023-10-09T06:44:36.495 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:36.495 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:36.496 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:36.521 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T06:44:36.540 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bae6139a4019 2023-10-09T06:44:36.541 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bae6139a4019 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bae6139a4019\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse2 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse2 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:36.542 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:36.549 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:36.576 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:36.577 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:36.577 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:36.577 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:36.577 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:36.590 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T06:44:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:36.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T06:44:36.607 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects=0 2023-10-09T06:44:36.607 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 0 2023-10-09T06:44:36.608 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:44:36.608 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:36.644 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:36.648 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:36.648 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:36.650 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse2 --format %b 2023-10-09T06:44:36.651 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=2048 2023-10-09T06:44:36.651 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse2.out --format %b 2023-10-09T06:44:36.652 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=2048 2023-10-09T06:44:36.652 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:36.652 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:36.652 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:44:36.652 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:36.653 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse2 2023-10-09T06:44:36.695 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:36.729 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:36.754 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T06:44:36.757 INFO:tasks.workunit.client.0.smithi107.stderr:+ truncate /tmp/rbd_import_export_644583/sparse1 -s 10M 2023-10-09T06:44:36.758 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T06:44:36.769 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:36.828 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:36.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:36.862 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:36.862 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:36.862 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:36.866 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd ls -l 2023-10-09T06:44:36.866 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep sparse1 2023-10-09T06:44:36.867 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T06:44:36.904 INFO:tasks.workunit.client.0.smithi107.stdout:sparse1 10 MiB 2 2023-10-09T06:44:36.905 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse1 2023-10-09T06:44:36.905 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse1 2023-10-09T06:44:36.905 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse1 2023-10-09T06:44:36.906 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:36.906 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:36.934 DEBUG:teuthology.orchestra.run.smithi150:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T06:44:36.943 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bb13d7871765 2023-10-09T06:44:36.944 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bb13d7871765 2023-10-09T06:44:36.944 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse1 2023-10-09T06:44:36.944 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bb13d7871765\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:36.945 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse1 2023-10-09T06:44:36.945 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:36.945 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse1 2023-10-09T06:44:36.945 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:36.946 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:36.960 INFO:teuthology.orchestra.run.smithi150.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:36.981 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:36.982 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:36.982 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:36.982 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:36.982 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:36.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:36 smithi107 ceph-mon[220921]: pgmap v1738: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 67 KiB/s rd, 7.2 KiB/s wr, 88 op/s 2023-10-09T06:44:37.002 INFO:teuthology.orchestra.run.smithi150.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T06:44:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:37.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi150 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T06:44:37.013 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects=1 2023-10-09T06:44:37.014 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 1 2023-10-09T06:44:37.014 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:44:37.014 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:37.050 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:37.053 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:37.053 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/sparse1 /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:37.061 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse1 --format %b 2023-10-09T06:44:37.062 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=2048 2023-10-09T06:44:37.062 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse1.out --format %b 2023-10-09T06:44:37.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=2048 2023-10-09T06:44:37.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:37.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:37.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:44:37.063 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse1.out 2023-10-09T06:44:37.064 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse1 2023-10-09T06:44:37.109 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:37.141 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:37.156 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:44:37.162 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_644583/sparse2 oflag=append conv=notrunc 2023-10-09T06:44:37.173 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records in 2023-10-09T06:44:37.173 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records out 2023-10-09T06:44:37.173 INFO:tasks.workunit.client.0.smithi107.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0115297 s, 182 MB/s 2023-10-09T06:44:37.173 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T06:44:37.244 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:36 smithi107 ceph-mon[212750]: pgmap v1738: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 67 KiB/s rd, 7.2 KiB/s wr, 88 op/s 2023-10-09T06:44:37.256 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:37.256 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:37.256 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:37.259 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd ls -l 2023-10-09T06:44:37.259 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep sparse2 2023-10-09T06:44:37.259 INFO:tasks.workunit.client.0.smithi107.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T06:44:37.294 INFO:tasks.workunit.client.0.smithi107.stdout:sparse2 4 MiB 2 2023-10-09T06:44:37.295 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse2 2023-10-09T06:44:37.295 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse2 2023-10-09T06:44:37.295 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse2 2023-10-09T06:44:37.295 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:37.295 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:37.330 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bb40654b2ed2 2023-10-09T06:44:37.331 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bb40654b2ed2 2023-10-09T06:44:37.331 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse2 2023-10-09T06:44:37.331 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse2 2023-10-09T06:44:37.332 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bb40654b2ed2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:37.332 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:37.332 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse2 2023-10-09T06:44:37.332 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:37.332 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:37.368 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:37.369 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:37.369 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:37.369 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:37.369 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:37.398 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects='0 2023-10-09T06:44:37.398 INFO:tasks.workunit.client.0.smithi107.stderr:2 2023-10-09T06:44:37.398 INFO:tasks.workunit.client.0.smithi107.stderr:3' 2023-10-09T06:44:37.399 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 0 2 3 2023-10-09T06:44:37.399 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T06:44:37.399 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:36 smithi150 ceph-mon[194178]: pgmap v1738: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 67 KiB/s rd, 7.2 KiB/s wr, 88 op/s 2023-10-09T06:44:37.496 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T06:44:37.499 INFO:tasks.workunit.client.0.smithi107.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_644583/sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:37.500 INFO:tasks.workunit.client.0.smithi107.stderr:+ cmp -l /tmp/rbd_import_export_644583/sparse2 /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:37.503 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse2 --format %b 2023-10-09T06:44:37.504 INFO:tasks.workunit.client.0.smithi107.stderr:+ origsize=6144 2023-10-09T06:44:37.504 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse2.out --format %b 2023-10-09T06:44:37.505 INFO:tasks.workunit.client.0.smithi107.stderr:+ exportsize=6144 2023-10-09T06:44:37.505 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:37.505 INFO:tasks.workunit.client.0.smithi107.stderr:+ difference=0 2023-10-09T06:44:37.505 INFO:tasks.workunit.client.0.smithi107.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T06:44:37.505 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse2.out 2023-10-09T06:44:37.506 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse2 2023-10-09T06:44:37.589 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T06:44:37.592 INFO:tasks.workunit.client.0.smithi107.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T06:44:37.593 INFO:tasks.workunit.client.0.smithi107.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T06:44:37.593 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_644583/sparse1 sparse 2023-10-09T06:44:37.653 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T06:44:37.653 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:37.653 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T06:44:37.656 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse 2023-10-09T06:44:37.656 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse 2023-10-09T06:44:37.656 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse 2023-10-09T06:44:37.657 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:37.657 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:37.692 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bb6d68c6b9c6 2023-10-09T06:44:37.693 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bb6d68c6b9c6 2023-10-09T06:44:37.693 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse 2023-10-09T06:44:37.693 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse 2023-10-09T06:44:37.693 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bb6d68c6b9c6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:37.693 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:37.694 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse 2023-10-09T06:44:37.694 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:37.694 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:37.728 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:37.728 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:37.728 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:37.728 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:37.728 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:37.759 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects=1 2023-10-09T06:44:37.759 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 1 2023-10-09T06:44:37.759 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T06:44:37.760 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse 2023-10-09T06:44:37.843 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T06:44:37.847 INFO:tasks.workunit.client.0.smithi107.stdout:zeros import through stdin to sparse image 2023-10-09T06:44:37.848 INFO:tasks.workunit.client.0.smithi107.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T06:44:37.848 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T06:44:37.848 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd import --new-format - sparse 2023-10-09T06:44:37.894 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:37.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:37.901 INFO:tasks.workunit.client.0.smithi107.stderr:4+0 records in 2023-10-09T06:44:37.902 INFO:tasks.workunit.client.0.smithi107.stderr:4+0 records out 2023-10-09T06:44:37.902 INFO:tasks.workunit.client.0.smithi107.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0536699 s, 78.2 MB/s 2023-10-09T06:44:37.913 INFO:tasks.workunit.client.0.smithi107.stderr: Importing image: 100% complete...done. 2023-10-09T06:44:37.913 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:37.916 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse 2023-10-09T06:44:37.916 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse 2023-10-09T06:44:37.916 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse 2023-10-09T06:44:37.916 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:37.916 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:37.955 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bb8e69caff4c 2023-10-09T06:44:37.955 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bb8e69caff4c 2023-10-09T06:44:37.955 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse 2023-10-09T06:44:37.955 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bb8e69caff4c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:37.956 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse 2023-10-09T06:44:37.956 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:37.956 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse 2023-10-09T06:44:37.956 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:37.957 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:37.994 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:37.994 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:37.994 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:37.994 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:37.994 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:38.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T06:44:38.031 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects= 2023-10-09T06:44:38.031 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 2023-10-09T06:44:38.031 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T06:44:38.031 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse 2023-10-09T06:44:38.047 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:38.126 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:38.129 INFO:tasks.workunit.client.0.smithi107.stdout:zeros export to sparse file 2023-10-09T06:44:38.129 INFO:tasks.workunit.client.0.smithi107.stderr:+ echo 'zeros export to sparse file' 2023-10-09T06:44:38.129 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T06:44:38.166 INFO:tasks.workunit.client.0.smithi107.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T06:44:38.168 INFO:tasks.workunit.client.0.smithi107.stderr:++ rbd info sparse 2023-10-09T06:44:38.168 INFO:tasks.workunit.client.0.smithi107.stderr:++ grep block_name_prefix 2023-10-09T06:44:38.169 INFO:tasks.workunit.client.0.smithi107.stderr:++ awk '{print $NF;}' 2023-10-09T06:44:38.205 INFO:tasks.workunit.client.0.smithi107.stderr:+ prefix=rbd_data.bbaf111722d7 2023-10-09T06:44:38.205 INFO:tasks.workunit.client.0.smithi107.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T06:44:38.206 INFO:tasks.workunit.client.0.smithi107.stderr:++ get_image_data_pool sparse 2023-10-09T06:44:38.206 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse 2023-10-09T06:44:38.206 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse 2023-10-09T06:44:38.206 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep 'data_pool: ' 2023-10-09T06:44:38.206 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T06:44:38.212 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T06:44:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:38.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T06:44:38.244 INFO:tasks.workunit.client.0.smithi107.stderr:++ data_pool= 2023-10-09T06:44:38.244 INFO:tasks.workunit.client.0.smithi107.stderr:++ '[' -z ']' 2023-10-09T06:44:38.244 INFO:tasks.workunit.client.0.smithi107.stderr:++ data_pool=rbd 2023-10-09T06:44:38.245 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo rbd 2023-10-09T06:44:38.245 INFO:tasks.workunit.client.0.smithi107.stderr:+ rados -p rbd put rbd_data.bbaf111722d7.000000000000 - 2023-10-09T06:44:38.273 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records in 2023-10-09T06:44:38.273 INFO:tasks.workunit.client.0.smithi107.stderr:1+0 records out 2023-10-09T06:44:38.274 INFO:tasks.workunit.client.0.smithi107.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.067724 s, 61.9 MB/s 2023-10-09T06:44:38.290 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects sparse 2023-10-09T06:44:38.290 INFO:tasks.workunit.client.0.smithi107.stderr:++ image=sparse 2023-10-09T06:44:38.290 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rbd info sparse 2023-10-09T06:44:38.290 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep block_name_prefix 2023-10-09T06:44:38.290 INFO:tasks.workunit.client.0.smithi107.stderr:+++ awk '{print $NF;}' 2023-10-09T06:44:38.317 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T06:44:38.327 INFO:tasks.workunit.client.0.smithi107.stderr:++ prefix=rbd_data.bbaf111722d7 2023-10-09T06:44:38.327 INFO:tasks.workunit.client.0.smithi107.stderr:+++ grep rbd_data.bbaf111722d7 2023-10-09T06:44:38.327 INFO:tasks.workunit.client.0.smithi107.stderr:++++ get_image_data_pool sparse 2023-10-09T06:44:38.327 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sed -e 's/rbd_data.bbaf111722d7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T06:44:38.328 INFO:tasks.workunit.client.0.smithi107.stderr:++++ image=sparse 2023-10-09T06:44:38.328 INFO:tasks.workunit.client.0.smithi107.stderr:+++ sort -u 2023-10-09T06:44:38.328 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ rbd info sparse 2023-10-09T06:44:38.328 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ grep 'data_pool: ' 2023-10-09T06:44:38.328 INFO:tasks.workunit.client.0.smithi107.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T06:44:38.347 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:38.368 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool= 2023-10-09T06:44:38.368 INFO:tasks.workunit.client.0.smithi107.stderr:++++ '[' -z ']' 2023-10-09T06:44:38.368 INFO:tasks.workunit.client.0.smithi107.stderr:++++ data_pool=rbd 2023-10-09T06:44:38.368 INFO:tasks.workunit.client.0.smithi107.stderr:++++ echo rbd 2023-10-09T06:44:38.369 INFO:tasks.workunit.client.0.smithi107.stderr:+++ rados ls -p rbd 2023-10-09T06:44:38.400 INFO:tasks.workunit.client.0.smithi107.stderr:++ objects=0 2023-10-09T06:44:38.400 INFO:tasks.workunit.client.0.smithi107.stderr:++ echo 0 2023-10-09T06:44:38.401 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T06:44:38.401 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse 2023-10-09T06:44:38.401 INFO:tasks.workunit.client.0.smithi107.stderr:rm: cannot remove '/tmp/rbd_import_export_644583/sparse': No such file or directory 2023-10-09T06:44:38.402 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:38.402 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd export sparse /tmp/rbd_import_export_644583/sparse 2023-10-09T06:44:38.458 INFO:tasks.workunit.client.0.smithi107.stderr: Exporting image: 100% complete...done. 2023-10-09T06:44:38.462 INFO:tasks.workunit.client.0.smithi107.stderr:++ stat /tmp/rbd_import_export_644583/sparse --format=%b 2023-10-09T06:44:38.463 INFO:tasks.workunit.client.0.smithi107.stderr:+ '[' 0 = 0 ']' 2023-10-09T06:44:38.463 INFO:tasks.workunit.client.0.smithi107.stderr:+ rbd rm sparse 2023-10-09T06:44:38.526 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T06:44:38.542 INFO:tasks.workunit.client.0.smithi107.stderr: Removing image: 100% complete...done. 2023-10-09T06:44:38.546 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm /tmp/rbd_import_export_644583/sparse /tmp/rbd_import_export_644583/sparse1 /tmp/rbd_import_export_644583/sparse2 /tmp/rbd_import_export_644583/sparse3 2023-10-09T06:44:38.548 INFO:tasks.workunit.client.0.smithi107.stderr:rm: cannot remove '/tmp/rbd_import_export_644583/sparse3': No such file or directory 2023-10-09T06:44:38.548 INFO:tasks.workunit.client.0.smithi107.stderr:+ true 2023-10-09T06:44:38.551 INFO:tasks.workunit.client.0.smithi107.stdout:OK 2023-10-09T06:44:38.552 INFO:tasks.workunit.client.0.smithi107.stderr:+ echo OK 2023-10-09T06:44:38.552 INFO:tasks.workunit.client.0.smithi107.stderr:+ rm -rf /tmp/rbd_import_export_644583 2023-10-09T06:44:38.552 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T06:44:38.552 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T06:44:38.630 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T06:44:38.631 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T06:44:38.782 INFO:teuthology.orchestra.run.smithi107.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-10-09T06:44:39.034 DEBUG:teuthology.parallel:result is None 2023-10-09T06:44:39.035 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:39.070 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:39.070 DEBUG:teuthology.orchestra.run.smithi107:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:39.098 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T06:44:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:39.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T06:44:39.116 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T06:44:39.116 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T06:44:39.117 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T06:44:39.117 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T06:44:39.117 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T06:44:39.117 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T06:44:39.118 DEBUG:teuthology.orchestra.run.smithi107:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T06:44:39.173 INFO:teuthology.orchestra.run.smithi107.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T06:44:39.174 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:39.202 DEBUG:teuthology.orchestra.run.smithi107:> sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[212750]: pgmap v1739: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 84 KiB/s rd, 6.8 KiB/s wr, 109 op/s 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1192553232' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[220921]: pgmap v1739: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 84 KiB/s rd, 6.8 KiB/s wr, 109 op/s 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1192553232' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.243 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.247 INFO:teuthology.orchestra.run.smithi107.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T06:44:39.248 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T06:44:39.248 DEBUG:teuthology.orchestra.run.smithi107:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T06:44:39.306 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T06:44:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:38 smithi150 ceph-mon[194178]: pgmap v1739: 103 pgs: 103 active+clean; 802 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 84 KiB/s rd, 6.8 KiB/s wr, 109 op/s 2023-10-09T06:44:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:38 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1192553232' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-10-09T06:44:39.742 INFO:teuthology.orchestra.run.smithi107.stderr:Error: no container with name or ID ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T06:44:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T06:44:39.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi107 with status 125: 'sudo docker kill -s 1 ceph-b7a98308-6667-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T06:44:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:44:40.001 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:44:40.029 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:39 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:40.029 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:39 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:39 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:40.516 INFO:teuthology.orchestra.run.smithi107.stdout:ERROR: (22) Invalid argument 2023-10-09T06:44:40.517 INFO:teuthology.orchestra.run.smithi107.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T06:44:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[212750]: pgmap v1740: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 774 KiB/s rd, 604 KiB/s wr, 257 op/s 2023-10-09T06:44:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[212750]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:44:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:41.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[212750]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T06:44:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[220921]: pgmap v1740: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 774 KiB/s rd, 604 KiB/s wr, 257 op/s 2023-10-09T06:44:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[220921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[220921]: failed to encode map e574 with expected crc 2023-10-09T06:44:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:41.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:40 smithi107 ceph-mon[220921]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T06:44:41.342 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:41.362 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T06:44:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:40 smithi150 ceph-mon[194178]: pgmap v1740: 103 pgs: 103 active+clean; 582 KiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 774 KiB/s rd, 604 KiB/s wr, 257 op/s 2023-10-09T06:44:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:40 smithi150 ceph-mon[194178]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T06:44:41.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:40 smithi150 ceph-mon[194178]: failed to encode map e574 with expected crc 2023-10-09T06:44:41.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1211161607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:41.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:40 smithi150 ceph-mon[194178]: osdmap e574: 8 total, 8 up, 6 in 2023-10-09T06:44:42.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:42] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2023-10-09T06:44:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[212750]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[220921]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[220921]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[220921]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:41 smithi107 ceph-mon[220921]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.317 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:41 smithi150 ceph-mon[194178]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:41 smithi150 ceph-mon[194178]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:41 smithi150 ceph-mon[194178]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:41 smithi150 ceph-mon[194178]: failed to encode map e574 with expected crc 2023-10-09T06:44:42.856 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T06:44:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[212750]: pgmap v1742: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1022 KiB/s rd, 1.4 MiB/s wr, 328 op/s 2023-10-09T06:44:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[212750]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T06:44:43.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[220921]: pgmap v1742: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1022 KiB/s rd, 1.4 MiB/s wr, 328 op/s 2023-10-09T06:44:43.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[220921]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T06:44:43.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:42 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:42 smithi150 ceph-mon[194178]: pgmap v1742: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1022 KiB/s rd, 1.4 MiB/s wr, 328 op/s 2023-10-09T06:44:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:42 smithi150 ceph-mon[194178]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T06:44:43.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:42 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:43.636 INFO:teuthology.orchestra.run.smithi107.stdout:osd.0: {} 2023-10-09T06:44:43.655 INFO:teuthology.orchestra.run.smithi107.stdout:osd.1: {} 2023-10-09T06:44:43.676 INFO:teuthology.orchestra.run.smithi107.stdout:osd.2: {} 2023-10-09T06:44:43.698 INFO:teuthology.orchestra.run.smithi107.stdout:osd.3: {} 2023-10-09T06:44:43.718 INFO:teuthology.orchestra.run.smithi107.stdout:osd.4: {} 2023-10-09T06:44:43.737 INFO:teuthology.orchestra.run.smithi107.stdout:osd.5: {} 2023-10-09T06:44:43.760 INFO:teuthology.orchestra.run.smithi107.stdout:osd.6: {} 2023-10-09T06:44:43.780 INFO:teuthology.orchestra.run.smithi107.stdout:osd.7: {} 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.781 INFO:teuthology.orchestra.run.smithi107.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.782 INFO:teuthology.orchestra.run.smithi107.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T06:44:43.879 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:43.980 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:44:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:44:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:44:43.981 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:44:43.981 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:44:43.998 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: failed to encode map e575 with expected crc 2023-10-09T06:44:43.998 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: failed to encode map e575 with expected crc 2023-10-09T06:44:43.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: failed to encode map e575 with expected crc 2023-10-09T06:44:43.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: failed to encode map e575 with expected crc 2023-10-09T06:44:43.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: 1.6 scrub starts 2023-10-09T06:44:43.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: 1.6 scrub ok 2023-10-09T06:44:43.999 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3664989040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:44.331 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: 1.6 scrub starts 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: 1.6 scrub ok 2023-10-09T06:44:44.332 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3664989040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: failed to encode map e575 with expected crc 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: 1.6 scrub starts 2023-10-09T06:44:44.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: 1.6 scrub ok 2023-10-09T06:44:44.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3664989040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T06:44:44.915 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:44:44.915 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T06:44:44.916 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[212750]: 1.11 scrub starts 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[212750]: 1.11 scrub ok 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[212750]: pgmap v1744: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[220921]: 1.11 scrub starts 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[220921]: 1.11 scrub ok 2023-10-09T06:44:45.226 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:44 smithi107 ceph-mon[220921]: pgmap v1744: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s 2023-10-09T06:44:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:44 smithi150 ceph-mon[194178]: 1.11 scrub starts 2023-10-09T06:44:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:44 smithi150 ceph-mon[194178]: 1.11 scrub ok 2023-10-09T06:44:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:44 smithi150 ceph-mon[194178]: pgmap v1744: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 356 op/s 2023-10-09T06:44:45.493 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T06:44:45.902 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:45.988 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:47.012 INFO:teuthology.orchestra.run.smithi107.stderr:noscrub is unset 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[220921]: pgmap v1745: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1008 KiB/s rd, 1.4 MiB/s wr, 322 op/s 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2978040461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:47.377 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[212750]: pgmap v1745: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1008 KiB/s rd, 1.4 MiB/s wr, 322 op/s 2023-10-09T06:44:47.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2978040461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:47.378 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:47 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:47 smithi150 ceph-mon[194178]: pgmap v1745: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 1008 KiB/s rd, 1.4 MiB/s wr, 322 op/s 2023-10-09T06:44:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:47 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2978040461' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T06:44:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:44:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:47 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:44:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[212750]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T06:44:48.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:44:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[220921]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T06:44:48.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:48 smithi107 ceph-mon[220921]: failed to encode map e576 with expected crc 2023-10-09T06:44:48.401 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T06:44:48.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:48 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T06:44:48.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:48 smithi150 ceph-mon[194178]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T06:44:48.441 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:48 smithi150 ceph-mon[194178]: failed to encode map e576 with expected crc 2023-10-09T06:44:48.627 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194026 12.e 12:7a8b5efc:::benchmark_data_smithi107_479504_object94415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:44:48.628 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 590.69150114599995, 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35583712499999998, 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.629 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:44:48.630 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:48.630 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194026 2023-10-09T06:44:48.630 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.630 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.630 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.631 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.631 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:44:48.631 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.631 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.631 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.632 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931254+0000", 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.633 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931257+0000", 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7099999999999999e-06 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931263+0000", 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2750000000000001e-06 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.634 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931268+0000", 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931301+0000", 2023-10-09T06:44:48.635 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3293999999999997e-05 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931411+0000", 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109904 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931511+0000", 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9732e-05 2023-10-09T06:44:48.636 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940660+0000", 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091492589999999999 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.940805+0000", 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014499900000000001 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.637 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946883+0000", 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006077753 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946930+0000", 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6730000000000002e-05 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.638 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287091+0000", 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016162900000002 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194028 12.e 12:78e231c3:::benchmark_data_smithi107_479504_object94417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 590.69147579699995, 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35590891000000002, 2023-10-09T06:44:48.639 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.640 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.640 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.640 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:44:48.640 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:48.640 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194028 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.641 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931280+0000", 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931279+0000", 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.642 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931282+0000", 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6410000000000002e-06 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931288+0000", 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9329999999999998e-06 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931293+0000", 2023-10-09T06:44:48.643 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.147e-06 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936701+0000", 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0054079769999999996 2023-10-09T06:44:48.644 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936818+0000", 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011694 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.645 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.646 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:44:48.646 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.936931+0000", 2023-10-09T06:44:48.646 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011294999999999999 2023-10-09T06:44:48.646 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.646 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946850+0000", 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099191780000000007 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.647 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.648 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946867+0000", 2023-10-09T06:44:48.648 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6532e-05 2023-10-09T06:44:48.648 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.648 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.648 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287104+0000", 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34023740699999999 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.649 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287151+0000", 2023-10-09T06:44:48.650 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7079000000000001e-05 2023-10-09T06:44:48.650 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.650 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.650 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.650 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287189+0000", 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7576000000000001e-05 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.651 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.652 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:194030 12.a 12:5bb1da5c:::benchmark_data_smithi107_479504_object94419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-10-09T06:44:48.652 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:44:48.652 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 590.69145843700005, 2023-10-09T06:44:48.652 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.35582703900000001, 2023-10-09T06:44:48.652 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.653 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.653 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.653 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:44:48.653 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:44:48.653 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 194030 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.654 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931297+0000", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931299+0000", 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0710000000000002e-06 2023-10-09T06:44:48.655 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931304+0000", 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9439999999999996e-06 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.931309+0000", 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4859999999999997e-06 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.656 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938266+0000", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069569489999999996 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938372+0000", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105999 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,3", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.938476+0000", 2023-10-09T06:44:48.657 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103669 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.942149+0000", 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003673277 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.945613+0000", 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034637270000000002 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.658 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:57.946916+0000", 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013029070000000001 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287077+0000", 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.34016102300000001 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.659 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:34:58.287124+0000", 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7277999999999997e-05 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98792 13.13 13:cfe6566e:::benchmark_data_smithi107_535588_object98791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 472.36912813499998, 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97467514200000005, 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:44:48.660 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98792 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.661 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253627+0000", 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253625+0000", 2023-10-09T06:44:48.662 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253660+0000", 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5043000000000003e-05 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253679+0000", 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9123999999999999e-05 2023-10-09T06:44:48.663 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.253690+0000", 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0679000000000001e-05 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259065+0000", 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0053750710000000004 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.664 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.259078+0000", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3473e-05 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228190+0000", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.96911134899999996 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.228302+0000", 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000112927 2023-10-09T06:44:48.665 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:98797 13.3 13:c2aca6be:::benchmark_data_smithi107_535588_object98796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 472.269633333, 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.85765422099999999, 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.666 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 98797 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353122+0000", 2023-10-09T06:44:48.667 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353120+0000", 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999976 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353150+0000", 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0685000000000002e-05 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.668 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353166+0000", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5914e-05 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353175+0000", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.935e-06 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353222+0000", 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6635999999999999e-05 2023-10-09T06:44:48.669 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:56.353232+0000", 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0478e-05 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188558+0000", 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83532556599999996 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.670 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188732+0000", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000173661 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.188879+0000", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014706499999999999 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.192739+0000", 2023-10-09T06:44:48.671 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038604580000000002 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.201831+0000", 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090918390000000009 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210658+0000", 2023-10-09T06:44:48.672 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088270789999999998 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210725+0000", 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7260000000000003e-05 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:36:57.210776+0000", 2023-10-09T06:44:48.673 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1085000000000003e-05 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:177412 14.b 14:de08f02e:::benchmark_data_smithi107_590976_object76639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:54.939639+0000", 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 113.683116166, 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20944237700000001, 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.674 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 177412 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939639+0000", 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.675 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939639+0000", 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939638+0000", 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939642+0000", 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8840000000000004e-06 2023-10-09T06:44:48.676 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939649+0000", 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6780000000000003e-06 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939654+0000", 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2190000000000003e-06 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.677 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939700+0000", 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5713000000000001e-05 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:54.939823+0000", 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012280400000000001 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.127218+0000", 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.187394905 2023-10-09T06:44:48.678 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.134956+0000", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077381919999999996 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.148889+0000", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013933759 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.149020+0000", 2023-10-09T06:44:48.679 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000130347 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.149052+0000", 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2381e-05 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:55.149082+0000", 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9468e-05 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.680 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182979 14.13 14:cc63cc27:::benchmark_data_smithi107_590976_object82206:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.943470+0000", 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.679285906, 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20794283699999999, 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.681 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182979 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943470+0000", 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943470+0000", 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.682 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943469+0000", 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943472+0000", 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5560000000000002e-06 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.683 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943477+0000", 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7039999999999999e-06 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.943482+0000", 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7489999999999998e-06 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.946333+0000", 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028503539999999998 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.684 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.947519+0000", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001186049 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.958217+0000", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010698882999999999 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.958316+0000", 2023-10-09T06:44:48.685 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8392000000000003e-05 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.958813+0000", 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000496742 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963386+0000", 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045734800000000004 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.686 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.968400+0000", 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050136529999999999 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.969740+0000", 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0013399729999999999 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.687 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.150559+0000", 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.180819534 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.150616+0000", 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6558999999999998e-05 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.151412+0000", 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00079662299999999999 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.688 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182982 14.c 14:3a84bf7a:::benchmark_data_smithi107_590976_object82209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.951794+0000", 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.670961445, 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.210340308, 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.689 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182982 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951794+0000", 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951794+0000", 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.690 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951793+0000", 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951797+0000", 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6430000000000001e-06 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.691 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951804+0000", 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7039999999999997e-06 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951809+0000", 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4210000000000003e-06 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951848+0000", 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8472e-05 2023-10-09T06:44:48.692 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951962+0000", 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011390799999999999 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.952069+0000", 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010728 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.693 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960286+0000", 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.008217271 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963795+0000", 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035090730000000001 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162037+0000", 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.198241735 2023-10-09T06:44:48.694 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162097+0000", 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9997999999999997e-05 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162134+0000", 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7351000000000002e-05 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.695 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182983 14.c 14:31c3a09d:::benchmark_data_smithi107_590976_object82210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.951819+0000", 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.67093601099999, 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21044273299999999, 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182983 2023-10-09T06:44:48.696 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951819+0000", 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951819+0000", 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.697 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951819+0000", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951822+0000", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5490000000000001e-06 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951827+0000", 2023-10-09T06:44:48.698 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6740000000000002e-06 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951832+0000", 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1760000000000001e-06 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955058+0000", 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0032257169999999999 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.699 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955186+0000", 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000127584 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955317+0000", 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013160900000000001 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.700 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.967459+0000", 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012141091 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.970006+0000", 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025479740000000002 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162177+0000", 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19217034399999999 2023-10-09T06:44:48.701 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162221+0000", 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4530999999999998e-05 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162262+0000", 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0837000000000002e-05 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.702 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182984 14.16 14:6e96a768:::benchmark_data_smithi107_590976_object82211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.951838+0000", 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.670916984, 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.202543427, 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.703 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182984 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951838+0000", 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951838+0000", 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.704 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951838+0000", 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951840+0000", 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3879999999999998e-06 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.705 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951845+0000", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8470000000000003e-06 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951850+0000", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.3499999999999999e-06 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955285+0000", 2023-10-09T06:44:48.706 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003435114 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955399+0000", 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114162 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.955835+0000", 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043564299999999999 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.707 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.962862+0000", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070274400000000002 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.962997+0000", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00013501999999999999 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.154296+0000", 2023-10-09T06:44:48.708 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19129910999999999 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.154345+0000", 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8469e-05 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.154382+0000", 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7285e-05 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.709 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182985 14.16 14:6999af59:::benchmark_data_smithi107_590976_object82212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.951858+0000", 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.670897694, 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.2104277, 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.710 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182985 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951858+0000", 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951858+0000", 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.711 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951857+0000", 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951860+0000", 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3740000000000001e-06 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.712 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951866+0000", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6829999999999996e-06 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951871+0000", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1000000000000003e-06 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.959641+0000", 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077701089999999999 2023-10-09T06:44:48.713 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.959755+0000", 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114239 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960153+0000", 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00039759099999999998 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.714 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963361+0000", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003207971 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.964871+0000", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001510801 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162210+0000", 2023-10-09T06:44:48.715 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19733832400000001 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162251+0000", 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1156999999999999e-05 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.162285+0000", 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4650999999999997e-05 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.716 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182986 14.c 14:3fe8a0bc:::benchmark_data_smithi107_590976_object82213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.951875+0000", 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.670880564, 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21442297599999999, 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.717 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182986 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951875+0000", 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951875+0000", 2023-10-09T06:44:48.718 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951875+0000", 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951877+0000", 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6299999999999998e-06 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.719 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951883+0000", 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4670000000000003e-06 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.951887+0000", 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4819999999999996e-06 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.720 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960317+0000", 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0084299200000000005 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960418+0000", 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010071 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960522+0000", 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000103677 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.721 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.967537+0000", 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0070157980000000002 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.970041+0000", 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002503178 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.722 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166207+0000", 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19616614900000001 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166259+0000", 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2593000000000003e-05 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166298+0000", 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8631999999999999e-05 2023-10-09T06:44:48.723 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182989 14.c 14:3a80d608:::benchmark_data_smithi107_590976_object82216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.960138+0000", 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.662617234, 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.206262626, 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.724 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182989 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960138+0000", 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.725 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960138+0000", 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960137+0000", 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960145+0000", 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0819999999999999e-06 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.726 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960167+0000", 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1838e-05 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960177+0000", 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0356e-05 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.727 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963828+0000", 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003650749 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963973+0000", 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014507100000000001 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.964099+0000", 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012611299999999999 2023-10-09T06:44:48.728 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.967680+0000", 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035809510000000002 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.975199+0000", 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0075191930000000004 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.729 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166341+0000", 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19114188200000001 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166373+0000", 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1733999999999998e-05 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.166401+0000", 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8093999999999999e-05 2023-10-09T06:44:48.730 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:182991 14.13 14:cd343dfd:::benchmark_data_smithi107_590976_object82218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:02.960185+0000", 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 105.66257078, 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.21202701099999999, 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.731 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 182991 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960185+0000", 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.732 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960185+0000", 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960184+0000", 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960187+0000", 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1669999999999999e-06 2023-10-09T06:44:48.733 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960193+0000", 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0630000000000002e-06 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.960199+0000", 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2569999999999998e-06 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.734 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.961902+0000", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001703097 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:02.963361+0000", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014596380000000001 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.153694+0000", 2023-10-09T06:44:48.735 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.19033257100000001 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.153796+0000", 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101827 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4,5", 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.154358+0000", 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00056230400000000002 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.736 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.158029+0000", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036704289999999998 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.159127+0000", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0010987810000000001 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.159471+0000", 2023-10-09T06:44:48.737 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00034387699999999997 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.171567+0000", 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.012096086000000001 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.171613+0000", 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5791999999999998e-05 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.738 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:03.172212+0000", 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00059859099999999999 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:198472 14.15 14:a811cbc5:::benchmark_data_smithi107_590976_object97699:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:24.833588+0000", 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 83.789167089000003, 2023-10-09T06:44:48.739 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20720122299999999, 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 198472 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833588+0000", 2023-10-09T06:44:48.740 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833588+0000", 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833587+0000", 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.741 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.742 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833595+0000", 2023-10-09T06:44:48.742 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5189999999999992e-06 2023-10-09T06:44:48.742 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.742 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833609+0000", 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4027e-05 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833619+0000", 2023-10-09T06:44:48.743 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9830000000000008e-06 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833873+0000", 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00025377899999999998 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833987+0000", 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000114153 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.744 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.834510+0000", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052247600000000002 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.837438+0000", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002927954 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040621+0000", 2023-10-09T06:44:48.745 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20318361099999999 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040710+0000", 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8592999999999997e-05 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040755+0000", 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5127999999999999e-05 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.746 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040790+0000", 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4641000000000003e-05 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:198473 14.c 14:3b7ae9c4:::benchmark_data_smithi107_590976_object97700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:24.833636+0000", 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 83.789119698999997, 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20711771700000001, 2023-10-09T06:44:48.747 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 198473 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833636+0000", 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.748 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833636+0000", 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833635+0000", 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.749 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833639+0000", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2160000000000002e-06 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833645+0000", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4699999999999999e-06 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833650+0000", 2023-10-09T06:44:48.750 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.395e-06 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833702+0000", 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1233999999999998e-05 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833820+0000", 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000118633 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.751 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.833924+0000", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000104217 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.837448+0000", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035239669999999998 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.837667+0000", 2023-10-09T06:44:48.752 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00021846699999999999 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040660+0000", 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20299329899999999 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040712+0000", 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1329000000000003e-05 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.753 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040753+0000", 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1952000000000001e-05 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:198476 14.8 14:10ae84a8:::benchmark_data_smithi107_590976_object97703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:24.836499+0000", 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 83.786256253999994, 2023-10-09T06:44:48.754 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20426994900000001, 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 198476 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836499+0000", 2023-10-09T06:44:48.755 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836499+0000", 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836498+0000", 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.756 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836505+0000", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0600000000000002e-06 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836517+0000", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1875999999999999e-05 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836525+0000", 2023-10-09T06:44:48.757 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1049999999999995e-06 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836622+0000", 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7472000000000005e-05 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.836735+0000", 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011299899999999999 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.758 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.837273+0000", 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00053727500000000004 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.840036+0000", 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027629740000000001 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.759 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040638+0000", 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20060252200000001 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040679+0000", 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0877000000000002e-05 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040733+0000", 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3823000000000003e-05 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.760 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040769+0000", 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6303000000000001e-05 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:198479 14.5 14:a1aabccf:::benchmark_data_smithi107_590976_object97706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:24.839760+0000", 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 83.782995120999999, 2023-10-09T06:44:48.761 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.20826924199999999, 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 198479 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839760+0000", 2023-10-09T06:44:48.762 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839760+0000", 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839759+0000", 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.763 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839764+0000", 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0799999999999996e-06 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839776+0000", 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1926999999999999e-05 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.764 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839784+0000", 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1869999999999997e-06 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839834+0000", 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0305e-05 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839946+0000", 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000111048 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.765 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,7", 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.040601+0000", 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.200655692 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.044352+0000", 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0037511250000000001 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.766 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.044388+0000", 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5564e-05 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.047953+0000", 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0035646380000000002 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.047995+0000", 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2307e-05 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.767 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.048030+0000", 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4755000000000002e-05 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:198480 14.b 14:d4905134:::benchmark_data_smithi107_590976_object97707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:24.839798+0000", 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 83.782957389000003, 2023-10-09T06:44:48.768 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.22474986799999999, 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 198480 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839798+0000", 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.769 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839798+0000", 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839798+0000", 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.770 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839800+0000", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8219999999999998e-06 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839807+0000", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5799999999999997e-06 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839812+0000", 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4559999999999999e-06 2023-10-09T06:44:48.771 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839854+0000", 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1292999999999999e-05 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.839962+0000", 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108595 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:44:48.772 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:24.840071+0000", 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108534 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.048018+0000", 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.207947512 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.048082+0000", 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3508000000000002e-05 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.773 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.064460+0000", 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.016378493000000001 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.064509+0000", 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8571e-05 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:44:48.774 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:25.064548+0000", 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8964999999999997e-05 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:44:48.775 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:44:49.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[212750]: pgmap v1747: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 376 KiB/s rd, 935 KiB/s wr, 139 op/s 2023-10-09T06:44:49.335 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:49 smithi107 conmon[212706]: 2023-10-09T06:44:49.035+0000 7f7c794f9700 -1 mon.a@0(leader).osd e576 definitely_dead 0 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[220921]: pgmap v1747: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 376 KiB/s rd, 935 KiB/s wr, 139 op/s 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[220921]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[220921]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.336 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:49 smithi107 ceph-mon[220921]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.398 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T06:44:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:49 smithi150 ceph-mon[194178]: pgmap v1747: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 376 KiB/s rd, 935 KiB/s wr, 139 op/s 2023-10-09T06:44:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:49 smithi150 ceph-mon[194178]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:49 smithi150 ceph-mon[194178]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:49 smithi150 ceph-mon[194178]: failed to encode map e576 with expected crc 2023-10-09T06:44:49.488 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:50.048 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.0. 2023-10-09T06:44:50.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:50 smithi107 ceph-mon[212750]: failed to encode map e576 with expected crc 2023-10-09T06:44:50.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:50 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:44:50.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:50 smithi107 ceph-mon[220921]: failed to encode map e576 with expected crc 2023-10-09T06:44:50.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:50 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:44:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:50 smithi150 ceph-mon[194178]: failed to encode map e576 with expected crc 2023-10-09T06:44:50.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:50 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T06:44:51.083 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:51.084 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T06:44:51.084 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T06:44:51.084 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T06:44:51.084 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: pgmap v1748: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 26 KiB/s rd, 1.4 KiB/s wr, 33 op/s 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.390 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[212750]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: pgmap v1748: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 26 KiB/s rd, 1.4 KiB/s wr, 33 op/s 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.391 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.392 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.392 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.392 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:51 smithi107 ceph-mon[220921]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: pgmap v1748: 103 pgs: 103 active+clean; 4.6 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 26 KiB/s rd, 1.4 KiB/s wr, 33 op/s 2023-10-09T06:44:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1469575131' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T06:44:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: osdmap e577: 8 total, 8 up, 7 in 2023-10-09T06:44:51.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:51.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:51 smithi150 ceph-mon[194178]: failed to encode map e577 with expected crc 2023-10-09T06:44:52.047 INFO:teuthology.orchestra.run.smithi107.stderr:nodeep-scrub is unset 2023-10-09T06:44:52.366 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:44:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:44:52] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.43.0" 2023-10-09T06:44:52.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:44:52.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.366 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:52.367 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[212750]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:52.368 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:52.369 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:44:52.369 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:52.369 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:52 smithi107 ceph-mon[220921]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:44:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: osdmap e578: 8 total, 8 up, 7 in 2023-10-09T06:44:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: failed to encode map e578 with expected crc 2023-10-09T06:44:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T06:44:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T06:44:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:44:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3992729091' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T06:44:52.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:52 smithi150 ceph-mon[194178]: osdmap e579: 8 total, 8 up, 7 in 2023-10-09T06:44:52.911 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:53.298 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:44:53.298 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:44:53.298 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:44:53.298 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: pgmap v1751: 103 pgs: 19 peering, 4 unknown, 80 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.315 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 6 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[212750]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: pgmap v1751: 103 pgs: 19 peering, 4 unknown, 80 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.316 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 6 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.317 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:53 smithi107 ceph-mon[220921]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:44:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: pgmap v1751: 103 pgs: 19 peering, 4 unknown, 80 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail 2023-10-09T06:44:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 6 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: failed to encode map e579 with expected crc 2023-10-09T06:44:53.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:53 smithi150 ceph-mon[194178]: osdmap e580: 8 total, 8 up, 7 in 2023-10-09T06:44:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[212750]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.154 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:54 smithi107 ceph-mon[220921]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:44:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: failed to encode map e580 with expected crc 2023-10-09T06:44:54.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:54 smithi150 ceph-mon[194178]: osdmap e581: 8 total, 8 up, 7 in 2023-10-09T06:44:54.504 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T06:44:55.033 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: pgmap v1754: 103 pgs: 4 active+undersized+remapped, 1 remapped+peering, 20 peering, 3 unknown, 75 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 33 KiB/s, 4 objects/s recovering 2023-10-09T06:44:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[212750]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: pgmap v1754: 103 pgs: 4 active+undersized+remapped, 1 remapped+peering, 20 peering, 3 unknown, 75 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 33 KiB/s, 4 objects/s recovering 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:55 smithi107 ceph-mon[220921]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: pgmap v1754: 103 pgs: 4 active+undersized+remapped, 1 remapped+peering, 20 peering, 3 unknown, 75 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 8 op/s; 33 KiB/s, 4 objects/s recovering 2023-10-09T06:44:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:55 smithi150 ceph-mon[194178]: failed to encode map e581 with expected crc 2023-10-09T06:44:55.606 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:55 smithi107 conmon[212706]: 2023-10-09T06:44:55.281+0000 7f7c794f9700 -1 mon.a@0(leader).osd e581 definitely_dead 0 2023-10-09T06:44:56.091 INFO:teuthology.orchestra.run.smithi107.stderr:marked in osd.6. 2023-10-09T06:44:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:56 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:44:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:56 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:44:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:56 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T06:44:56.831 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T06:44:56.831 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T06:44:56.832 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 41 2023-10-09T06:44:56.990 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:44:56.990 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:44:56.990 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:44:56.990 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:44:56.990 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:44:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: 1.1 scrub starts 2023-10-09T06:44:57.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: 1.1 scrub ok 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: pgmap v1756: 103 pgs: 4 active+undersized+remapped, 8 peering, 91 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 7 op/s; 97 KiB/s, 11 objects/s recovering 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[212750]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: 1.1 scrub starts 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: 1.1 scrub ok 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: pgmap v1756: 103 pgs: 4 active+undersized+remapped, 8 peering, 91 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 7 op/s; 97 KiB/s, 11 objects/s recovering 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:57 smithi107 ceph-mon[220921]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: 1.1 scrub starts 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: 1.1 scrub ok 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: pgmap v1756: 103 pgs: 4 active+undersized+remapped, 8 peering, 91 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 7 op/s; 97 KiB/s, 11 objects/s recovering 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3885807446' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T06:44:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: osdmap e582: 8 total, 8 up, 8 in 2023-10-09T06:44:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:57 smithi150 ceph-mon[194178]: failed to encode map e582 with expected crc 2023-10-09T06:44:57.711 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T06:44:57.799 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[212750]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.364 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T06:44:58.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.365 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:58 smithi107 ceph-mon[220921]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: osdmap e583: 8 total, 8 up, 8 in 2023-10-09T06:44:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:58 smithi150 ceph-mon[194178]: failed to encode map e583 with expected crc 2023-10-09T06:44:58.854 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:44:59.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: pgmap v1759: 103 pgs: 4 active+undersized+remapped, 1 peering, 98 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 65 KiB/s, 10 objects/s recovering 2023-10-09T06:44:59.198 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.199 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.200 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.200 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[212750]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.200 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: pgmap v1759: 103 pgs: 4 active+undersized+remapped, 1 peering, 98 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 65 KiB/s, 10 objects/s recovering 2023-10-09T06:44:59.200 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.202 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.202 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:44:59 smithi107 ceph-mon[220921]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: pgmap v1759: 103 pgs: 4 active+undersized+remapped, 1 peering, 98 active+clean; 4.6 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 65 KiB/s, 10 objects/s recovering 2023-10-09T06:44:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: osdmap e584: 8 total, 8 up, 8 in 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:44:59.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:44:59 smithi150 ceph-mon[194178]: failed to encode map e584 with expected crc 2023-10-09T06:45:00.059 INFO:teuthology.orchestra.run.smithi107.stderr:Error EINVAL: specified pgp_num 41 > pg_num 31 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: 1.11 deep-scrub starts 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: 1.11 deep-scrub ok 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/4176082526' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: 1.11 deep-scrub starts 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: 1.11 deep-scrub ok 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/4176082526' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:00 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: 1.11 deep-scrub starts 2023-10-09T06:45:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: 1.11 deep-scrub ok 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: osdmap e585: 8 total, 8 up, 8 in 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: failed to encode map e585 with expected crc 2023-10-09T06:45:00.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:45:00.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/4176082526' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:00 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-10-09T06:45:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T06:45:00.785 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T06:45:00.951 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:00.951 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:45:00.951 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:45:00.951 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:45:00.953 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.953 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196534 12.13 12:cce4bd0b:::benchmark_data_smithi107_479504_object96923:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.953 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:45:00.953 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.315430225, 2023-10-09T06:45:00.953 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0267955550000001, 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196534 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.954 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631557+0000", 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631555+0000", 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:00.955 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631564+0000", 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.9290000000000004e-06 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631578+0000", 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4518000000000001e-05 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.956 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631588+0000", 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6190000000000006e-06 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631676+0000", 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8369999999999996e-05 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op must wait for map", 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631682+0000", 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.49e-06 2023-10-09T06:45:00.957 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634365+0000", 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026835169999999998 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634376+0000", 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0621999999999999e-05 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.958 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658287+0000", 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.023911 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658352+0000", 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5395999999999995e-05 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.959 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196538 12.13 12:cdcf32ec:::benchmark_data_smithi107_479504_object96927:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.315390686, 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0273450609999999, 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196538 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.960 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.961 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631596+0000", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631599+0000", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.411e-06 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631606+0000", 2023-10-09T06:45:00.962 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6889999999999998e-06 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631611+0000", 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2970000000000003e-06 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631755+0000", 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014380400000000001 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.963 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting_for_map not empty", 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.631760+0000", 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5670000000000002e-06 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634398+0000", 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026376569999999998 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.964 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634406+0000", 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6170000000000001e-06 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658849+0000", 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.024443005 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.658941+0000", 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.2565999999999994e-05 2023-10-09T06:45:00.965 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196551 12.13 12:cbc1bc89:::benchmark_data_smithi107_479504_object96940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.31251784699998, 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.039733314, 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.966 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196551 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.967 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634469+0000", 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634468+0000", 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634475+0000", 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0679999999999999e-06 2023-10-09T06:45:00.968 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634487+0000", 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1916000000000001e-05 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634495+0000", 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2039999999999997e-06 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.969 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634673+0000", 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000178653 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.634683+0000", 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3039999999999994e-06 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.970 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659290+0000", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0246074949999999 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659410+0000", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000119831 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.659844+0000", 2023-10-09T06:45:00.971 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043353199999999998 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.668999+0000", 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0091554749999999997 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671821+0000", 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002821839 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.972 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674089+0000", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0022680840000000001 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674150+0000", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0591000000000003e-05 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674202+0000", 2023-10-09T06:45:00.973 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2828000000000003e-05 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196554 12.13 12:cbabae2a:::benchmark_data_smithi107_479504_object96943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.30155835799997, 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0289864500000001, 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.974 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196554 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.975 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645429+0000", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645427+0000", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645435+0000", 2023-10-09T06:45:00.976 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1009999999999992e-06 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645449+0000", 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4063000000000001e-05 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645458+0000", 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4950000000000008e-06 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.977 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645541+0000", 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2985999999999995e-05 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.645553+0000", 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2082999999999999e-05 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.978 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.665916+0000", 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0203636300000001 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.666022+0000", 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010592 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.666461+0000", 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000438412 2023-10-09T06:45:00.979 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669034+0000", 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025731090000000001 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674222+0000", 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.005187779 2023-10-09T06:45:00.980 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674318+0000", 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6755000000000005e-05 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674371+0000", 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2889000000000001e-05 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.981 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674415+0000", 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.388e-05 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196590 12.13 12:c878c42d:::benchmark_data_smithi107_479504_object96979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.25729100700005, 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99367597600000002, 2023-10-09T06:45:00.982 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196590 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.983 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689696+0000", 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689695+0000", 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.984 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689701+0000", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2829999999999998e-06 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689713+0000", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1993e-05 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689721+0000", 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9650000000000008e-06 2023-10-09T06:45:00.985 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689774+0000", 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2966000000000001e-05 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.689786+0000", 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1858000000000001e-05 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.986 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669169+0000", 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97938338700000005 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669302+0000", 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0001326 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:45:00.987 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.669720+0000", 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041790600000000001 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674273+0000", 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045535749999999998 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674431+0000", 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00015736100000000001 2023-10-09T06:45:00.988 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683264+0000", 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0088337339999999993 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683324+0000", 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9746e-05 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.989 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.683372+0000", 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7858000000000002e-05 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46340.0:196596 12.13 12:cc65364a:::benchmark_data_smithi107_479504_object96985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 599.25360279200004, 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.99598029200000004, 2023-10-09T06:45:00.990 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46340", 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1565682414", 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 196596 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693384+0000", 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693382+0000", 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:00.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693389+0000", 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5930000000000002e-06 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693404+0000", 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5085999999999999e-05 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693413+0000", 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8019999999999998e-06 2023-10-09T06:45:00.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693622+0000", 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000208634 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:01.693632+0000", 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0638999999999999e-05 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671865+0000", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97823273200000005 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.671973+0000", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000107703 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,7", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.672280+0000", 2023-10-09T06:45:00.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00030727 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674438+0000", 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0021582810000000002 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.674519+0000", 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0468000000000003e-05 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689254+0000", 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014735259000000001 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689316+0000", 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1838e-05 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:00.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:35:02.689364+0000", 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8640999999999997e-05 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:205747 13.11 13:8972c788:::benchmark_data_smithi107_535588_object99640:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e482)", 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.748439+0000", 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 357.198548254, 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.978070828, 2023-10-09T06:45:00.998 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 205747 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748439+0000", 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:00.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748439+0000", 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748438+0000", 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748443+0000", 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7489999999999998e-06 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748454+0000", 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0847e-05 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748477+0000", 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3620000000000001e-05 2023-10-09T06:45:01.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748541+0000", 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3482e-05 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.748570+0000", 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.955e-05 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:04.726431+0000", 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97786040600000002 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:04.726510+0000", 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8930999999999994e-05 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.003 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169262 14.12 14:4c635361:::benchmark_data_smithi107_590976_object68489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:39.999667+0000", 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.94732041500001, 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2420530030000001, 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169262 2023-10-09T06:45:01.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999667+0000", 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999667+0000", 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999665+0000", 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999669+0000", 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5170000000000001e-06 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999678+0000", 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.0389999999999993e-06 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:39.999685+0000", 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6630000000000004e-06 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.000321+0000", 2023-10-09T06:45:01.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00063626299999999998 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.000332+0000", 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0392e-05 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980822+0000", 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.98049031399999997 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980837+0000", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4799e-05 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.159214+0000", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.178377437 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.159873+0000", 2023-10-09T06:45:01.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00065889900000000001 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.170626+0000", 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010753151000000001 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.170689+0000", 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2155000000000004e-05 2023-10-09T06:45:01.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.171198+0000", 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00050938800000000005 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.177291+0000", 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006093528 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220768+0000", 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.043476263000000001 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.240801+0000", 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.020033649000000001 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.240865+0000", 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3151000000000006e-05 2023-10-09T06:45:01.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.241720+0000", 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00085500300000000001 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169269 14.12 14:4d4b4c75:::benchmark_data_smithi107_590976_object68496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.006073+0000", 2023-10-09T06:45:01.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.94091401399999, 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2369730780000001, 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169269 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006073+0000", 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006073+0000", 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006072+0000", 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006078+0000", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7420000000000003e-06 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006089+0000", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1559000000000001e-05 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006097+0000", 2023-10-09T06:45:01.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7170000000000008e-06 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006141+0000", 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.4193999999999997e-05 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.006153+0000", 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2286000000000001e-05 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980862+0000", 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.97470905399999996 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980873+0000", 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0046e-05 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.018 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.160651+0000", 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1797785869999999 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.161279+0000", 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062757000000000004 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.174966+0000", 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013687725 2023-10-09T06:45:01.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.175040+0000", 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3509000000000004e-05 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.175423+0000", 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00038323600000000002 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.181558+0000", 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.006134469 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220827+0000", 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.039269351000000001 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.242431+0000", 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.021603581 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.242500+0000", 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9084000000000005e-05 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243046+0000", 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00054636799999999998 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.022 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169335 14.12 14:4d765013:::benchmark_data_smithi107_590976_object68562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.082151+0000", 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.864835489, 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1618175829999999, 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169335 2023-10-09T06:45:01.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082151+0000", 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082151+0000", 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082149+0000", 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082157+0000", 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.0800000000000006e-06 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082171+0000", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3986e-05 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.082180+0000", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8109999999999992e-06 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.085078+0000", 2023-10-09T06:45:01.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0028976459999999998 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.085091+0000", 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2731000000000001e-05 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980914+0000", 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.89582344599999997 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980926+0000", 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2266000000000001e-05 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.161331+0000", 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1804044600000001 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.161902+0000", 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00057127199999999995 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.179186+0000", 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017283898999999998 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.179314+0000", 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000128236 2023-10-09T06:45:01.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.179744+0000", 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000429739 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.189913+0000", 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010168689999999999 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220884+0000", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.030971406 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243511+0000", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.022626844 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243544+0000", 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3343000000000002e-05 2023-10-09T06:45:01.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243969+0000", 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00042475599999999998 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169353 14.12 14:4d0d2a0a:::benchmark_data_smithi107_590976_object68580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.032 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.107397+0000", 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.83958992300001, 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.137653233, 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169353 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107397+0000", 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107397+0000", 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.034 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107396+0000", 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107400+0000", 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5719999999999996e-06 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107410+0000", 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3139999999999993e-06 2023-10-09T06:45:01.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.107416+0000", 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4099999999999996e-06 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.111106+0000", 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0036902829999999999 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.036 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.111119+0000", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2723e-05 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980963+0000", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.869843955 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.980974+0000", 2023-10-09T06:45:01.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0485e-05 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.161959+0000", 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1809853160000001 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.162611+0000", 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000652467 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.182680+0000", 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.020068784999999999 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.182788+0000", 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000108081 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.183292+0000", 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00050358399999999995 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.189924+0000", 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0066320520000000003 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220909+0000", 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.030985505999999999 2023-10-09T06:45:01.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.244488+0000", 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.023578396000000001 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.244545+0000", 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7290999999999997e-05 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245050+0000", 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00050525400000000001 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169386 14.12 14:4e5a2101:::benchmark_data_smithi107_590976_object68613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.133378+0000", 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.813608641, 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1123492979999998, 2023-10-09T06:45:01.042 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169386 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133378+0000", 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133378+0000", 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133377+0000", 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133384+0000", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.379e-06 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133401+0000", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6529999999999999e-05 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133409+0000", 2023-10-09T06:45:01.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1960000000000008e-06 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133479+0000", 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0704000000000006e-05 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.133492+0000", 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2432999999999999e-05 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981001+0000", 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.84750909900000004 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981011+0000", 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0189e-05 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.162753+0000", 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1817422799999999 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.163399+0000", 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00064534300000000005 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.187442+0000", 2023-10-09T06:45:01.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024042801999999999 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.187524+0000", 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.2613000000000003e-05 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.187961+0000", 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00043632499999999999 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.194815+0000", 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0068546700000000002 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220981+0000", 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.02616626 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245150+0000", 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.024168761 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245207+0000", 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.7037999999999998e-05 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245728+0000", 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052034399999999995 2023-10-09T06:45:01.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169393 14.12 14:4da9bdba:::benchmark_data_smithi107_590976_object68620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.141913+0000", 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.80507360300001, 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1042851819999999, 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.052 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169393 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141913+0000", 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141913+0000", 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141912+0000", 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141917+0000", 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9109999999999998e-06 2023-10-09T06:45:01.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141930+0000", 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2945999999999999e-05 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141937+0000", 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.6780000000000003e-06 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141977+0000", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9764000000000001e-05 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.141985+0000", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9680000000000006e-06 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981041+0000", 2023-10-09T06:45:01.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.83905662400000003 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981051+0000", 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0036e-05 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.163460+0000", 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1824081930000001 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.164155+0000", 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00069567399999999997 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.191625+0000", 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027469189000000001 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.191726+0000", 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000100988 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.192145+0000", 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00041983299999999999 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.198035+0000", 2023-10-09T06:45:01.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0058900360000000004 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.221011+0000", 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.022975526999999999 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245772+0000", 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.02476076 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245824+0000", 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2206999999999997e-05 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246199+0000", 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00037462599999999997 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169408 14.12 14:4cbcc400:::benchmark_data_smithi107_590976_object68635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.158635+0000", 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.78835226000001, 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.088013627, 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169408 2023-10-09T06:45:01.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158635+0000", 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158635+0000", 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158634+0000", 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158639+0000", 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0669999999999999e-06 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158649+0000", 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0257e-05 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158656+0000", 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1539999999999996e-06 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158721+0000", 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4927999999999999e-05 2023-10-09T06:45:01.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.158735+0000", 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3738e-05 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981175+0000", 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.82243982100000002 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981185+0000", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.03e-05 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.164206+0000", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.183021256 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.164889+0000", 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00068296000000000003 2023-10-09T06:45:01.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.195701+0000", 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.030812091 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.195772+0000", 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0506999999999994e-05 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.196167+0000", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00039527099999999999 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.203111+0000", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0069433919999999996 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.221061+0000", 2023-10-09T06:45:01.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.017950745000000001 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246226+0000", 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025164540999999999 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246284+0000", 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8669999999999999e-05 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246648+0000", 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00036381800000000002 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169427 14.12 14:4cf0b6c8:::benchmark_data_smithi107_590976_object68654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.173139+0000", 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.77384819599999, 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0739891990000001, 2023-10-09T06:45:01.071 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169427 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173139+0000", 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173139+0000", 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173137+0000", 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173144+0000", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2960000000000004e-06 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173159+0000", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5012e-05 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173165+0000", 2023-10-09T06:45:01.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2750000000000001e-06 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173211+0000", 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6391999999999999e-05 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.173219+0000", 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7829999999999999e-06 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981214+0000", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.80799496199999998 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981221+0000", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2710000000000003e-06 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.164932+0000", 2023-10-09T06:45:01.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1837105860000001 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.165542+0000", 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000610387 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.199969+0000", 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.034426443000000001 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.200052+0000", 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3625e-05 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.200548+0000", 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00049567199999999995 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.207419+0000", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0068711190000000002 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.221129+0000", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.013710238 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246690+0000", 2023-10-09T06:45:01.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025561016999999998 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.246739+0000", 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8188e-05 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247128+0000", 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00038928299999999998 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.080 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169481 14.12 14:4e1c9bfc:::benchmark_data_smithi107_590976_object68708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.235905+0000", 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.71108183499999, 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.011703528, 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.081 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169481 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235905+0000", 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235905+0000", 2023-10-09T06:45:01.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235904+0000", 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235910+0000", 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.964e-06 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235919+0000", 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.5789999999999993e-06 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235926+0000", 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.117e-06 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235989+0000", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3019000000000001e-05 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.235999+0000", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0224000000000001e-05 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981255+0000", 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.745256523 2023-10-09T06:45:01.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981264+0000", 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4300000000000006e-06 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.165593+0000", 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1843294099999999 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.166229+0000", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00063549099999999996 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.204392+0000", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.038163073999999998 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.204507+0000", 2023-10-09T06:45:01.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000115727 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.205076+0000", 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00056886000000000003 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220917+0000", 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.015840797 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.221175+0000", 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00025738600000000002 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247155+0000", 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.025980250999999999 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247188+0000", 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.3423000000000002e-05 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247609+0000", 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00042044000000000001 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.090 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169491 14.12 14:4c664fbf:::benchmark_data_smithi107_590976_object68718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.321890+0000", 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.62509732000001, 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9261466120000001, 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169491 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321890+0000", 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321890+0000", 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321888+0000", 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321895+0000", 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0140000000000001e-06 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321909+0000", 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4175999999999999e-05 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321917+0000", 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9710000000000004e-06 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321973+0000", 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5342e-05 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.321985+0000", 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2238999999999999e-05 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981285+0000", 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.65930058700000005 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981293+0000", 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2649999999999999e-06 2023-10-09T06:45:01.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.166283+0000", 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1849901979999999 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.166911+0000", 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062775600000000004 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.209930+0000", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.043018925999999999 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.210038+0000", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00010809700000000001 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.210619+0000", 2023-10-09T06:45:01.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058177200000000004 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220923+0000", 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.010303633 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.221236+0000", 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00031319900000000001 2023-10-09T06:45:01.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247639+0000", 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.026402852000000001 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.247672+0000", 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2536999999999997e-05 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.099 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.248036+0000", 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000364533 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169528 14.12 14:49506dfb:::benchmark_data_smithi107_590976_object68755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.464278+0000", 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.48270926800001, 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8074158630000001, 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.100 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169528 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464278+0000", 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464278+0000", 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464276+0000", 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464283+0000", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8689999999999998e-06 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464300+0000", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6894000000000001e-05 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464308+0000", 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9510000000000006e-06 2023-10-09T06:45:01.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464372+0000", 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4286000000000001e-05 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.464394+0000", 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2651999999999999e-05 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981312+0000", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.51691800700000001 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981320+0000", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3289999999999996e-06 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.166959+0000", 2023-10-09T06:45:01.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.185638891 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.167609+0000", 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00065027300000000004 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.216258+0000", 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.048648944999999999 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.216400+0000", 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014170200000000001 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.217028+0000", 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00062872799999999995 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.220928+0000", 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0039001000000000001 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.223055+0000", 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00212685 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.270679+0000", 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.047623642000000001 2023-10-09T06:45:01.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.270738+0000", 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9036000000000002e-05 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.271694+0000", 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00095553700000000001 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.109 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169570 14.12 14:4dc0a8a0:::benchmark_data_smithi107_590976_object68797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.516111+0000", 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.430875668, 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7563166369999998, 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.110 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169570 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516111+0000", 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516111+0000", 2023-10-09T06:45:01.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516110+0000", 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516116+0000", 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.4110000000000001e-06 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516132+0000", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5586999999999998e-05 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516141+0000", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6850000000000007e-06 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516189+0000", 2023-10-09T06:45:01.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8510999999999997e-05 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.516200+0000", 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1226000000000001e-05 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981341+0000", 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.46514078399999997 2023-10-09T06:45:01.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981349+0000", 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1489999999999994e-06 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.167645+0000", 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1862953679999999 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.168131+0000", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00048679000000000002 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.233354+0000", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.065223120999999995 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.233483+0000", 2023-10-09T06:45:01.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000128799 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.234066+0000", 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00058241200000000003 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243070+0000", 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0090046680000000004 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.245108+0000", 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020372229999999999 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.271775+0000", 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.026667093999999999 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.271824+0000", 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9502999999999997e-05 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.272428+0000", 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00060374400000000003 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:169626 14.12 14:4fdefa03:::benchmark_data_smithi107_590976_object68853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e528)", 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:40.596759+0000", 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 140.350228018, 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6762469179999999, 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 169626 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.120 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596759+0000", 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596759+0000", 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.121 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596757+0000", 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596763+0000", 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8900000000000004e-06 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:01.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596778+0000", 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4202999999999999e-05 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596785+0000", 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.7710000000000006e-06 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596851+0000", 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5427999999999998e-05 2023-10-09T06:45:01.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.596866+0000", 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5608999999999999e-05 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981370+0000", 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.38450362399999999 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for peered", 2023-10-09T06:45:01.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:40.981377+0000", 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9659999999999999e-06 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.168977+0000", 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1876004090000001 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.169736+0000", 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00075836999999999996 2023-10-09T06:45:01.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.237574+0000", 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.067837884000000001 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.237714+0000", 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000140858 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 2,4", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.238228+0000", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00051336000000000003 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.243084+0000", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0048559069999999996 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.271738+0000", 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.028654051999999999 2023-10-09T06:45:01.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.272505+0000", 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00076701199999999997 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.272553+0000", 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8381999999999999e-05 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:01.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:42.273006+0000", 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000452683 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:01.129 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: pgmap v1762: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 81 KiB/s, 9 objects/s recovering 2023-10-09T06:45:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: pgmap v1762: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 81 KiB/s, 9 objects/s recovering 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.403 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:01 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: pgmap v1762: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 81 KiB/s, 9 objects/s recovering 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: osdmap e586: 8 total, 8 up, 8 in 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: failed to encode map e586 with expected crc 2023-10-09T06:45:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.406 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:01 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:01.987 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T06:45:02.024 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:02.246 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:02] "GET /metrics HTTP/1.1" 200 34129 "" "Prometheus/2.43.0" 2023-10-09T06:45:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:02.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[220921]: pgmap v1764: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 930 KiB/s, 6 objects/s recovering 2023-10-09T06:45:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:02.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:02.902 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:02 smithi107 ceph-mon[212750]: pgmap v1764: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 930 KiB/s, 6 objects/s recovering 2023-10-09T06:45:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:02.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:02 smithi150 ceph-mon[194178]: pgmap v1764: 103 pgs: 3 peering, 5 remapped+peering, 95 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 930 KiB/s, 6 objects/s recovering 2023-10-09T06:45:03.069 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:03.887 INFO:teuthology.orchestra.run.smithi107.stdout:mon.a: {} 2023-10-09T06:45:03.897 INFO:teuthology.orchestra.run.smithi107.stdout:mon.b: {} 2023-10-09T06:45:03.906 INFO:teuthology.orchestra.run.smithi107.stdout:mon.c: {} 2023-10-09T06:45:03.907 INFO:teuthology.orchestra.run.smithi107.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:45:03.907 INFO:teuthology.orchestra.run.smithi107.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:45:03.907 INFO:teuthology.orchestra.run.smithi107.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:45:04.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:45:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:03 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:45:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:03 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:45:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:04.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:04 smithi107 ceph-mon[212750]: pgmap v1765: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 715 KiB/s, 4 objects/s recovering 2023-10-09T06:45:05.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:04 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2277773830' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:45:05.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:04 smithi107 ceph-mon[220921]: pgmap v1765: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 715 KiB/s, 4 objects/s recovering 2023-10-09T06:45:05.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:04 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2277773830' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:45:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:04 smithi150 ceph-mon[194178]: pgmap v1765: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 715 KiB/s, 4 objects/s recovering 2023-10-09T06:45:05.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:04 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2277773830' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T06:45:05.561 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T06:45:05.802 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:05.803 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:45:05.803 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:45:05.803 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:06.385 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T06:45:06.477 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: pgmap v1766: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 563 KiB/s, 3 objects/s recovering 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:45:06.979 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: pgmap v1766: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 563 KiB/s, 3 objects/s recovering 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:45:06.980 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:06 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: pgmap v1766: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 563 KiB/s, 3 objects/s recovering 2023-10-09T06:45:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:07.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:45:07.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:45:07.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:06 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:07.441 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:08.811 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T06:45:08.987 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[212750]: pgmap v1767: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 498 KiB/s, 1 objects/s recovering 2023-10-09T06:45:08.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3191213449' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:08.988 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:08.988 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[220921]: pgmap v1767: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 498 KiB/s, 1 objects/s recovering 2023-10-09T06:45:08.988 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3191213449' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:08.988 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:08 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:08 smithi150 ceph-mon[194178]: pgmap v1767: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 498 KiB/s, 1 objects/s recovering 2023-10-09T06:45:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:08 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3191213449' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:09.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:08 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:09.612 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T06:45:09.629 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:09.629 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:45:09.630 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:45:09.630 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:45:09.630 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:45:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:45:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[212750]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T06:45:09.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[212750]: failed to encode map e587 with expected crc 2023-10-09T06:45:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[220921]: failed to encode map e587 with expected crc 2023-10-09T06:45:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[220921]: failed to encode map e587 with expected crc 2023-10-09T06:45:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[220921]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T06:45:09.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:09 smithi107 ceph-mon[220921]: failed to encode map e587 with expected crc 2023-10-09T06:45:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:09 smithi150 ceph-mon[194178]: failed to encode map e587 with expected crc 2023-10-09T06:45:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:09 smithi150 ceph-mon[194178]: failed to encode map e587 with expected crc 2023-10-09T06:45:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:09 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:10.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:09 smithi150 ceph-mon[194178]: osdmap e587: 8 total, 8 up, 8 in 2023-10-09T06:45:10.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:09 smithi150 ceph-mon[194178]: failed to encode map e587 with expected crc 2023-10-09T06:45:10.326 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T06:45:10.400 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[212750]: pgmap v1769: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 413 KiB/s, 1 objects/s recovering 2023-10-09T06:45:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[212750]: 1.f scrub starts 2023-10-09T06:45:10.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[212750]: 1.f scrub ok 2023-10-09T06:45:10.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[220921]: pgmap v1769: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 413 KiB/s, 1 objects/s recovering 2023-10-09T06:45:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[220921]: 1.f scrub starts 2023-10-09T06:45:10.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:10 smithi107 ceph-mon[220921]: 1.f scrub ok 2023-10-09T06:45:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:10 smithi150 ceph-mon[194178]: pgmap v1769: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 413 KiB/s, 1 objects/s recovering 2023-10-09T06:45:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:10 smithi150 ceph-mon[194178]: 1.f scrub starts 2023-10-09T06:45:11.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:10 smithi150 ceph-mon[194178]: 1.f scrub ok 2023-10-09T06:45:11.481 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:12.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:12] "GET /metrics HTTP/1.1" 200 34129 "" "Prometheus/2.43.0" 2023-10-09T06:45:12.830 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T06:45:12.988 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:12.988 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:45:12.988 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:45:12.988 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:173040 13.11 13:88f72a75:::benchmark_data_smithi107_535588_object66933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:38:24.154061+0000", 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 408.832233772, 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.246020357, 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 173040 2023-10-09T06:45:12.989 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154061+0000", 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154061+0000", 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:12.990 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154059+0000", 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154067+0000", 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4390000000000003e-06 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.991 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154085+0000", 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8519999999999999e-05 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.154094+0000", 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3529999999999995e-06 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:12.992 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.155089+0000", 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00099480399999999996 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.155190+0000", 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101378 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.155661+0000", 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00047087600000000001 2023-10-09T06:45:12.993 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.165566+0000", 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0099056279999999997 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.165620+0000", 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3517000000000003e-05 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.994 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.165632+0000", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1831000000000001e-05 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.398728+0000", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.233096466 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.398800+0000", 2023-10-09T06:45:12.995 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.1917999999999998e-05 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:38:24.400081+0000", 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001280979 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:205747 13.11 13:8972c788:::benchmark_data_smithi107_535588_object99640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:45:12.996 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.494407+0000", 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 369.491887389, 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25333318500000002, 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "waiting for sub ops", 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 205747 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:12.997 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494407+0000", 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494407+0000", 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494406+0000", 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:12.998 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494411+0000", 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.3120000000000001e-06 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494421+0000", 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6549999999999999e-06 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:12.999 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494427+0000", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.7460000000000003e-06 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.497567+0000", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0031391909999999999 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.497720+0000", 2023-10-09T06:45:13.000 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000153098 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.498237+0000", 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000517221 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.502807+0000", 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045700339999999997 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.001 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.504745+0000", 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0019382780000000001 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.747740+0000", 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24299500199999999 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.002 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:205759 13.f e481/437 13:f2883f41:::benchmark_data_smithi107_535588_object99652:head v 481'12936, mlcod=481'12934)", 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.506317+0000", 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 369.47997739800002, 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24127711600000001, 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.003 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506317+0000", 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506317+0000", 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506316+0000", 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.004 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506326+0000", 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0361e-05 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506336+0000", 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.8700000000000004e-06 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.005 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506344+0000", 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.8180000000000004e-06 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.743913+0000", 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.23756882700000001 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.006 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.743952+0000", 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.9106000000000001e-05 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.747503+0000", 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003550952 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.747594+0000", 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1365000000000004e-05 2023-10-09T06:45:13.007 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:207728 13.f e483/437 13:f12b5a29:::benchmark_data_smithi107_535588_object101621:head v 483'13062, mlcod=483'13060)", 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:06.713231+0000", 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 366.27306342200001, 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26388755800000002, 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.008 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713231+0000", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713231+0000", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713230+0000", 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.009 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713236+0000", 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6450000000000002e-06 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713251+0000", 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4684e-05 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.010 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713256+0000", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2959999999999998e-06 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713293+0000", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7004e-05 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.713308+0000", 2023-10-09T06:45:13.011 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.5123000000000001e-05 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716728+0000", 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034196119999999998 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.977119+0000", 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26039087999999999 2023-10-09T06:45:13.012 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:207730 13.18 e483/459 13:1bb025f6:::benchmark_data_smithi107_535588_object101623:head v 483'9591, mlcod=483'9589)", 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:06.714112+0000", 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 366.272182961, 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25915702400000001, 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.013 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714112+0000", 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714112+0000", 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.014 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714111+0000", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714117+0000", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.3810000000000004e-06 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714126+0000", 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5730000000000008e-06 2023-10-09T06:45:13.015 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714131+0000", 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2079999999999999e-06 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714173+0000", 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2194999999999998e-05 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.016 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.714193+0000", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9632000000000001e-05 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716596+0000", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0024032250000000002 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973269+0000", 2023-10-09T06:45:13.017 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.256672444 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:207734 13.b e483/459 13:d8855769:::benchmark_data_smithi107_535588_object101627:head v 483'12894, mlcod=483'12892)", 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:06.715957+0000", 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 366.27033785999998, 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25805947400000001, 2023-10-09T06:45:13.018 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715957+0000", 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715957+0000", 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.019 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715956+0000", 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715963+0000", 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5449999999999998e-06 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.020 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715975+0000", 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1134999999999999e-05 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.715982+0000", 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2790000000000001e-06 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.021 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716034+0000", 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2472999999999997e-05 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716061+0000", 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6636999999999999e-05 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973975+0000", 2023-10-09T06:45:13.022 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.257914476 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.974016+0000", 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0701999999999998e-05 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.023 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:207729 13.f e483/437 13:f4cda011:::benchmark_data_smithi107_535588_object101622:head v 483'13063, mlcod=483'13060)", 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:06.716766+0000", 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 366.26952842100002, 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26842553400000002, 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716766+0000", 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.024 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716766+0000", 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716765+0000", 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.025 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.716771+0000", 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9710000000000001e-06 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973298+0000", 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25652618599999999 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.026 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973306+0000", 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4160000000000004e-06 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.977150+0000", 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0038443330000000001 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.977168+0000", 2023-10-09T06:45:13.027 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7663e-05 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.985159+0000", 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0079908730000000008 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.985192+0000", 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2793999999999998e-05 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.028 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:207733 13.0 e483/454 13:0665e03e:::benchmark_data_smithi107_535588_object101626:head v 483'9575, mlcod=483'9573)", 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:06.719406+0000", 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 366.26688895299998, 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26016535600000001, 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.029 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.719406+0000", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.719406+0000", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.719405+0000", 2023-10-09T06:45:13.030 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.719412+0000", 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2849999999999997e-06 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973294+0000", 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.25388189300000003 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.031 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973304+0000", 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0318e-05 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973561+0000", 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00025680400000000002 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.032 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.973583+0000", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1631e-05 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.979532+0000", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0059494190000000001 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:06.979571+0000", 2023-10-09T06:45:13.033 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8708999999999998e-05 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:6721 14.c e493/489 14:377cfcb6:::benchmark_data_smithi107_590976_object6720:head v 493'438, mlcod=493'436)", 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:47.850110+0000", 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 325.13618427300003, 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26804920500000001, 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.034 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850110+0000", 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850110+0000", 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.035 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850110+0000", 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850130+0000", 2023-10-09T06:45:13.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.0370999999999999e-05 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850137+0000", 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9120000000000001e-06 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850143+0000", 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.6300000000000003e-06 2023-10-09T06:45:13.038 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850259+0000", 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000116674 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850278+0000", 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8685000000000002e-05 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851460+0000", 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001181938 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118159+0000", 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26669937799999999 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.040 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:6723 14.b e493/489 14:db5f8272:::benchmark_data_smithi107_590976_object6722:head v 493'441, mlcod=493'438)", 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:47.850782+0000", 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 325.13551250500001, 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26745190899999999, 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850782+0000", 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850782+0000", 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850781+0000", 2023-10-09T06:45:13.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850804+0000", 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2886000000000001e-05 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850813+0000", 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1659999999999994e-06 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850819+0000", 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0159999999999997e-06 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850860+0000", 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1279999999999998e-05 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850879+0000", 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9386999999999999e-05 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851690+0000", 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00081063299999999995 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118234+0000", 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26654408800000001 2023-10-09T06:45:13.045 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:6722 14.c e493/489 14:390cb483:::benchmark_data_smithi107_590976_object6721:head v 493'439, mlcod=493'436)", 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:47.850941+0000", 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 325.13535326200002, 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26732770500000003, 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.046 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850941+0000", 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850941+0000", 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850941+0000", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850963+0000", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2116999999999998e-05 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850972+0000", 2023-10-09T06:45:13.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8529999999999998e-06 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.850978+0000", 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8210000000000004e-06 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851160+0000", 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00018151899999999999 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851178+0000", 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8366999999999999e-05 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118176+0000", 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26699757600000001 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118269+0000", 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.3054999999999996e-05 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:6727 14.6 e493/489 14:6a763dca:::benchmark_data_smithi107_590976_object6726:head v 493'420, mlcod=493'416)", 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:47.851115+0000", 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 325.13517982899998, 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26722818799999998, 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.051 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851115+0000", 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851115+0000", 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.052 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851114+0000", 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851140+0000", 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6234e-05 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851150+0000", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.4779999999999997e-06 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851156+0000", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2709999999999998e-06 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851191+0000", 2023-10-09T06:45:13.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5324000000000003e-05 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.851209+0000", 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7056000000000001e-05 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118284+0000", 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26707578999999998 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118343+0000", 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8585999999999998e-05 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:6731 14.b e493/489 14:dd3af013:::benchmark_data_smithi107_590976_object6730:head v 493'442, mlcod=493'438)", 2023-10-09T06:45:13.056 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:47.852879+0000", 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 325.13341553100003, 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26695954500000002, 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.852879+0000", 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.057 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.852879+0000", 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.852878+0000", 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:47.852901+0000", 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2829000000000001e-05 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118171+0000", 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.26526977400000001 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118196+0000", 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.5443e-05 2023-10-09T06:45:13.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118333+0000", 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000136804 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.118347+0000", 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4239e-05 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.119747+0000", 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001399391 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:48.119839+0000", 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1848999999999994e-05 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:47790 14.8 e495/489 14:1e0c44e5:::benchmark_data_smithi107_590976_object47789:head v 495'3069, mlcod=495'3067)", 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.358621+0000", 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 297.62767377400002, 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28598778000000002, 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358621+0000", 2023-10-09T06:45:13.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358621+0000", 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358620+0000", 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358642+0000", 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.2498999999999999e-05 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358653+0000", 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0900000000000001e-05 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358661+0000", 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2409999999999998e-06 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358722+0000", 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.1400000000000002e-05 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358755+0000", 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2849999999999999e-05 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644560+0000", 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28580514800000001 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644609+0000", 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8542999999999999e-05 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.066 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:47795 14.e e495/489 14:73df0311:::benchmark_data_smithi107_590976_object47794:head v 495'2948, mlcod=495'2946)", 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.359316+0000", 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 297.62697853499998, 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.287282861, 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359316+0000", 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359316+0000", 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359315+0000", 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359370+0000", 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4373000000000003e-05 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359380+0000", 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0445000000000001e-05 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359387+0000", 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.725e-06 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359440+0000", 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2930000000000003e-05 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.359458+0000", 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8107000000000001e-05 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.646557+0000", 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28709906000000002 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.646599+0000", 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1943000000000001e-05 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:47793 14.6 e495/489 14:62ae615f:::benchmark_data_smithi107_590976_object47792:head v 495'2971, mlcod=495'2969)", 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.360440+0000", 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 297.625854678, 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.292018312, 2023-10-09T06:45:13.072 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360440+0000", 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360440+0000", 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360439+0000", 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360465+0000", 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.6046000000000001e-05 2023-10-09T06:45:13.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360474+0000", 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8049999999999996e-06 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.360481+0000", 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.5540000000000003e-06 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.646247+0000", 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28576616599999999 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.646264+0000", 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7731999999999999e-05 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652428+0000", 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0061634639999999996 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652458+0000", 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0210999999999999e-05 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.077 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:47801 14.e e495/489 14:7669b078:::benchmark_data_smithi107_590976_object47800:head v 495'2949, mlcod=495'2946)", 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.362979+0000", 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 297.62331551599999, 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28944378500000001, 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362979+0000", 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362979+0000", 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362978+0000", 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:13.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.363019+0000", 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0818000000000001e-05 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644537+0000", 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28151833100000001 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644554+0000", 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7071e-05 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644590+0000", 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5688000000000001e-05 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.644615+0000", 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4859999999999999e-05 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652377+0000", 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0077625710000000002 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652423+0000", 2023-10-09T06:45:13.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5683999999999998e-05 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47051.0:89696 14.c e508/489 14:3f8bfe61:::benchmark_data_smithi107_590976_object89695:head v 508'5671, mlcod=508'5669)", 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:41:02.074074+0000", 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 250.912220682, 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.24061595899999999, 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.083 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074074+0000", 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074074+0000", 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074073+0000", 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074105+0000", 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.2427e-05 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074122+0000", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6124999999999999e-05 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074131+0000", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.7589999999999992e-06 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074183+0000", 2023-10-09T06:45:13.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.134e-05 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.074205+0000", 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.23e-05 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.314642+0000", 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.240436552 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:41:02.314690+0000", 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8282999999999998e-05 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(osd.3.0:166082 15.e e550/543 15:725d3449:::image47:2 v 550'38, mlcod=549'35)", 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:59.835849+0000", 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 73.150445085000001, 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001790844, 2023-10-09T06:45:13.088 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835849+0000", 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835849+0000", 2023-10-09T06:45:13.089 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835849+0000", 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835859+0000", 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.9890000000000004e-06 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835866+0000", 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0600000000000002e-06 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835874+0000", 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.091 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835911+0000", 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7240999999999998e-05 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.835926+0000", 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.519e-05 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.837602+0000", 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001676227 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:59.837640+0000", 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7902999999999999e-05 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.37416.0:81 2.3 e554/459 2:cb541ef2:::rbd_data.b915a4787591.0000000000000002:head v 554'6, mlcod=554'4)", 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:44:11.813584+0000", 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 61.172710387000002, 2023-10-09T06:45:13.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0015265319999999999, 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813584+0000", 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:13.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813584+0000", 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813583+0000", 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813591+0000", 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9650000000000008e-06 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813606+0000", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4399999999999999e-05 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813614+0000", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.3950000000000001e-06 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813684+0000", 2023-10-09T06:45:13.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9999999999999994e-05 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.813712+0000", 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8068999999999998e-05 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.815064+0000", 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001351884 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:13.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:44:11.815111+0000", 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6718000000000002e-05 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:13.098 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[212750]: pgmap v1770: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3279501397' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:13.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[220921]: pgmap v1770: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3279501397' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:13.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:12 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:12 smithi150 ceph-mon[194178]: pgmap v1770: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:12 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3279501397' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:12 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T06:45:14.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:14.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:14.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[220921]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T06:45:14.102 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[220921]: failed to encode map e588 with expected crc 2023-10-09T06:45:14.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:14.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:14.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[212750]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T06:45:14.103 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:13 smithi107 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:45:14.123 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T06:45:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:13 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:45:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:13 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T06:45:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:13 smithi150 ceph-mon[194178]: osdmap e588: 8 total, 8 up, 8 in 2023-10-09T06:45:14.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:13 smithi150 ceph-mon[194178]: failed to encode map e588 with expected crc 2023-10-09T06:45:14.189 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T06:45:14.536 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:14.651 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:15.002 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[212750]: pgmap v1772: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:15.002 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[212750]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[220921]: pgmap v1772: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[220921]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[220921]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.003 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:14 smithi107 ceph-mon[220921]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:14 smithi150 ceph-mon[194178]: pgmap v1772: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:14 smithi150 ceph-mon[194178]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:14 smithi150 ceph-mon[194178]: failed to encode map e588 with expected crc 2023-10-09T06:45:15.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:14 smithi150 ceph-mon[194178]: failed to encode map e588 with expected crc 2023-10-09T06:45:16.560 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:16.560 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:45:16.560 INFO:teuthology.orchestra.run.smithi107.stdout: "num_ops": 0 2023-10-09T06:45:16.560 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:16.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:16 smithi107 ceph-mon[212750]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:16.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:16 smithi107 ceph-mon[220921]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:16.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:16 smithi150 ceph-mon[194178]: pgmap v1773: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:17.332 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T06:45:17.673 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:17.845 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T06:45:18.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:17 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:18.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:17 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:18.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:17 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:18.765 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[212750]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.153 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:18 smithi107 ceph-mon[220921]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: pgmap v1774: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3451179491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: osdmap e589: 8 total, 8 up, 8 in 2023-10-09T06:45:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:18 smithi150 ceph-mon[194178]: failed to encode map e589 with expected crc 2023-10-09T06:45:19.260 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:19.681 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:19.682 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [], 2023-10-09T06:45:19.682 INFO:teuthology.orchestra.run.smithi107.stdout: "complaint_time": 30, 2023-10-09T06:45:19.682 INFO:teuthology.orchestra.run.smithi107.stdout: "num_blocked_ops": 0 2023-10-09T06:45:19.682 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:20.434 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T06:45:21.003 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:21.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:20 smithi107 ceph-mon[212750]: pgmap v1776: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:21.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:20 smithi107 ceph-mon[220921]: pgmap v1776: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:20 smithi150 ceph-mon[194178]: pgmap v1776: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:21.854 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T06:45:22.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:22] "GET /metrics HTTP/1.1" 200 34103 "" "Prometheus/2.43.0" 2023-10-09T06:45:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:21 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2258650374' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:21 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:21 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2258650374' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:21 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:21 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2258650374' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:21 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T06:45:22.757 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T06:45:23.037 INFO:teuthology.orchestra.run.smithi107.stdout:{ 2023-10-09T06:45:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "size": 20, 2023-10-09T06:45:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 600, 2023-10-09T06:45:23.037 INFO:teuthology.orchestra.run.smithi107.stdout: "ops": [ 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:205750 13.12 13:4c6c037d:::benchmark_data_smithi107_535588_object99643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.489982+0000", 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.54504263699999, 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.342861421, 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:23.038 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 205750 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.489982+0000", 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.489982+0000", 2023-10-09T06:45:23.039 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.489980+0000", 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.489989+0000", 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.5390000000000007e-06 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.040 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.490003+0000", 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.4772e-05 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.490012+0000", 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.7099999999999996e-06 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.041 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.490064+0000", 2023-10-09T06:45:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1554000000000001e-05 2023-10-09T06:45:23.042 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.490169+0000", 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000105704 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 0,1,4", 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.490736+0000", 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00056698600000000001 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.043 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.494214+0000", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0034773820000000002 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.497299+0000", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030858610000000001 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.498426+0000", 2023-10-09T06:45:23.044 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001126092 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.504849+0000", 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064230440000000001 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.831429+0000", 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32657989399999998 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.045 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.832843+0000", 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0014146300000000001 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.46706.0:205750 13.12 e481/465 ondisk, result = 0)", 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.503792+0000", 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.53123196299998, 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32906552999999999, 2023-10-09T06:45:23.046 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503792+0000", 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503792+0000", 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.047 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503792+0000", 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.048 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503797+0000", 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9620000000000001e-06 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503807+0000", 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0061000000000001e-05 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.049 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503812+0000", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.809e-06 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.504829+0000", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001017067 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.504840+0000", 2023-10-09T06:45:23.050 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1446e-05 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.832858+0000", 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32801785500000002 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.051 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "MOSDPGPushReply(13.12 481/465 [PushReplyOp(13:4953a16d:::benchmark_data_smithi107_535588_object99778:head)])", 2023-10-09T06:45:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.503820+0000", 2023-10-09T06:45:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.53120481299999, 2023-10-09T06:45:23.052 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33175398699999997, 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "reached pg", 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503820+0000", 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.053 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503820+0000", 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503819+0000", 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.054 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503824+0000", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.7720000000000002e-06 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503831+0000", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.2060000000000001e-06 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503835+0000", 2023-10-09T06:45:23.055 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6049999999999998e-06 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.833657+0000", 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32982217800000002 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.835574+0000", 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001916507 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.056 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:205757 13.3 e481/454 13:c5de4471:::benchmark_data_smithi107_535588_object99650:head v 481'9614, mlcod=481'9612)", 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.503909+0000", 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.53111561200001, 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.33448110599999997, 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.057 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503909+0000", 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503909+0000", 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.058 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503908+0000", 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503915+0000", 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4229999999999999e-06 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.059 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503926+0000", 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0409e-05 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.503933+0000", 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.4150000000000002e-06 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830894+0000", 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32696118899999999 2023-10-09T06:45:23.060 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830918+0000", 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3419000000000001e-05 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.838359+0000", 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0074412790000000003 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.061 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.838390+0000", 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.0744999999999998e-05 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.46706.0:205764 13.2 13:45f5f714:::benchmark_data_smithi107_535588_object99657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.506408+0000", 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.52861675700001, 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.336680594, 2023-10-09T06:45:23.062 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.46706", 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 205764 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506408+0000", 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.063 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506408+0000", 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506406+0000", 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.064 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.506415+0000", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.799e-06 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830790+0000", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.32437524400000001 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830820+0000", 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.9882e-05 2023-10-09T06:45:23.065 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.832886+0000", 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002065791 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.833005+0000", 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00011938300000000001 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.066 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.833485+0000", 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00048039600000000001 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.839915+0000", 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0064291870000000003 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.067 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.842977+0000", 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.003061968 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.843014+0000", 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7162000000000001e-05 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.843054+0000", 2023-10-09T06:45:23.068 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0107000000000001e-05 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.843088+0000", 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.4304999999999998e-05 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.46706.0:205773 13.6 e481/454 13:624e0fc4:::benchmark_data_smithi107_535588_object99666:head v 481'9341, mlcod=481'9339)", 2023-10-09T06:45:23.069 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:03.521931+0000", 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 379.51309328399998, 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.314630941, 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.521931+0000", 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.070 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.521931+0000", 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.521930+0000", 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.521939+0000", 2023-10-09T06:45:23.071 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.8510000000000005e-06 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830839+0000", 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.30890018600000002 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830853+0000", 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3529999999999999e-05 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.072 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830921+0000", 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.8678999999999998e-05 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.830944+0000", 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3136999999999999e-05 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.073 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.836544+0000", 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0056000360000000001 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:03.836562+0000", 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7649999999999999e-05 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.074 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:15447 14.1 14:8efb5a0b:::benchmark_data_smithi107_590976_object15446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:39:51.769119+0000", 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 331.265905498, 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28356402000000003, 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 15447 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.075 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:51.769119+0000", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:51.769119+0000", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:51.769118+0000", 2023-10-09T06:45:23.076 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:51.769138+0000", 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.9978000000000001e-05 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.047490+0000", 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27835232300000001 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.077 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.047501+0000", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0528e-05 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.048030+0000", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00052986999999999999 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.048192+0000", 2023-10-09T06:45:23.078 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016142399999999999 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.048340+0000", 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00014810499999999999 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.050718+0000", 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0023780279999999999 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.079 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.052569+0000", 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0018506429999999999 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.052606+0000", 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7212000000000002e-05 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.080 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.052648+0000", 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.1646000000000001e-05 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:39:52.052683+0000", 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5330000000000002e-05 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.081 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:47781 14.1 14:86a19a42:::benchmark_data_smithi107_590976_object47780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.351447+0000", 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 307.683576883, 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29402699199999999, 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.082 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 47781 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351447+0000", 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351447+0000", 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.083 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351446+0000", 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351475+0000", 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8319000000000001e-05 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.084 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351485+0000", 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.0672999999999999e-05 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.351492+0000", 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0589999999999997e-06 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.353589+0000", 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020969610000000001 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.085 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.353691+0000", 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000101846 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.353778+0000", 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6887999999999999e-05 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.086 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.357696+0000", 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0039181820000000001 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.358295+0000", 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00059829200000000005 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.645373+0000", 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28707870499999999 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.087 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.088 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.088 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.645432+0000", 2023-10-09T06:45:23.090 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8288999999999997e-05 2023-10-09T06:45:23.090 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.090 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.090 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.090 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.645474+0000", 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2806000000000003e-05 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:47799 14.1 14:8460f796:::benchmark_data_smithi107_590976_object47798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.362549+0000", 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 307.67247525099998, 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28704400499999999, 2023-10-09T06:45:23.091 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 47799 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362549+0000", 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.092 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362549+0000", 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362547+0000", 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.093 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362571+0000", 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3411e-05 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362582+0000", 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1333e-05 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.094 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362590+0000", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5820000000000004e-06 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362676+0000", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6673000000000002e-05 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362802+0000", 2023-10-09T06:45:23.095 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012580600000000001 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362935+0000", 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000132824 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.366271+0000", 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0033358089999999999 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.096 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.368335+0000", 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0020644489999999999 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.649508+0000", 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28117283500000001 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.097 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.649557+0000", 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.8989000000000001e-05 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.649593+0000", 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.6109999999999998e-05 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.098 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:47803 14.1 14:89f80413:::benchmark_data_smithi107_590976_object47802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:40:15.362603+0000", 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 307.672421687, 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.29021487099999999, 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 47803 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.099 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362603+0000", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362603+0000", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362602+0000", 2023-10-09T06:45:23.100 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362619+0000", 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.7025e-05 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362626+0000", 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.596e-06 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.101 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.362631+0000", 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5709999999999997e-06 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.364440+0000", 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001808788 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.364565+0000", 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012460800000000001 2023-10-09T06:45:23.102 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.364731+0000", 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00016612800000000001 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.367784+0000", 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0030533000000000001 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.103 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.368490+0000", 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00070560100000000004 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652721+0000", 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28423128800000003 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.104 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652780+0000", 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8925000000000002e-05 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:40:15.652818+0000", 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7503000000000003e-05 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.105 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:147217 14.1 14:8138e951:::benchmark_data_smithi107_590976_object46444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:11.325374+0000", 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 191.709650584, 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.28761360600000002, 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 147217 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.106 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325374+0000", 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325374+0000", 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.107 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325372+0000", 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325378+0000", 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2110000000000003e-06 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325391+0000", 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2422000000000001e-05 2023-10-09T06:45:23.108 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325399+0000", 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1079999999999993e-06 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325451+0000", 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.2444999999999997e-05 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.109 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.325538+0000", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.6959999999999994e-05 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.326019+0000", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000480423 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.328616+0000", 2023-10-09T06:45:23.110 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0025974280000000001 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.333191+0000", 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0045745480000000003 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.612895+0000", 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.27970414900000001 2023-10-09T06:45:23.111 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.612947+0000", 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.1743999999999998e-05 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:11.612987+0000", 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0708999999999999e-05 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.112 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:180112 14.1 14:8520e289:::benchmark_data_smithi107_590976_object79339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:58.755967+0000", 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 144.27905783400001, 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.17135952099999999, 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.113 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 180112 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.755967+0000", 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.114 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.755967+0000", 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.115 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.755965+0000", 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.755973+0000", 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9030000000000004e-06 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.116 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.755996+0000", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.3266000000000001e-05 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.756004+0000", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.9650000000000008e-06 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.770274+0000", 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.014270064000000001 2023-10-09T06:45:23.117 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.770402+0000", 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00012802500000000001 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.770936+0000", 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00053369400000000003 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.118 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.782458+0000", 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.011522225000000001 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.782486+0000", 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.7832000000000001e-05 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.119 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.927223+0000", 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.14473704800000001 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.927284+0000", 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.0955000000000002e-05 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.927326+0000", 2023-10-09T06:45:23.120 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.2358999999999999e-05 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:180125 14.1 14:81b7c308:::benchmark_data_smithi107_590976_object79352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:58.771276+0000", 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 144.26374878499999, 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.16484210699999999, 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.121 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 180125 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771276+0000", 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.122 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771276+0000", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771274+0000", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771280+0000", 2023-10-09T06:45:23.123 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.8000000000000004e-06 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771293+0000", 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.3532e-05 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771302+0000", 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.4870000000000002e-06 2023-10-09T06:45:23.124 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.776889+0000", 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0055872730000000002 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.776998+0000", 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000109164 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.125 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.777448+0000", 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000449457 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.782480+0000", 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0050324819999999996 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.126 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.782675+0000", 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00019437400000000001 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936025+0000", 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15335031900000001 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936079+0000", 2023-10-09T06:45:23.127 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.4221999999999997e-05 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936118+0000", 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8578000000000002e-05 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:180129 14.1 14:83fb45eb:::benchmark_data_smithi107_590976_object79356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:23.128 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:42:58.771996+0000", 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 144.263028752, 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.164251586, 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 180129 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.129 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771996+0000", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771996+0000", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.771995+0000", 2023-10-09T06:45:23.130 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.772000+0000", 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9269999999999996e-06 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.772021+0000", 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.1452000000000001e-05 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.131 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.772029+0000", 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.3289999999999996e-06 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.779926+0000", 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0078974679999999995 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.132 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.780039+0000", 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000113047 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.780483+0000", 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00044399799999999997 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.782680+0000", 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002196543 2023-10-09T06:45:23.133 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.784416+0000", 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.001736039 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936171+0000", 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.151755323 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.134 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936208+0000", 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.7360999999999997e-05 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:42:58.936247+0000", 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.8860999999999999e-05 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.135 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:184746 14.14 14:2b858da1:::benchmark_data_smithi107_590976_object83973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:05.325092+0000", 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 137.70993260500001, 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.11776937899999999, 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 184746 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.136 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325092+0000", 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325092+0000", 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.137 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325090+0000", 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325097+0000", 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.9269999999999999e-06 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.138 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325109+0000", 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2397e-05 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325117+0000", 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.5360000000000004e-06 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325162+0000", 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5262999999999997e-05 2023-10-09T06:45:23.139 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325335+0000", 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000172652 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325794+0000", 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00045901200000000002 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.140 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.328453+0000", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.002659116 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331221+0000", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0027683069999999998 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442267+0000", 2023-10-09T06:45:23.141 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.111046122 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442322+0000", 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5024e-05 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442861+0000", 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00053894900000000005 2023-10-09T06:45:23.142 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_op(client.47051.0:184752 14.9 14:955c40ee:::benchmark_data_smithi107_590976_object83979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:05.325433+0000", 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 137.70959147100001, 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.11693273699999999, 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.143 INFO:teuthology.orchestra.run.smithi107.stdout: "client_info": { 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "client": "client.47051", 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "tid": 184752 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325433+0000", 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.144 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325433+0000", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325432+0000", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325437+0000", 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.536e-06 2023-10-09T06:45:23.145 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325449+0000", 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.1816e-05 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325456+0000", 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 6.2330000000000003e-06 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.146 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325496+0000", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.0771e-05 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.325588+0000", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1559999999999998e-05 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.326035+0000", 2023-10-09T06:45:23.147 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.000446947 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "op_commit", 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.328678+0000", 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.0026436609999999998 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442188+0000", 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.113509601 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.148 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442264+0000", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.6087999999999994e-05 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442320+0000", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.5646000000000002e-05 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442366+0000", 2023-10-09T06:45:23.149 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.5813999999999999e-05 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.47051.0:184752 14.9 e536/500 ondisk, result = 0)", 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:05.331406+0000", 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 137.703618694, 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.11079265100000001, 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:45:23.150 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331406+0000", 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331406+0000", 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.151 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331405+0000", 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331408+0000", 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.249e-06 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331412+0000", 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6979999999999997e-06 2023-10-09T06:45:23.152 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331415+0000", 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.061e-06 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331444+0000", 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.8138000000000001e-05 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.153 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.331449+0000", 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.0030000000000002e-06 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442198+0000", 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.110749791 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.154 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop_reply(client.47051.0:184746 14.14 e536/533 ondisk, result = 0)", 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:05.332021+0000", 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 137.703003784, 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.110844258, 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "started", 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.332021+0000", 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.155 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.332021+0000", 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.332020+0000", 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.999999 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.156 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.332025+0000", 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.074e-06 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442180+0000", 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.110155058 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442198+0000", 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.8155000000000002e-05 2023-10-09T06:45:23.157 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442248+0000", 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.9487999999999998e-05 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442260+0000", 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.2616000000000001e-05 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.158 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:05.442865+0000", 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.00060465200000000001 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47342.0:893 15.6 e545/543 15:67cae2ed:::rbd_mirroring:head v 545'399, mlcod=544'397)", 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:54.803399+0000", 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 88.231624859999997, 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.182441141, 2023-10-09T06:45:23.159 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.803399+0000", 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.803399+0000", 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.160 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.803399+0000", 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.803406+0000", 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 7.0060000000000003e-06 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.161 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.830873+0000", 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.027466781999999999 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.830882+0000", 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.6690000000000001e-06 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.162 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.830964+0000", 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 8.1676999999999997e-05 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.830989+0000", 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 2.4530000000000001e-05 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.985810+0000", 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.15482100800000001 2023-10-09T06:45:23.163 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:54.985841+0000", 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.1114000000000001e-05 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "description": "osd_repop(client.47342.0:1929 15.6 e546/543 15:67cae2ed:::rbd_mirroring:head v 546'1435, mlcod=546'1433)", 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "initiated_at": "2023-10-09T06:43:56.489403+0000", 2023-10-09T06:45:23.164 INFO:teuthology.orchestra.run.smithi107.stdout: "age": 86.545620955000004, 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.17608258600000001, 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "type_data": { 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "events": [ 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "initiated", 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489403+0000", 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "throttled", 2023-10-09T06:45:23.165 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489403+0000", 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "header_read", 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489403+0000", 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "all_read", 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489408+0000", 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.604e-06 2023-10-09T06:45:23.166 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "dispatched", 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489418+0000", 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 9.1889999999999996e-06 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "queued_for_pg", 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489424+0000", 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 5.9830000000000001e-06 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.167 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "reached_pg", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489459+0000", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 3.5487000000000001e-05 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "started", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.489476+0000", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 1.6608000000000001e-05 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "commit_sent", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.665439+0000", 2023-10-09T06:45:23.168 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 0.17596371899999999 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: }, 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: { 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: "event": "done", 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: "time": "2023-10-09T06:43:56.665486+0000", 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: "duration": 4.6587e-05 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: } 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout: ] 2023-10-09T06:45:23.169 INFO:teuthology.orchestra.run.smithi107.stdout:} 2023-10-09T06:45:23.170 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:23.170 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.170 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.170 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:22 smithi150 ceph-mon[194178]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[212750]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.171 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: pgmap v1777: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:23.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T06:45:23.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: osdmap e590: 8 total, 8 up, 8 in 2023-10-09T06:45:23.172 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:22 smithi107 ceph-mon[220921]: failed to encode map e590 with expected crc 2023-10-09T06:45:23.679 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:23.701 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T06:45:24.060 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:24.926 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:24 smithi107 ceph-mon[212750]: pgmap v1779: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:24.927 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:24 smithi107 ceph-mon[220921]: pgmap v1779: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:25.141 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:24 smithi150 ceph-mon[194178]: pgmap v1779: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:25.595 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:25.595 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:25.595 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:45:25.595 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:25.862 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T06:45:26.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1329143912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.114 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:25 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.131 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T06:45:26.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1329143912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.154 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:25 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1329143912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.155 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:25 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:26.481 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:26.520 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T06:45:26.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[212750]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:26.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:45:26.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:26.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[212750]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T06:45:26.857 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:45:26.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[220921]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:26.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[220921]: failed to encode map e591 with expected crc 2023-10-09T06:45:26.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:26.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[220921]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T06:45:26.858 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:26 smithi107 ceph-mon[220921]: failed to encode map e591 with expected crc 2023-10-09T06:45:26.878 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:27.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:26 smithi150 ceph-mon[194178]: pgmap v1780: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:27.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:26 smithi150 ceph-mon[194178]: failed to encode map e591 with expected crc 2023-10-09T06:45:27.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:26 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:27.024 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:26 smithi150 ceph-mon[194178]: osdmap e591: 8 total, 8 up, 8 in 2023-10-09T06:45:27.025 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:26 smithi150 ceph-mon[194178]: failed to encode map e591 with expected crc 2023-10-09T06:45:27.857 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:27 smithi107 ceph-mon[220921]: failed to encode map e591 with expected crc 2023-10-09T06:45:27.912 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:27.912 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:27.912 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:45:27.913 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:45:27.913 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:28.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:27 smithi150 ceph-mon[194178]: failed to encode map e591 with expected crc 2023-10-09T06:45:28.082 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:27 smithi150 ceph-mon[194178]: failed to encode map e591 with expected crc 2023-10-09T06:45:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:27 smithi107 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:45:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:27 smithi107 ceph-mon[212750]: failed to encode map e591 with expected crc 2023-10-09T06:45:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:27 smithi107 ceph-mon[220921]: failed to encode map e591 with expected crc 2023-10-09T06:45:28.523 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T06:45:28.873 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:28.877 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T06:45:29.139 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:28 smithi150 ceph-mon[194178]: pgmap v1782: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:29.139 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:28 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2185492745' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.140 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:28 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.150 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[212750]: pgmap v1782: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2185492745' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[220921]: pgmap v1782: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2185492745' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:28 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T06:45:29.573 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.885 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[212750]: failed to encode map e592 with expected crc 2023-10-09T06:45:29.931 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:30.066 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.067 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:29 smithi150 ceph-mon[194178]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.146 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: osdmap e592: 8 total, 8 up, 8 in 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.147 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:29 smithi107 ceph-mon[220921]: failed to encode map e592 with expected crc 2023-10-09T06:45:30.279 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:30.279 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:45:30.279 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.6 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 514.13200027699997, 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.4013005, 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.280 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144532+0000", 2023-10-09T06:45:30.281 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144530+0000", 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144569+0000", 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8152000000000003e-05 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.282 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343804+0000", 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.19923497000000001 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.343819+0000", 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.579e-05 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.283 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344902+0000", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0010829030000000001 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345087+0000", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000184298 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.345267+0000", 2023-10-09T06:45:30.284 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000180574 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.355828+0000", 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010561034 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.285 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545825+0000", 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.189996794 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545832+0000", 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.4490000000000002e-06 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.286 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98786 13.16 13:6f581b2c:::benchmark_data_smithi107_535588_object98785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 513.93167939099999, 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.90554467699999996, 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98786 2023-10-09T06:45:30.287 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344853+0000", 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.288 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344851+0000", 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344911+0000", 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0288999999999998e-05 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.289 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344924+0000", 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3145999999999999e-05 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.344930+0000", 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3849999999999999e-06 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.290 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545781+0000", 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20085007999999999 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545788+0000", 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5059999999999998e-06 2023-10-09T06:45:30.291 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244178+0000", 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69839010700000004 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for degraded object", 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244920+0000", 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00074191700000000003 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.292 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250315+0000", 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053952330000000001 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.250397+0000", 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2120000000000007e-05 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.293 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98798 13.d 13:b3d56a11:::benchmark_data_smithi107_535588_object98797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 513.92041674500001, 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.32981040700000003, 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98798 2023-10-09T06:45:30.294 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.295 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356114+0000", 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3579000000000003e-05 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.296 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356182+0000", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5062000000000001e-05 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356191+0000", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9409999999999996e-06 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356320+0000", 2023-10-09T06:45:30.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012913700000000001 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356328+0000", 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2300000000000002e-06 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578270+0000", 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.221941894 2023-10-09T06:45:30.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.578431+0000", 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016151199999999999 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,7", 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.579090+0000", 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00065903200000000004 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617475+0000", 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.038384553000000002 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.617547+0000", 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1931e-05 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685815+0000", 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.068268076999999996 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685868+0000", 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3386e-05 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.685926+0000", 2023-10-09T06:45:30.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7726000000000003e-05 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98803 13.15 13:abc5b445:::benchmark_data_smithi107_535588_object98802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 513.70854572600001, 2023-10-09T06:45:30.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.67498770900000005, 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98803 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567986+0000", 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.567984+0000", 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:30.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568068+0000", 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4116999999999994e-05 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568098+0000", 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9153999999999999e-05 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.305 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.568108+0000", 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0000000000000001e-05 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594900+0000", 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.026791966 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.594910+0000", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0601000000000001e-05 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231876+0000", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.63696583200000001 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.231989+0000", 2023-10-09T06:45:30.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000113386 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,5", 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.232227+0000", 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00023801399999999999 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235621+0000", 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033936639999999998 2023-10-09T06:45:30.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.235647+0000", 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5706000000000001e-05 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242883+0000", 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072361309999999998 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242933+0000", 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0300000000000003e-05 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.242974+0000", 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0917000000000001e-05 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.310 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98804 13.16 13:6bab783a:::benchmark_data_smithi107_535588_object98803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 513.59039826499998, 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57256559900000004, 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98804 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686134+0000", 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.312 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686133+0000", 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686232+0000", 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9147999999999993e-05 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686245+0000", 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3025e-05 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686253+0000", 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0690000000000002e-06 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686291+0000", 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8095e-05 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686300+0000", 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.1460000000000002e-06 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.244979+0000", 2023-10-09T06:45:30.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.55867869000000003 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245112+0000", 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000133514 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.245364+0000", 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000251501 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252198+0000", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068343839999999998 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.252224+0000", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5789999999999999e-05 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.255637+0000", 2023-10-09T06:45:30.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0034128890000000001 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258639+0000", 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003001709 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.258699+0000", 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0885999999999999e-05 2023-10-09T06:45:30.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:205773 13.6 13:624e0fc4:::benchmark_data_smithi107_535588_object99666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 386.755352577, 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31593875100000002, 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 205773 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521180+0000", 2023-10-09T06:45:30.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521178+0000", 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521184+0000", 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.366e-06 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521194+0000", 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0015e-05 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521200+0000", 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8919999999999997e-06 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521246+0000", 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6483000000000002e-05 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521364+0000", 2023-10-09T06:45:30.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117922 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,5", 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.521840+0000", 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047612499999999999 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.524383+0000", 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0025425209999999998 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.525153+0000", 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00077040099999999999 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837031+0000", 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.31187785499999998 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837079+0000", 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8609999999999997e-05 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:03.837118+0000", 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8887999999999998e-05 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165426 14.11 14:89616edc:::benchmark_data_smithi107_590976_object64653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.37154286000001, 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.033732128, 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165426 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.327 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904989+0000", 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904991+0000", 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0470000000000001e-06 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904996+0000", 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1580000000000004e-06 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.904999+0000", 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6850000000000001e-06 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.905040+0000", 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0907e-05 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.905046+0000", 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8409999999999996e-06 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.911594+0000", 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0065488229999999 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.911687+0000", 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2645999999999994e-05 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.912085+0000", 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000397558 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919179+0000", 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0070941600000000004 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938564+0000", 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019385083000000001 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938629+0000", 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5233000000000004e-05 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938682+0000", 2023-10-09T06:45:30.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2992999999999999e-05 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938721+0000", 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9322000000000001e-05 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165446 14.11 14:8f96ca76:::benchmark_data_smithi107_590976_object64673:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.34779894600001, 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0102008440000001, 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165446 2023-10-09T06:45:30.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928733+0000", 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928732+0000", 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928738+0000", 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6089999999999997e-06 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928756+0000", 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.7442999999999999e-05 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928763+0000", 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1840000000000002e-06 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928844+0000", 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1043000000000003e-05 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.928856+0000", 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1476e-05 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914241+0000", 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.98538567899999996 2023-10-09T06:45:30.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914346+0000", 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000105036 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.914586+0000", 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00023933799999999999 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919184+0000", 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045983209999999998 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938791+0000", 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.019606734000000001 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938843+0000", 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2163999999999997e-05 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938894+0000", 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0708999999999998e-05 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938934+0000", 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0425e-05 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165473 14.11 14:8e8a7a16:::benchmark_data_smithi107_590976_object64700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.309375891, 2023-10-09T06:45:30.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97204445699999997, 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165473 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967156+0000", 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967155+0000", 2023-10-09T06:45:30.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967161+0000", 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.327e-06 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967173+0000", 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2575000000000001e-05 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967178+0000", 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1460000000000004e-06 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967226+0000", 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7862999999999999e-05 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967236+0000", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0349e-05 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916014+0000", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.94877750699999996 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916115+0000", 2023-10-09T06:45:30.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000101289 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.916309+0000", 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000193933 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922232+0000", 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0059224799999999999 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938972+0000", 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.016740718000000002 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939114+0000", 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014144799999999999 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939161+0000", 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7512000000000002e-05 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939201+0000", 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9348999999999999e-05 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.352 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165478 14.11 14:8e82682f:::benchmark_data_smithi107_590976_object64705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.309336159, 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.97228993600000002, 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165478 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967196+0000", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967198+0000", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4700000000000001e-06 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967206+0000", 2023-10-09T06:45:30.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6839999999999996e-06 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967210+0000", 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.578e-06 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967278+0000", 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7393999999999999e-05 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:34.967286+0000", 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4999999999999999e-06 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917507+0000", 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.95022087 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917613+0000", 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010586699999999999 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.917829+0000", 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000215978 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.924990+0000", 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071616329999999997 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939247+0000", 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014256863 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939406+0000", 2023-10-09T06:45:30.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000159006 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939450+0000", 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3582999999999999e-05 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939486+0000", 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5973000000000003e-05 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165589 14.11 14:8ce7d8ce:::benchmark_data_smithi107_590976_object64816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.16766406400001, 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.83075238100000004, 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.361 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165589 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108868+0000", 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108871+0000", 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.534e-06 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108881+0000", 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0372e-05 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108887+0000", 2023-10-09T06:45:30.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0599999999999996e-06 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108950+0000", 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2433000000000003e-05 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.108960+0000", 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.04e-05 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919215+0000", 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.81025469800000005 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919288+0000", 2023-10-09T06:45:30.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3063000000000002e-05 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.919501+0000", 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00021323199999999999 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.925000+0000", 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0054988820000000001 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939289+0000", 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0142892 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939550+0000", 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00026099400000000002 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939588+0000", 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7834999999999999e-05 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939620+0000", 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2251e-05 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165615 14.11 14:8ee05e4b:::benchmark_data_smithi107_590976_object64842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.141727828, 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.80491372500000002, 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.370 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165615 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134804+0000", 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134803+0000", 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134809+0000", 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.614e-06 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.372 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134819+0000", 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9410000000000002e-06 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134826+0000", 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.686e-06 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134882+0000", 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6459999999999998e-05 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.134890+0000", 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1890000000000007e-06 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.920872+0000", 2023-10-09T06:45:30.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.78598192099999997 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.920951+0000", 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8248e-05 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.921135+0000", 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00018456700000000001 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929219+0000", 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0080839090000000002 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939522+0000", 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010303321000000001 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939652+0000", 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012991899999999999 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939688+0000", 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5663000000000001e-05 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939718+0000", 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0117999999999999e-05 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165695 14.11 14:8d5362c1:::benchmark_data_smithi107_590976_object64922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.055003696, 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.71835888999999997, 2023-10-09T06:45:30.378 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165695 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221528+0000", 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221527+0000", 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221533+0000", 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8719999999999999e-06 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221546+0000", 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2943e-05 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221553+0000", 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6579999999999996e-06 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221623+0000", 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0397999999999999e-05 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.221636+0000", 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3148e-05 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922267+0000", 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.7006308 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922368+0000", 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010130500000000001 2023-10-09T06:45:30.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.922574+0000", 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020514700000000001 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929228+0000", 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006654755 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939749+0000", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010520659 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939819+0000", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0282000000000002e-05 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939856+0000", 2023-10-09T06:45:30.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7147999999999997e-05 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939887+0000", 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0889000000000002e-05 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.386 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165730 14.11 14:8a95b677:::benchmark_data_smithi107_590976_object64957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 175.00325678799999, 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.666711689, 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165730 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273275+0000", 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273274+0000", 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273280+0000", 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8220000000000003e-06 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273293+0000", 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3315999999999999e-05 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273300+0000", 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6499999999999999e-06 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273360+0000", 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9812999999999998e-05 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.273369+0000", 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8089999999999999e-06 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923653+0000", 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.65028459800000005 2023-10-09T06:45:30.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923733+0000", 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9450999999999995e-05 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.923954+0000", 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022080600000000001 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933196+0000", 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0092422059999999993 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939791+0000", 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0065949349999999997 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939920+0000", 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012873800000000001 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939955+0000", 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5553000000000003e-05 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.939987+0000", 2023-10-09T06:45:30.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1987999999999999e-05 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165764 14.11 14:8ec3c7ab:::benchmark_data_smithi107_590976_object64991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.95543038700001, 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.61904860500000003, 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.395 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165764 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.397 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321102+0000", 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321101+0000", 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321105+0000", 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2169999999999996e-06 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321117+0000", 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1663e-05 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321121+0000", 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5360000000000003e-06 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321166+0000", 2023-10-09T06:45:30.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4274999999999999e-05 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.321173+0000", 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.6350000000000006e-06 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.925047+0000", 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.60387425900000002 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.926164+0000", 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011165070000000001 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.926373+0000", 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020852800000000001 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933203+0000", 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.006830806 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940022+0000", 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0068189089999999997 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940079+0000", 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6690000000000001e-05 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940117+0000", 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8528000000000001e-05 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940150+0000", 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2891000000000001e-05 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165786 14.11 14:88b78780:::benchmark_data_smithi107_590976_object65013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.923931006, 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.58767717600000002, 2023-10-09T06:45:30.405 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165786 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352601+0000", 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352600+0000", 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352606+0000", 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9499999999999998e-06 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352619+0000", 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2561000000000001e-05 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352624+0000", 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1420000000000001e-06 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352683+0000", 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9027000000000003e-05 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.352693+0000", 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0054999999999999e-05 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.927842+0000", 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57514921399999996 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.927915+0000", 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2760000000000001e-05 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.928102+0000", 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000187524 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938023+0000", 2023-10-09T06:45:30.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0099206620000000002 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940052+0000", 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020286380000000001 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940210+0000", 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000157882 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940247+0000", 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7533000000000001e-05 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940278+0000", 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1272000000000001e-05 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.413 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165798 14.11 14:8d92ee1f:::benchmark_data_smithi107_590976_object65025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.910039058, 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.57388538, 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.414 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165798 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366493+0000", 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366492+0000", 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366498+0000", 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4689999999999996e-06 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366509+0000", 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1209999999999999e-05 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366513+0000", 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4410000000000003e-06 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366582+0000", 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8691999999999999e-05 2023-10-09T06:45:30.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.366592+0000", 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6490000000000003e-06 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929266+0000", 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.562674919 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929342+0000", 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5185999999999995e-05 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.929544+0000", 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00020193000000000001 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938033+0000", 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0084898899999999999 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940183+0000", 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002149523 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940310+0000", 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012735900000000001 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940347+0000", 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6547000000000001e-05 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940378+0000", 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1557000000000003e-05 2023-10-09T06:45:30.421 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165900 14.11 14:8e1dafdb:::benchmark_data_smithi107_590976_object65127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.789836235, 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.45380924900000003, 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.422 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165900 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486696+0000", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486695+0000", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486701+0000", 2023-10-09T06:45:30.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9829999999999999e-06 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486718+0000", 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6121e-05 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486725+0000", 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5519999999999999e-06 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486784+0000", 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8655e-05 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.486790+0000", 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6320000000000002e-06 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931232+0000", 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.44444197699999999 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931347+0000", 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000114487 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.931582+0000", 2023-10-09T06:45:30.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000235212 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938171+0000", 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0065892980000000004 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940410+0000", 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0022390679999999999 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940438+0000", 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7515999999999999e-05 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940474+0000", 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6340000000000001e-05 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940505+0000", 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0802000000000001e-05 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165928 14.11 14:88753501:::benchmark_data_smithi107_590976_object65155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.75714918099999, 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.42130255999999999, 2023-10-09T06:45:30.430 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165928 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:45:30.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519383+0000", 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519382+0000", 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519388+0000", 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0050000000000001e-06 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519401+0000", 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3086e-05 2023-10-09T06:45:30.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519406+0000", 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8119999999999998e-06 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519450+0000", 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4354000000000003e-05 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.519456+0000", 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1999999999999999e-06 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933244+0000", 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.41378736399999999 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933371+0000", 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012719200000000001 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.933614+0000", 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000243494 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938178+0000", 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0045638270000000003 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940536+0000", 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023584399999999998 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940617+0000", 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0304999999999998e-05 2023-10-09T06:45:30.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940655+0000", 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7996e-05 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940686+0000", 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0746e-05 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.438 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:165988 14.11 14:88a9b60e:::benchmark_data_smithi107_590976_object65215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 174.670879859, 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.33515009800000001, 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:30.439 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 165988 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605652+0000", 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605651+0000", 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605657+0000", 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3140000000000003e-06 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605669+0000", 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2645e-05 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:30.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605674+0000", 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0119999999999996e-06 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605727+0000", 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3069999999999998e-05 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.605736+0000", 2023-10-09T06:45:30.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.3219999999999993e-06 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.934794+0000", 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.329058146 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.934909+0000", 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115077 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.444 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,6", 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.935148+0000", 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000239423 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.938185+0000", 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030366360000000001 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940564+0000", 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023789010000000001 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940717+0000", 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015333600000000001 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:30.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940757+0000", 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9394000000000003e-05 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:35.940802+0000", 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5697e-05 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.447 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:30.448 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:30.448 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:30.839 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T06:45:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: pgmap v1784: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[212750]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: pgmap v1784: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:30 smithi107 ceph-mon[220921]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: pgmap v1784: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: failed to encode map e593 with expected crc 2023-10-09T06:45:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:30 smithi150 ceph-mon[194178]: osdmap e593: 8 total, 8 up, 8 in 2023-10-09T06:45:31.190 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:31.892 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T06:45:32.151 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:32] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.43.0" 2023-10-09T06:45:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[212750]: 1.a scrub starts 2023-10-09T06:45:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[212750]: 1.a scrub ok 2023-10-09T06:45:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:32.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[220921]: 1.a scrub starts 2023-10-09T06:45:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[220921]: 1.a scrub ok 2023-10-09T06:45:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:32.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:31 smithi150 ceph-mon[194178]: 1.a scrub starts 2023-10-09T06:45:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:31 smithi150 ceph-mon[194178]: 1.a scrub ok 2023-10-09T06:45:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:31 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:32.549 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T06:45:32.644 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:32.644 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:32.644 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:45:32.644 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:32.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[212750]: pgmap v1786: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:32.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:45:32.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[212750]: failed to encode map e594 with expected crc 2023-10-09T06:45:32.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:32.890 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[212750]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T06:45:32.897 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[220921]: pgmap v1786: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[220921]: failed to encode map e594 with expected crc 2023-10-09T06:45:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[220921]: failed to encode map e594 with expected crc 2023-10-09T06:45:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:33.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:32 smithi107 ceph-mon[220921]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T06:45:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:32 smithi150 ceph-mon[194178]: pgmap v1786: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:32 smithi150 ceph-mon[194178]: failed to encode map e594 with expected crc 2023-10-09T06:45:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:32 smithi150 ceph-mon[194178]: failed to encode map e594 with expected crc 2023-10-09T06:45:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:32 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2760031123' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:33.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:32 smithi150 ceph-mon[194178]: osdmap e594: 8 total, 8 up, 8 in 2023-10-09T06:45:33.221 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T06:45:33.569 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:34.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:33 smithi150 ceph-mon[194178]: 1.8 scrub starts 2023-10-09T06:45:34.090 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:33 smithi150 ceph-mon[194178]: 1.8 scrub ok 2023-10-09T06:45:34.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:33 smithi107 ceph-mon[212750]: 1.8 scrub starts 2023-10-09T06:45:34.158 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:33 smithi107 ceph-mon[212750]: 1.8 scrub ok 2023-10-09T06:45:34.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:33 smithi107 ceph-mon[220921]: 1.8 scrub starts 2023-10-09T06:45:34.159 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:33 smithi107 ceph-mon[220921]: 1.8 scrub ok 2023-10-09T06:45:34.913 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T06:45:34.953 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:34.953 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:34.953 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:45:34.953 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:45:34.953 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[212750]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3921801556' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[220921]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:35.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3921801556' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:34 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:34 smithi150 ceph-mon[194178]: pgmap v1788: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 3 op/s; 57 KiB/s, 0 objects/s recovering 2023-10-09T06:45:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:34 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3921801556' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:34 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T06:45:35.548 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T06:45:35.555 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T06:45:35.900 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:35.903 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:36.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: 1.14 scrub starts 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: 1.14 scrub ok 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[212750]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: 1.14 scrub starts 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: 1.14 scrub ok 2023-10-09T06:45:36.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.267 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.267 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:36.267 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:35 smithi107 ceph-mon[220921]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: 1.14 scrub starts 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: 1.14 scrub ok 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T06:45:36.303 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:35 smithi150 ceph-mon[194178]: osdmap e595: 8 total, 8 up, 8 in 2023-10-09T06:45:36.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[212750]: 1.1a scrub starts 2023-10-09T06:45:36.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[212750]: 1.1a scrub ok 2023-10-09T06:45:36.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[212750]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.920 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[212750]: pgmap v1790: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:45:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[220921]: 1.1a scrub starts 2023-10-09T06:45:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[220921]: 1.1a scrub ok 2023-10-09T06:45:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[220921]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[220921]: failed to encode map e595 with expected crc 2023-10-09T06:45:36.921 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:36 smithi107 ceph-mon[220921]: pgmap v1790: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:45:37.124 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:36 smithi150 ceph-mon[194178]: 1.1a scrub starts 2023-10-09T06:45:37.124 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:36 smithi150 ceph-mon[194178]: 1.1a scrub ok 2023-10-09T06:45:37.124 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:36 smithi150 ceph-mon[194178]: failed to encode map e595 with expected crc 2023-10-09T06:45:37.124 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:36 smithi150 ceph-mon[194178]: failed to encode map e595 with expected crc 2023-10-09T06:45:37.124 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:36 smithi150 ceph-mon[194178]: pgmap v1790: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 21 KiB/s, 0 objects/s recovering 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:45:37.296 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 521.15060846100005, 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.50336221199999998, 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.297 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.298 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.142975+0000", 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143027+0000", 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1832000000000001e-05 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.299 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143033+0000", 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.5590000000000002e-06 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.143038+0000", 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7360000000000001e-06 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.300 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144824+0000", 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017859289999999999 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.144963+0000", 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000139577 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,4", 2023-10-09T06:45:37.301 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145090+0000", 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012642299999999999 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148188+0000", 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030985750000000001 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.302 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.149512+0000", 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0013238119999999999 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646328+0000", 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49681566199999999 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.303 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.646337+0000", 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5510000000000005e-06 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98785 13.18 13:1808f816:::benchmark_data_smithi107_535588_object98784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:37.304 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 520.95266971399997, 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.89694675400000001, 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98785 2023-10-09T06:45:37.305 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.306 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340914+0000", 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340912+0000", 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340963+0000", 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0998999999999997e-05 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340979+0000", 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5761999999999999e-05 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.308 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340986+0000", 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0770000000000002e-06 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545392+0000", 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.20440555299999999 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.309 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.545401+0000", 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.0270000000000001e-06 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237781+0000", 2023-10-09T06:45:37.310 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69237996599999996 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.237861+0000", 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0128000000000003e-05 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.311 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98788 13.2 13:407544dc:::benchmark_data_smithi107_535588_object98787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 520.95258397199996, 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.87606833799999995, 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98788 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.312 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341000+0000", 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.313 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.340999+0000", 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341044+0000", 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4756999999999999e-05 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.314 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341054+0000", 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7010000000000008e-06 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341058+0000", 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.459e-06 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341171+0000", 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011276 2023-10-09T06:45:37.315 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.341177+0000", 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0839999999999997e-06 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.316 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647137+0000", 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30595975600000003 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.647149+0000", 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2629000000000001e-05 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.216997+0000", 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.56984790500000004 2023-10-09T06:45:37.317 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.217068+0000", 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0773999999999997e-05 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.318 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178397 14.7 14:e0fa40a8:::benchmark_data_smithi107_590976_object77624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.892671041, 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10589730999999999, 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178397 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.319 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400913+0000", 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.320 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400911+0000", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400918+0000", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3579999999999997e-06 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400934+0000", 2023-10-09T06:45:37.321 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5434000000000002e-05 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400939+0000", 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3739999999999997e-06 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400983+0000", 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3353000000000003e-05 2023-10-09T06:45:37.322 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401109+0000", 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012683599999999999 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401585+0000", 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00047565699999999999 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.323 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.404448+0000", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0028627769999999999 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.407749+0000", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003300716 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506743+0000", 2023-10-09T06:45:37.324 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.098994487000000006 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506784+0000", 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1288000000000002e-05 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506810+0000", 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5507000000000002e-05 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.325 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178401 14.17 14:ece702fc:::benchmark_data_smithi107_590976_object77628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.89263486999999, 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.10584073200000001, 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.326 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178401 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.327 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400949+0000", 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400948+0000", 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.328 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400951+0000", 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0419999999999998e-06 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400956+0000", 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8930000000000004e-06 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400959+0000", 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5969999999999999e-06 2023-10-09T06:45:37.329 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.400993+0000", 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4440999999999998e-05 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401093+0000", 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9690000000000003e-05 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.330 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.401500+0000", 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00040725000000000001 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.404443+0000", 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0029427580000000002 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.331 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.407560+0000", 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031173659999999999 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506722+0000", 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.099162103000000001 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506757+0000", 2023-10-09T06:45:37.332 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4103000000000002e-05 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506790+0000", 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2941000000000002e-05 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178419 14.17 14:ebc07a88:::benchmark_data_smithi107_590976_object77646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.333 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.86908770900001, 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.082372165999999997, 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178419 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.334 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424496+0000", 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.335 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424495+0000", 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424501+0000", 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8969999999999996e-06 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.336 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424513+0000", 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2408999999999999e-05 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424519+0000", 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5910000000000003e-06 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424555+0000", 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6338999999999999e-05 2023-10-09T06:45:37.337 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.424666+0000", 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011116900000000001 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.425113+0000", 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000446771 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.338 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427285+0000", 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021724420000000001 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.431654+0000", 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0043690209999999998 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.339 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506811+0000", 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.075156195999999995 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506842+0000", 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0868000000000003e-05 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.506868+0000", 2023-10-09T06:45:37.340 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.658e-05 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178421 14.7 14:e5df54a1:::benchmark_data_smithi107_590976_object77648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.86581680800001, 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.082669139000000003, 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.341 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178421 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.342 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427767+0000", 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427765+0000", 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.343 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427772+0000", 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9040000000000003e-06 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427793+0000", 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0621000000000001e-05 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427800+0000", 2023-10-09T06:45:37.344 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.956e-06 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427837+0000", 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6757000000000001e-05 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.427932+0000", 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5500000000000004e-05 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.345 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.428346+0000", 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041347500000000001 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.430654+0000", 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023079559999999999 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.346 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.431882+0000", 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00122803 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510384+0000", 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.078502646999999995 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.347 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510412+0000", 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7922000000000001e-05 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.510436+0000", 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3734e-05 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.348 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178436 14.17 14:ec20489c:::benchmark_data_smithi107_590976_object77663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.7751059, 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.141909865, 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178436 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.349 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518478+0000", 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.350 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518477+0000", 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518483+0000", 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7509999999999997e-06 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.351 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518494+0000", 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1367e-05 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518500+0000", 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3770000000000001e-06 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518540+0000", 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0073000000000001e-05 2023-10-09T06:45:37.352 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.518660+0000", 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119566 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.519125+0000", 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00046468099999999999 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.353 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.523888+0000", 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0047633709999999998 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.526865+0000", 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002976682 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.354 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660288+0000", 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13342374700000001 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660352+0000", 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3979000000000005e-05 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660388+0000", 2023-10-09T06:45:37.355 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5169000000000002e-05 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:178444 14.7 14:e1dee3ca:::benchmark_data_smithi107_590976_object77671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e535)", 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 160.76525653300001, 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13265004399999999, 2023-10-09T06:45:37.356 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 178444 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:45:37.357 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528327+0000", 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528326+0000", 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.358 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528331+0000", 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3589999999999999e-06 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528344+0000", 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2918999999999999e-05 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.359 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528350+0000", 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8139999999999997e-06 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528419+0000", 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8691999999999999e-05 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.360 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.528548+0000", 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012942800000000001 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.529066+0000", 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00051763099999999995 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.532147+0000", 2023-10-09T06:45:37.361 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030808070000000001 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.532871+0000", 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072465400000000003 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.362 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660886+0000", 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.128015035 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660937+0000", 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0776999999999999e-05 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:56.660977+0000", 2023-10-09T06:45:37.363 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9959000000000001e-05 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184443 14.7 14:e06ab74b:::benchmark_data_smithi107_590976_object83670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 152.360404714, 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.084506680000000001, 2023-10-09T06:45:37.364 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184443 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:45:37.365 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933179+0000", 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933177+0000", 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.366 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933183+0000", 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.049e-06 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933196+0000", 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605999999999999e-05 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.367 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933202+0000", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5500000000000002e-06 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933237+0000", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5765000000000001e-05 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933341+0000", 2023-10-09T06:45:37.368 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103628 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.933801+0000", 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00046017999999999997 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.937489+0000", 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036878739999999998 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.369 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:04.939261+0000", 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001771931 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017589+0000", 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.078328362999999998 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.370 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017643+0000", 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3695e-05 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.017686+0000", 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2534000000000003e-05 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.371 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184742 14.7 14:e5ca66eb:::benchmark_data_smithi107_590976_object83969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 151.97225999700001, 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14423997699999999, 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.372 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184742 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321324+0000", 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.373 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321323+0000", 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321328+0000", 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1789999999999999e-06 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.374 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321340+0000", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2048000000000001e-05 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321345+0000", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1329999999999998e-06 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321383+0000", 2023-10-09T06:45:37.375 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7926999999999998e-05 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321493+0000", 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110463 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.322025+0000", 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00053155800000000001 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.376 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.324589+0000", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0025637350000000001 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.334123+0000", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.009534486 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465475+0000", 2023-10-09T06:45:37.377 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.131351775 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465525+0000", 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0402999999999998e-05 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.465564+0000", 2023-10-09T06:45:37.378 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8377999999999997e-05 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:184743 14.17 14:ef0552c5:::benchmark_data_smithi107_590976_object83970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 151.972228871, 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14719207600000001, 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.379 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 184743 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.380 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321355+0000", 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321354+0000", 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.381 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321357+0000", 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6019999999999998e-06 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321363+0000", 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9950000000000002e-06 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321367+0000", 2023-10-09T06:45:37.382 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1180000000000002e-06 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321415+0000", 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.833e-05 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.321540+0000", 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012452399999999999 2023-10-09T06:45:37.383 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.322031+0000", 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00049152899999999999 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.324715+0000", 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026839500000000001 2023-10-09T06:45:37.384 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.331930+0000", 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072146340000000002 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468466+0000", 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.136535939 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.385 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468511+0000", 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5204000000000003e-05 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:05.468547+0000", 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5644e-05 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.386 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:197475 14.7 14:e72db5f8:::benchmark_data_smithi107_590976_object96702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 133.96748094899999, 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.13116150100000001, 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.387 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 197475 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326103+0000", 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326101+0000", 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326108+0000", 2023-10-09T06:45:37.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8090000000000004e-06 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326123+0000", 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5167999999999999e-05 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.326130+0000", 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3879999999999997e-06 2023-10-09T06:45:37.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.328431+0000", 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002301127 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.328566+0000", 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013549100000000001 2023-10-09T06:45:37.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.329085+0000", 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00051890300000000003 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.334068+0000", 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0049826990000000002 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.334082+0000", 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4457000000000001e-05 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457186+0000", 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.123103543 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457224+0000", 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8163e-05 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:23.457264+0000", 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0188000000000002e-05 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.394 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:198497 14.17 14:e9f032a0:::benchmark_data_smithi107_590976_object97724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 132.43343226499999, 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.18453463000000001, 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:37.395 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 198497 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860151+0000", 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860150+0000", 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860160+0000", 2023-10-09T06:45:37.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0236e-05 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860182+0000", 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1721000000000001e-05 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860191+0000", 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.7790000000000007e-06 2023-10-09T06:45:37.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860241+0000", 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9258999999999997e-05 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860345+0000", 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000104053 2023-10-09T06:45:37.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 0,4", 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.860870+0000", 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00052574000000000002 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.863319+0000", 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024490380000000002 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:24.863763+0000", 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044367000000000001 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044608+0000", 2023-10-09T06:45:37.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.180844857 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044649+0000", 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0763000000000003e-05 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:43:25.044686+0000", 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7264000000000001e-05 2023-10-09T06:45:37.402 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDScrubReserve(1.f REQUEST e587)", 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:10.330846+0000", 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.962737164, 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000133342, 2023-10-09T06:45:37.403 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330846+0000", 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330846+0000", 2023-10-09T06:45:37.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330846+0000", 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330852+0000", 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6359999999999997e-06 2023-10-09T06:45:37.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330867+0000", 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5014999999999999e-05 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330874+0000", 2023-10-09T06:45:37.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7989999999999996e-06 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330938+0000", 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3783999999999998e-05 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330952+0000", 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4674e-05 2023-10-09T06:45:37.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.330980+0000", 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7344000000000001e-05 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.408 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "replica_scrub(pg: 1.f,from:0'0,to:0'0,epoch:587/585,start:1:f0000000::::0,end:MAX,chunky:1,deep:0,version:9,allow_preemption:1,priority=5)", 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:10.331596+0000", 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.961987787999998, 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000149939, 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331596+0000", 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331596+0000", 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331595+0000", 2023-10-09T06:45:37.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331603+0000", 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.1839999999999999e-06 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331624+0000", 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1301000000000002e-05 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331634+0000", 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5400000000000001e-06 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331697+0000", 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2757999999999998e-05 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331709+0000", 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2281e-05 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.331746+0000", 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6774999999999999e-05 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDScrubReserve(1.f RELEASE e587)", 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:10.332671+0000", 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 26.960912602000001, 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126253, 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332671+0000", 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332671+0000", 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332670+0000", 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332677+0000", 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6950000000000003e-06 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332691+0000", 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4019000000000001e-05 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332698+0000", 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5499999999999997e-06 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332747+0000", 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8990999999999999e-05 2023-10-09T06:45:37.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332763+0000", 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5478e-05 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:10.332797+0000", 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4468999999999998e-05 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.418 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDScrubReserve(1.a REQUEST e593)", 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:30.589760+0000", 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 6.7038232569999998, 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0001673, 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589760+0000", 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589760+0000", 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589759+0000", 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589769+0000", 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6469999999999993e-06 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589810+0000", 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0994000000000001e-05 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589820+0000", 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0457e-05 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589870+0000", 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0238999999999998e-05 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589892+0000", 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1940999999999999e-05 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:30.589928+0000", 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5326e-05 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.424 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDScrubReserve(1.8 REQUEST e594)", 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:32.623468+0000", 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.6701155100000005, 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000159183, 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623468+0000", 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623468+0000", 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623467+0000", 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623477+0000", 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0186999999999999e-05 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623498+0000", 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0803999999999999e-05 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.427 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623523+0000", 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5854e-05 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623572+0000", 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8272999999999997e-05 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.428 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623592+0000", 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0537999999999999e-05 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.623627+0000", 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5006999999999999e-05 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.429 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "replica_scrub(pg: 1.8,from:0'0,to:0'0,epoch:594/592,start:1:10000000::::0,end:MAX,chunky:1,deep:0,version:9,allow_preemption:1,priority=5)", 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:32.624334+0000", 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 4.6692494780000002, 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012474099999999999, 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:37.430 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624334+0000", 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624334+0000", 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:37.431 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624334+0000", 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624339+0000", 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3709999999999999e-06 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624352+0000", 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3125999999999999e-05 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624374+0000", 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.209e-05 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624415+0000", 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0760999999999998e-05 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624426+0000", 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1090000000000001e-05 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:32.624459+0000", 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2811000000000001e-05 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.434 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.435 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:37.435 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:37.435 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:37.923 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T06:45:37.927 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T06:45:38.266 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:37 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3580659176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:37 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:37 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3580659176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:37 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:37 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3580659176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:37 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:38.628 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T06:45:38.974 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:39.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: 1.18 scrub starts 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: 1.18 scrub ok 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: pgmap v1791: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 16 KiB/s, 0 objects/s recovering 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T06:45:39.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:38 smithi150 ceph-mon[194178]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: 1.18 scrub starts 2023-10-09T06:45:39.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: 1.18 scrub ok 2023-10-09T06:45:39.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: pgmap v1791: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 16 KiB/s, 0 objects/s recovering 2023-10-09T06:45:39.180 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: 1.18 scrub starts 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: 1.18 scrub ok 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: pgmap v1791: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 16 KiB/s, 0 objects/s recovering 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:39.181 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T06:45:39.182 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:38 smithi107 ceph-mon[220921]: failed to encode map e596 with expected crc 2023-10-09T06:45:39.701 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:39.701 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:39.701 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:45:39.702 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:39 smithi150 ceph-mon[194178]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:39 smithi150 ceph-mon[194178]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:39 smithi107 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.265 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:39 smithi107 ceph-mon[212750]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:39 smithi107 ceph-mon[220921]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.266 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:39 smithi107 ceph-mon[220921]: failed to encode map e596 with expected crc 2023-10-09T06:45:40.330 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T06:45:40.676 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:40.935 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T06:45:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:40 smithi150 ceph-mon[194178]: pgmap v1793: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:41.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:40 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:40 smithi107 ceph-mon[212750]: pgmap v1793: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:41.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:40 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:40 smithi107 ceph-mon[220921]: pgmap v1793: 103 pgs: 103 active+clean; 4.4 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:41.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:40 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T06:45:41.596 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T06:45:41.943 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:42.076 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:42.076 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:42.076 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:45:42.076 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:45:42.076 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:42.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:42.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[212750]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T06:45:42.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.210 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.211 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:42 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:42] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.43.0" 2023-10-09T06:45:42.211 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:42.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[220921]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T06:45:42.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[220921]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[220921]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.212 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:41 smithi107 ceph-mon[220921]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.255 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:41 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2761028050' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T06:45:42.256 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:41 smithi150 ceph-mon[194178]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T06:45:42.256 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:41 smithi150 ceph-mon[194178]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.256 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:41 smithi150 ceph-mon[194178]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.256 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:41 smithi150 ceph-mon[194178]: failed to encode map e597 with expected crc 2023-10-09T06:45:42.639 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T06:45:42.987 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:43.212 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:42 smithi150 ceph-mon[194178]: failed to encode map e597 with expected crc 2023-10-09T06:45:43.213 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:42 smithi150 ceph-mon[194178]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:43.231 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:42 smithi107 ceph-mon[212750]: failed to encode map e597 with expected crc 2023-10-09T06:45:43.231 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:42 smithi107 ceph-mon[212750]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:43.232 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:42 smithi107 ceph-mon[220921]: failed to encode map e597 with expected crc 2023-10-09T06:45:43.232 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:42 smithi107 ceph-mon[220921]: pgmap v1795: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:43.960 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T06:45:44.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:43 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/941178205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.228 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:43 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:173036 13.14 13:2d3acfae:::benchmark_data_smithi107_535588_object66929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:38:24.154075+0000", 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 440.23038413099999, 2023-10-09T06:45:44.387 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.252457706, 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 173036 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154075+0000", 2023-10-09T06:45:44.388 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154075+0000", 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154074+0000", 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.389 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154080+0000", 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.071e-06 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154096+0000", 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6282000000000001e-05 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.390 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154103+0000", 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.127e-06 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154223+0000", 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000120565 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154359+0000", 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013634399999999999 2023-10-09T06:45:44.391 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.154983+0000", 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00062314200000000003 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.160824+0000", 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005841349 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.392 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.161819+0000", 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00099457499999999993 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.406453+0000", 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.244634831 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.393 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.406495+0000", 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1885000000000003e-05 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:24.406533+0000", 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7781999999999998e-05 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.394 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:190510 13.14 13:2d203d1b:::benchmark_data_smithi107_535588_object84403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:38:45.031423+0000", 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 419.35303683799998, 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27246901299999998, 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 190510 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.395 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031423+0000", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031423+0000", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031422+0000", 2023-10-09T06:45:44.396 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031425+0000", 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.208e-06 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031433+0000", 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4060000000000005e-06 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.397 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031437+0000", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5899999999999999e-06 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031486+0000", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9240999999999998e-05 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.031621+0000", 2023-10-09T06:45:44.398 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013421700000000001 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.032122+0000", 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00050143599999999996 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.035112+0000", 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002990506 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.399 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.036121+0000", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001008913 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.303798+0000", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26767693599999998 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.303851+0000", 2023-10-09T06:45:44.400 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.2871000000000003e-05 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.303892+0000", 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0465e-05 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:190512 13.4 13:23b72c32:::benchmark_data_smithi107_535588_object84405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T06:45:44.401 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:38:45.033395+0000", 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 419.35106415199999, 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26792397800000001, 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 190512 2023-10-09T06:45:44.402 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033395+0000", 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033395+0000", 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.403 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033395+0000", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033398+0000", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9569999999999998e-06 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033405+0000", 2023-10-09T06:45:44.404 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.109e-06 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033409+0000", 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5530000000000003e-06 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033444+0000", 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.506e-05 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.405 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.033564+0000", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000119234 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.034071+0000", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00050771599999999996 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.036943+0000", 2023-10-09T06:45:44.406 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002871415 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.040443+0000", 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0035005980000000002 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.301230+0000", 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26078653000000002 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.407 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.301278+0000", 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7783e-05 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:38:45.301319+0000", 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1520000000000002e-05 2023-10-09T06:45:44.409 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207718 13.14 13:2ab45c79:::benchmark_data_smithi107_535588_object101611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.702213+0000", 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.682246475, 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26023054800000001, 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.410 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207718 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702213+0000", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702213+0000", 2023-10-09T06:45:44.411 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702212+0000", 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702219+0000", 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0110000000000003e-06 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.412 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702234+0000", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5673999999999999e-05 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702241+0000", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.353e-06 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702303+0000", 2023-10-09T06:45:44.413 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.2702999999999999e-05 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702418+0000", 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000114454 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703003+0000", 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00058506300000000003 2023-10-09T06:45:44.414 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711440+0000", 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0084375370000000002 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711471+0000", 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0998999999999999e-05 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.415 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962376+0000", 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25090486499999998 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962408+0000", 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1788999999999997e-05 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962444+0000", 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5451999999999999e-05 2023-10-09T06:45:44.416 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207720 13.14 13:29a514fe:::benchmark_data_smithi107_535588_object101613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.702256+0000", 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.682203313, 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26034876699999998, 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.417 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207720 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702256+0000", 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702256+0000", 2023-10-09T06:45:44.418 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702256+0000", 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702259+0000", 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1070000000000001e-06 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.419 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702268+0000", 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.6229999999999992e-06 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.702272+0000", 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9729999999999999e-06 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.420 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.421 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.421 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.707884+0000", 2023-10-09T06:45:44.421 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0056121010000000004 2023-10-09T06:45:44.422 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.422 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.422 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.708020+0000", 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135389 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.708546+0000", 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00052609200000000003 2023-10-09T06:45:44.423 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711561+0000", 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030147020000000002 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711853+0000", 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029281499999999998 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.424 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.425 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962542+0000", 2023-10-09T06:45:44.425 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25068839300000001 2023-10-09T06:45:44.425 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.425 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.425 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962574+0000", 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2595999999999998e-05 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962605+0000", 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0454999999999999e-05 2023-10-09T06:45:44.426 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207721 13.4 13:20528f6e:::benchmark_data_smithi107_535588_object101614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.703411+0000", 2023-10-09T06:45:44.427 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.68104801499999, 2023-10-09T06:45:44.428 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25911039699999999, 2023-10-09T06:45:44.428 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.429 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.430 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.431 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.432 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.432 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207721 2023-10-09T06:45:44.432 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.432 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.432 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703411+0000", 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.433 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703411+0000", 2023-10-09T06:45:44.434 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.435 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.435 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.435 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.435 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703411+0000", 2023-10-09T06:45:44.435 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703416+0000", 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4530000000000001e-06 2023-10-09T06:45:44.436 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703430+0000", 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3929e-05 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.437 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.703436+0000", 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1240000000000002e-06 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.438 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.705949+0000", 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00251278 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706055+0000", 2023-10-09T06:45:44.439 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000106465 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.706584+0000", 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00052893199999999997 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711450+0000", 2023-10-09T06:45:44.440 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048658410000000001 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.711714+0000", 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000263661 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.441 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962467+0000", 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25075360299999999 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962493+0000", 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5639999999999998e-05 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.442 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.962522+0000", 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.889e-05 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207728 13.f 13:f12b5a29:::benchmark_data_smithi107_535588_object101621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.712822+0000", 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.67163794200002, 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26877374700000001, 2023-10-09T06:45:44.443 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207728 2023-10-09T06:45:44.444 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712822+0000", 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712822+0000", 2023-10-09T06:45:44.445 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712820+0000", 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712825+0000", 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9869999999999999e-06 2023-10-09T06:45:44.446 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712840+0000", 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4929000000000001e-05 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712847+0000", 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.533e-06 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.447 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712920+0000", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.3058000000000004e-05 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.713037+0000", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011697900000000001 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.713147+0000", 2023-10-09T06:45:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011004 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.723969+0000", 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.010821913000000001 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.724846+0000", 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00087739899999999995 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981493+0000", 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25664684199999999 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981552+0000", 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8717e-05 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.450 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981595+0000", 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3402000000000002e-05 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207729 13.f 13:f4cda011:::benchmark_data_smithi107_535588_object101622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.451 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.712861+0000", 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.67159876300002, 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28033673399999998, 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207729 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.452 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712861+0000", 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712861+0000", 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.453 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712860+0000", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712863+0000", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.2559999999999998e-06 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712870+0000", 2023-10-09T06:45:44.454 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0859999999999998e-06 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.712874+0000", 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.69e-06 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716451+0000", 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003577186 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.455 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716575+0000", 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012372300000000001 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716691+0000", 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011594700000000001 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.456 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.723975+0000", 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0072835210000000003 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.724874+0000", 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00089977099999999995 2023-10-09T06:45:44.457 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993131+0000", 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.268256783 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.458 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.459 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993172+0000", 2023-10-09T06:45:44.459 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0459e-05 2023-10-09T06:45:44.459 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.460 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.460 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.460 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993197+0000", 2023-10-09T06:45:44.460 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.5788000000000001e-05 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.461 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207731 13.18 13:191ee689:::benchmark_data_smithi107_535588_object101624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.462 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715300+0000", 2023-10-09T06:45:44.462 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.669159229, 2023-10-09T06:45:44.462 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26631238899999998, 2023-10-09T06:45:44.462 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.462 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207731 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.463 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715300+0000", 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715300+0000", 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.464 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.465 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.465 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.465 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715299+0000", 2023-10-09T06:45:44.465 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715305+0000", 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8499999999999999e-06 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.466 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715316+0000", 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1578e-05 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715323+0000", 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8140000000000003e-06 2023-10-09T06:45:44.467 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.468 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.468 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.468 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716438+0000", 2023-10-09T06:45:44.468 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011147360000000001 2023-10-09T06:45:44.468 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716564+0000", 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012581800000000001 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.469 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.470 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,5", 2023-10-09T06:45:44.470 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.717014+0000", 2023-10-09T06:45:44.470 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00045022900000000002 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.719967+0000", 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0029525580000000001 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.471 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.721914+0000", 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001947297 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.472 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981546+0000", 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25963244000000002 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981583+0000", 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6708999999999997e-05 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.473 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.981613+0000", 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9658999999999998e-05 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.474 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207732 13.f 13:f83e351c:::benchmark_data_smithi107_535588_object101625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715435+0000", 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 397.66902428999998, 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27785414200000003, 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.475 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.476 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.476 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.476 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207732 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.477 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715435+0000", 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715435+0000", 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.478 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715434+0000", 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.479 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715440+0000", 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8590000000000001e-06 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715452+0000", 2023-10-09T06:45:44.480 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2153e-05 2023-10-09T06:45:44.481 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.481 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.481 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.481 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715459+0000", 2023-10-09T06:45:44.481 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3609999999999997e-06 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.720828+0000", 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053693839999999996 2023-10-09T06:45:44.482 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.720938+0000", 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110407 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.483 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 2,7", 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.721017+0000", 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8818000000000003e-05 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.724082+0000", 2023-10-09T06:45:44.484 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003065225 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.724966+0000", 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00088321899999999997 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.485 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.486 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993232+0000", 2023-10-09T06:45:44.486 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26826683800000001 2023-10-09T06:45:44.486 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.486 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.486 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.487 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993260+0000", 2023-10-09T06:45:44.487 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7948000000000001e-05 2023-10-09T06:45:44.487 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.487 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.993289+0000", 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8901e-05 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.488 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211406 13.4 13:23f07691:::benchmark_data_smithi107_535588_object105299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.665183+0000", 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 392.719276557, 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26643109700000001, 2023-10-09T06:45:44.489 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211406 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.490 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.491 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.491 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.491 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665183+0000", 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665183+0000", 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.492 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665182+0000", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665187+0000", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4530000000000001e-06 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665197+0000", 2023-10-09T06:45:44.493 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2129999999999997e-06 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665201+0000", 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6530000000000002e-06 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665238+0000", 2023-10-09T06:45:44.494 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7014000000000002e-05 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665333+0000", 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4796000000000005e-05 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.665817+0000", 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00048384799999999999 2023-10-09T06:45:44.495 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.928203+0000", 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26238574100000001 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931485+0000", 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.003282634 2023-10-09T06:45:44.496 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931516+0000", 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0601000000000001e-05 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931573+0000", 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6864999999999999e-05 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.497 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931614+0000", 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1287e-05 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211408 13.4 13:20efda77:::benchmark_data_smithi107_535588_object105301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.668390+0000", 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 392.71606987299998, 2023-10-09T06:45:44.498 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26628301900000001, 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211408 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668390+0000", 2023-10-09T06:45:44.499 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668390+0000", 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668388+0000", 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.500 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668396+0000", 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0819999999999999e-06 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668409+0000", 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3212e-05 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.501 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668416+0000", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8499999999999996e-06 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668643+0000", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00022657999999999999 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668751+0000", 2023-10-09T06:45:44.502 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010799899999999999 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.924655+0000", 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.255904517 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.503 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931683+0000", 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0070282119999999998 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.932096+0000", 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00041207700000000001 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.934585+0000", 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024895410000000001 2023-10-09T06:45:44.504 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.934634+0000", 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8664e-05 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.934673+0000", 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8894999999999999e-05 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.505 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.46706.0:211417 13.11 e484/482 13:8e220393:::benchmark_data_smithi107_535588_object105310:head v 484'9627, mlcod=484'9625)", 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.668572+0000", 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 392.71588757699999, 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25611188099999999, 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.506 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668572+0000", 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668572+0000", 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668571+0000", 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.507 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668578+0000", 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.9689999999999997e-06 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668587+0000", 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.9500000000000007e-06 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.508 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668592+0000", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6630000000000001e-06 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668649+0000", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7413000000000001e-05 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.668669+0000", 2023-10-09T06:45:44.509 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.9562999999999999e-05 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.924650+0000", 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25598159100000001 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.510 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.924684+0000", 2023-10-09T06:45:44.511 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.3589e-05 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46706.0:211406 13.4 e484/454 ondisk, result = 0)", 2023-10-09T06:45:44.513 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.669720+0000", 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 392.71473980899998, 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26190038799999998, 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669720+0000", 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.514 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669720+0000", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669719+0000", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669725+0000", 2023-10-09T06:45:44.515 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6489999999999996e-06 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.925569+0000", 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.255844287 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.925578+0000", 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.7660000000000004e-06 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.516 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931506+0000", 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0059286900000000003 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931512+0000", 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1719999999999998e-06 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.517 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931620+0000", 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010854299999999999 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop_reply(client.46706.0:211406 13.4 e484/454 ondisk, result = 0)", 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.671069+0000", 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 392.71339032700001, 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26041841900000001, 2023-10-09T06:45:44.518 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.671069+0000", 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.671069+0000", 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.519 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.671068+0000", 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.671074+0000", 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9880000000000001e-06 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.520 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.924709+0000", 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25363444200000002 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.924718+0000", 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.8769999999999999e-06 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.521 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931468+0000", 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0067509270000000003 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931477+0000", 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0690000000000002e-06 2023-10-09T06:45:44.522 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.931488+0000", 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1029000000000001e-05 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:15444 14.4 14:28932222:::benchmark_data_smithi107_590976_object15443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:51.767138+0000", 2023-10-09T06:45:44.523 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 352.61732183100003, 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28361447400000001, 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 15444 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767138+0000", 2023-10-09T06:45:44.524 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767138+0000", 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767136+0000", 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.525 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767201+0000", 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.4863000000000006e-05 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767216+0000", 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.4912999999999999e-05 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.526 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767223+0000", 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8929999999999999e-06 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767265+0000", 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2339999999999998e-05 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767364+0000", 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8203999999999997e-05 2023-10-09T06:45:44.527 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.767499+0000", 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013575499999999999 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768257+0000", 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00075726799999999996 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.528 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768580+0000", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00032334900000000001 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.050642+0000", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.282062278 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.050702+0000", 2023-10-09T06:45:44.529 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9982999999999998e-05 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.050752+0000", 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9808999999999997e-05 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:15446 14.2 14:4f6a6abb:::benchmark_data_smithi107_590976_object15445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:45:44.530 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:51.768834+0000", 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 352.61562577900003, 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.282526263, 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 15446 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.531 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768834+0000", 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768834+0000", 2023-10-09T06:45:44.532 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768833+0000", 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768904+0000", 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0789000000000003e-05 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.533 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768914+0000", 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0769e-05 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768920+0000", 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7849999999999997e-06 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.534 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.768967+0000", 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7054e-05 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.769089+0000", 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000121298 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.769195+0000", 2023-10-09T06:45:44.535 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00010632 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:51.769923+0000", 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00072805000000000005 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.050151+0000", 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.28022836000000001 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.536 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.051288+0000", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001137086 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.051322+0000", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4062e-05 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:52.051360+0000", 2023-10-09T06:45:44.537 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7452e-05 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:53422 14.2 14:44e39f4a:::benchmark_data_smithi107_590976_object53421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:40:20.624502+0000", 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 323.75995759, 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25930877600000002, 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.538 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 53422 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624502+0000", 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.539 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624502+0000", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624501+0000", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624616+0000", 2023-10-09T06:45:44.540 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000115049 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624629+0000", 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2995e-05 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624636+0000", 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7909999999999999e-06 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.541 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624685+0000", 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.9178000000000002e-05 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.542 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624833+0000", 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014819200000000001 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.624960+0000", 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012650699999999999 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.626857+0000", 2023-10-09T06:45:44.543 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0018971470000000001 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.631496+0000", 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0046392150000000004 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.883690+0000", 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25219404000000001 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.544 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.883757+0000", 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.7107000000000006e-05 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.883811+0000", 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3563999999999997e-05 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.545 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:53442 14.2 14:4553b393:::benchmark_data_smithi107_590976_object53441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:40:20.641217+0000", 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 323.74324241099998, 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.250607199, 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 53442 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.546 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641217+0000", 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641217+0000", 2023-10-09T06:45:44.547 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641216+0000", 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641246+0000", 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.0167000000000001e-05 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.548 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641257+0000", 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0936e-05 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641262+0000", 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6689999999999997e-06 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.549 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641297+0000", 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5284999999999998e-05 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641421+0000", 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00012395099999999999 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:44.550 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641593+0000", 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00017220199999999999 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.643682+0000", 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002088758 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.889672+0000", 2023-10-09T06:45:44.551 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24599037100000001 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891724+0000", 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002052211 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891785+0000", 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0214000000000003e-05 2023-10-09T06:45:44.552 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.553 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.553 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891824+0000", 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9575e-05 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93211 14.4 14:24d25822:::benchmark_data_smithi107_590976_object93210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.042276+0000", 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 278.34218320299999, 2023-10-09T06:45:44.554 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25431479899999998, 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93211 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:44.555 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042276+0000", 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042276+0000", 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042275+0000", 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:44.556 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042299+0000", 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3450000000000001e-05 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042306+0000", 2023-10-09T06:45:44.557 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0809999999999996e-06 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042310+0000", 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7019999999999999e-06 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042346+0000", 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5984e-05 2023-10-09T06:45:44.558 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042492+0000", 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014663 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 3,7", 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.042650+0000", 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015793 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.044633+0000", 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001982705 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.044718+0000", 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5532999999999996e-05 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296527+0000", 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25180862500000001 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296565+0000", 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7976999999999999e-05 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296591+0000", 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.6010999999999999e-05 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.561 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.562 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.562 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:44.562 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:44.562 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:44.562 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:43 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/941178205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.563 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:43 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:43 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/941178205' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.563 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:43 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:44.593 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T06:45:44.939 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:44.989 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T06:45:45.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[212750]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:45.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:45.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[212750]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T06:45:45.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:45:45.303 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:45:45.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[220921]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:45.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:45.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[220921]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T06:45:45.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[220921]: failed to encode map e598 with expected crc 2023-10-09T06:45:45.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:44 smithi107 ceph-mon[220921]: failed to encode map e598 with expected crc 2023-10-09T06:45:45.338 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:44 smithi150 ceph-mon[194178]: pgmap v1796: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:45.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:44 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:44 smithi150 ceph-mon[194178]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T06:45:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:44 smithi150 ceph-mon[194178]: failed to encode map e598 with expected crc 2023-10-09T06:45:45.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:44 smithi150 ceph-mon[194178]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:45 smithi150 ceph-mon[194178]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:45 smithi150 ceph-mon[194178]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.201 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:45 smithi107 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.201 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:45 smithi107 ceph-mon[212750]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.201 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:45 smithi107 ceph-mon[220921]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.202 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:45 smithi107 ceph-mon[220921]: failed to encode map e598 with expected crc 2023-10-09T06:45:46.792 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:46.792 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:46.792 INFO:teuthology.orchestra.run.smithi150.stdout: "num_ops": 0 2023-10-09T06:45:46.793 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:46.975 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[220921]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2096573443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[212750]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:47.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2096573443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.402 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:46 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:46 smithi150 ceph-mon[194178]: pgmap v1798: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:46 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2096573443' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:46 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T06:45:47.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:46 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:45:47.429 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T06:45:47.728 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T06:45:47.776 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:47.981 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.981 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.981 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[212750]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.981 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:47.981 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[212750]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T06:45:47.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[220921]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[220921]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[220921]: failed to encode map e599 with expected crc 2023-10-09T06:45:47.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:47.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:47 smithi107 ceph-mon[220921]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T06:45:48.085 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:48.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:47 smithi150 ceph-mon[194178]: failed to encode map e599 with expected crc 2023-10-09T06:45:48.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:47 smithi150 ceph-mon[194178]: failed to encode map e599 with expected crc 2023-10-09T06:45:48.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:47 smithi150 ceph-mon[194178]: failed to encode map e599 with expected crc 2023-10-09T06:45:48.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:47 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T06:45:48.321 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:47 smithi150 ceph-mon[194178]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T06:45:49.022 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:48 smithi150 ceph-mon[194178]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.218 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:49.218 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [], 2023-10-09T06:45:49.218 INFO:teuthology.orchestra.run.smithi150.stdout: "complaint_time": 30, 2023-10-09T06:45:49.218 INFO:teuthology.orchestra.run.smithi150.stdout: "num_blocked_ops": 0 2023-10-09T06:45:49.218 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T06:45:49.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[212750]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: pgmap v1800: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T06:45:49.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:48 smithi107 ceph-mon[220921]: failed to encode map e600 with expected crc 2023-10-09T06:45:49.803 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T06:45:49.990 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T06:45:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:49 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2742017818' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:49 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.154 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T06:45:50.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:49 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2742017818' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:49 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:49 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2742017818' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:49 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:50.667 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T06:45:50.981 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: pgmap v1802: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:50.981 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: Health check failed: Reduced data availability: 2 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:50.981 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: failed to encode map e601 with expected crc 2023-10-09T06:45:50.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: failed to encode map e601 with expected crc 2023-10-09T06:45:50.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:50.982 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[220921]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T06:45:51.021 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:51.235 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: pgmap v1802: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:51.235 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: Health check failed: Reduced data availability: 2 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:51.235 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:45:51.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: failed to encode map e601 with expected crc 2023-10-09T06:45:51.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:51.236 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:50 smithi107 ceph-mon[212750]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T06:45:51.355 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: pgmap v1802: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:51.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: Health check failed: Reduced data availability: 2 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2023-10-09T06:45:51.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: failed to encode map e601 with expected crc 2023-10-09T06:45:51.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: failed to encode map e601 with expected crc 2023-10-09T06:45:51.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:51.356 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:50 smithi150 ceph-mon[194178]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "size": 20, 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 600, 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "ops": [ 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98783 13.e 13:7e44f156:::benchmark_data_smithi107_535588_object98782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 535.43436037100003, 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.23612880999999999, 2023-10-09T06:45:51.576 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98783 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.577 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138909+0000", 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.578 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138907+0000", 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138985+0000", 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.8372000000000001e-05 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.138998+0000", 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.254e-05 2023-10-09T06:45:51.579 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139004+0000", 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1290000000000001e-06 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139052+0000", 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8241000000000001e-05 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.580 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139170+0000", 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000117379 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,5", 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139302+0000", 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013261800000000001 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.581 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.145027+0000", 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0057249759999999997 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.148343+0000", 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033158129999999999 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374933+0000", 2023-10-09T06:45:51.582 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22659017100000001 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.374992+0000", 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8931000000000002e-05 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.375038+0000", 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5528000000000002e-05 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.583 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 535.43424625499995, 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.30779935400000002, 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.584 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.585 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139023+0000", 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.586 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139054+0000", 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1420000000000001e-05 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139062+0000", 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.5079999999999999e-06 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.587 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.139066+0000", 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0690000000000003e-06 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140814+0000", 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.001748793 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.140929+0000", 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00011453799999999999 2023-10-09T06:45:51.588 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.141055+0000", 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126563 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.146811+0000", 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.005755418 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.589 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446811+0000", 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.29999972899999999 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.446822+0000", 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1856e-05 2023-10-09T06:45:51.590 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98784 13.7 13:e75c46fb:::benchmark_data_smithi107_535588_object98783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:51.591 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 535.31880392999994, 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.93419914299999995, 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98784 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.592 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254465+0000", 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.593 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254463+0000", 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254519+0000", 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6208999999999997e-05 2023-10-09T06:45:51.594 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254547+0000", 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.8109000000000001e-05 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254562+0000", 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5073e-05 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.595 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254724+0000", 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000161974 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.254729+0000", 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.6260000000000003e-06 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.596 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497899+0000", 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24316990899999999 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.497910+0000", 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0874e-05 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188593+0000", 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.69068296699999998 2023-10-09T06:45:51.597 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188665+0000", 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.1802000000000001e-05 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.598 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98796 13.7 13:e44dc813:::benchmark_data_smithi107_535588_object98795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 535.21715432300005, 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.84800129700000004, 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98796 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.599 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356115+0000", 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.600 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356113+0000", 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356167+0000", 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.4719000000000002e-05 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356184+0000", 2023-10-09T06:45:51.601 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6277e-05 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.356192+0000", 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9789999999999993e-06 2023-10-09T06:45:51.602 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498051+0000", 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.14185953700000001 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:51.603 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.498058+0000", 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2799999999999998e-06 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188683+0000", 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.690624601 2023-10-09T06:45:51.604 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188792+0000", 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000108497 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.605 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.188939+0000", 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014728699999999999 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.606 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.191074+0000", 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0021353840000000002 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.203068+0000", 2023-10-09T06:45:51.607 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.011993634 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204019+0000", 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000951625 2023-10-09T06:45:51.608 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204076+0000", 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6569e-05 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.609 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.204116+0000", 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.0346999999999998e-05 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.610 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:98805 13.0 13:054831b9:::benchmark_data_smithi107_535588_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 534.88705037900002, 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.528813278, 2023-10-09T06:45:51.611 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.612 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.612 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.612 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.612 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.612 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 98805 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.613 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686219+0000", 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686214+0000", 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999952 2023-10-09T06:45:51.614 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686248+0000", 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.4001000000000002e-05 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686269+0000", 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0539000000000001e-05 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.615 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686278+0000", 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2469999999999997e-06 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686339+0000", 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0735e-05 2023-10-09T06:45:51.616 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for peered", 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:56.686348+0000", 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.5559999999999996e-06 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181836+0000", 2023-10-09T06:45:51.617 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.49548772400000002 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.181965+0000", 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000129448 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.182343+0000", 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00037732500000000002 2023-10-09T06:45:51.618 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.184651+0000", 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0023081030000000002 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.199956+0000", 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015305375 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.619 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214937+0000", 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.014981213 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.214993+0000", 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5884999999999998e-05 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:36:57.215032+0000", 2023-10-09T06:45:51.620 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9060999999999998e-05 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:207733 13.0 13:0665e03e:::benchmark_data_smithi107_535588_object101626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 404.85766793699997, 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26422807999999998, 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.621 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 207733 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.622 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715601+0000", 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.623 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715600+0000", 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.624 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715606+0000", 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.3589999999999996e-06 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715622+0000", 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6038000000000001e-05 2023-10-09T06:45:51.625 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715630+0000", 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2320000000000004e-06 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.626 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715686+0000", 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.6475999999999999e-05 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.715825+0000", 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013932800000000001 2023-10-09T06:45:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,2", 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.716377+0000", 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055127400000000003 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.629 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.722227+0000", 2023-10-09T06:45:51.629 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0058503299999999999 2023-10-09T06:45:51.629 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.964701+0000", 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24247376800000001 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979740+0000", 2023-10-09T06:45:51.630 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.015039132 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979785+0000", 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5225999999999997e-05 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:06.979830+0000", 2023-10-09T06:45:51.631 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4431000000000002e-05 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211410 13.7 13:e70c994c:::benchmark_data_smithi107_535588_object105303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.90382316300003, 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.267478086, 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.632 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211410 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.633 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669446+0000", 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669444+0000", 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999976 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.634 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669454+0000", 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.8539999999999992e-06 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669466+0000", 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1838000000000001e-05 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.635 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.636 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.636 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669472+0000", 2023-10-09T06:45:51.636 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.0059999999999998e-06 2023-10-09T06:45:51.637 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.637 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.637 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.637 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669508+0000", 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6468999999999999e-05 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.669585+0000", 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.7124000000000002e-05 2023-10-09T06:45:51.638 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.670141+0000", 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00055599200000000005 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.674991+0000", 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0048495869999999998 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.639 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936590+0000", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26159882800000001 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936837+0000", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00024747099999999998 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936885+0000", 2023-10-09T06:45:51.640 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8071000000000001e-05 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936924+0000", 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.8995000000000002e-05 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211418 13.17 13:e9af5116:::benchmark_data_smithi107_535588_object105311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:45:51.641 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.900864189, 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.264762895, 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211418 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.642 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672405+0000", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672404+0000", 2023-10-09T06:45:51.643 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672414+0000", 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0522e-05 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672428+0000", 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3747999999999999e-05 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.644 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672435+0000", 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.878e-06 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672490+0000", 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5414999999999997e-05 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.645 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.672574+0000", 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.4127000000000003e-05 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673020+0000", 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00044588000000000001 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.646 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676720+0000", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0036994749999999998 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936560+0000", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25984011699999998 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937086+0000", 2023-10-09T06:45:51.647 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000526558 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937128+0000", 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1659999999999998e-05 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.937168+0000", 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9950999999999997e-05 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.648 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.46706.0:211420 13.17 13:ef772928:::benchmark_data_smithi107_535588_object105313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e484)", 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 399.90002824999999, 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.265498495, 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.46706", 2023-10-09T06:45:51.649 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/299457421", 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 211420 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:45:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673241+0000", 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673244+0000", 2023-10-09T06:45:51.651 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7500000000000001e-06 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673253+0000", 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2239999999999995e-06 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.673257+0000", 2023-10-09T06:45:51.652 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.4170000000000002e-06 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675010+0000", 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017534130000000001 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675083+0000", 2023-10-09T06:45:51.653 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.2701999999999996e-05 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 4,6", 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.675377+0000", 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00029354000000000001 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.676887+0000", 2023-10-09T06:45:51.654 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0015100999999999999 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.936617+0000", 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.259729969 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.655 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938661+0000", 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020442260000000001 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938704+0000", 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3467000000000002e-05 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:11.938740+0000", 2023-10-09T06:45:51.656 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.5256999999999998e-05 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:6728 14.f 14:fd78970a:::benchmark_data_smithi107_590976_object6727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 363.72140777800001, 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24803666199999999, 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.657 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 6728 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.658 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851862+0000", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851861+0000", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851941+0000", 2023-10-09T06:45:51.659 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.0459999999999999e-05 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851954+0000", 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2605999999999999e-05 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.851959+0000", 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8440000000000003e-06 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.660 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852021+0000", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.1607999999999997e-05 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852156+0000", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000135234 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852317+0000", 2023-10-09T06:45:51.661 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00016078500000000001 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.852928+0000", 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00061110299999999999 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:47.854096+0000", 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0011680460000000001 2023-10-09T06:45:51.662 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099820+0000", 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24572355200000001 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099856+0000", 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6050999999999998e-05 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.663 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:39:48.099898+0000", 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2388000000000001e-05 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:53446 14.f 14:fdbc8938:::benchmark_data_smithi107_590976_object53445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T06:45:51.664 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 330.93135964800001, 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24989693599999999, 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 53446 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.665 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641910+0000", 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.666 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.641908+0000", 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642000+0000", 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2066999999999998e-05 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.667 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642019+0000", 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8488999999999999e-05 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642027+0000", 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2989999999999997e-06 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.668 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642077+0000", 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0349999999999997e-05 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642204+0000", 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000126422 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.669 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.642347+0000", 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000142903 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.644067+0000", 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0017200099999999999 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.670 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.889696+0000", 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245629034 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891721+0000", 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0020251079999999999 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.671 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891762+0000", 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1298000000000003e-05 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:40:20.891807+0000", 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.46e-05 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.672 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93215 14.f 14:fff2cbd4:::benchmark_data_smithi107_590976_object93214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 285.52679035199998, 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.247730006, 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.673 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93215 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.674 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046479+0000", 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046477+0000", 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.675 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046588+0000", 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000110818 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046609+0000", 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.0880999999999999e-05 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.676 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046617+0000", 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9980000000000003e-06 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046668+0000", 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.0525000000000002e-05 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046819+0000", 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015156699999999999 2023-10-09T06:45:51.677 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.046943+0000", 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000123709 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050044+0000", 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0031011939999999998 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.678 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.291715+0000", 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24167045500000001 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294145+0000", 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0024305669999999998 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.679 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294182+0000", 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.6606999999999997e-05 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.294209+0000", 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7245000000000001e-05 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.680 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:93221 14.f 14:f8a7a189:::benchmark_data_smithi107_590976_object93220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 285.52279719199998, 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.245878864, 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 93221 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.681 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050472+0000", 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.682 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050471+0000", 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050529+0000", 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7805e-05 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050544+0000", 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5617999999999999e-05 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.683 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050550+0000", 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8390000000000003e-06 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050596+0000", 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.5843000000000001e-05 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.684 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050742+0000", 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014604700000000001 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.050897+0000", 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00015521900000000001 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.053948+0000", 2023-10-09T06:45:51.685 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0030502860000000001 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.293048+0000", 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.239100441 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296266+0000", 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032179029999999998 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.686 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296321+0000", 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.5438e-05 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:06.296351+0000", 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.9584999999999999e-05 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.687 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.688 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.688 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114305 14.10 e511/504 14:0e814216:::benchmark_data_smithi107_590976_object13532:head v 511'5874, mlcod=495'2568)", 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 261.48184388499999, 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26106436700000002, 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:45:51.689 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091425+0000", 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091424+0000", 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.690 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091437+0000", 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.3426e-05 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091458+0000", 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.039e-05 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.691 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091468+0000", 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.0360000000000001e-05 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091572+0000", 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000103453 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091599+0000", 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7172999999999999e-05 2023-10-09T06:45:51.692 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352452+0000", 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26085362600000001 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352490+0000", 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.7518999999999998e-05 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.693 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_repop(client.47051.0:114306 14.10 e511/504 14:0c753e0c:::benchmark_data_smithi107_590976_object13533:head v 511'5875, mlcod=495'2568)", 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 261.48176983600001, 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26101210699999999, 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.694 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091500+0000", 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091499+0000", 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:51.695 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091507+0000", 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5369999999999997e-06 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091524+0000", 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.6308999999999999e-05 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.696 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.091533+0000", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.4569999999999994e-06 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345321+0000", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.25378754399999998 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.345349+0000", 2023-10-09T06:45:51.697 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.7840000000000001e-05 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352493+0000", 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0071447630000000002 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.352512+0000", 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.8321e-05 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.698 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:114323 14.f 14:f062eb52:::benchmark_data_smithi107_590976_object13550:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 261.45973385899998, 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.24122175000000001, 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.699 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 114323 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113536+0000", 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.700 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113533+0000", 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999981 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.113543+0000", 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.2560000000000008e-06 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.701 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.342992+0000", 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22944947199999999 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343004+0000", 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.1386000000000001e-05 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343062+0000", 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.8409000000000003e-05 2023-10-09T06:45:51.702 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343203+0000", 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00014109999999999999 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.343349+0000", 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000146308 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.703 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.348700+0000", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0053506509999999997 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.351340+0000", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0026400450000000002 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354664+0000", 2023-10-09T06:45:51.704 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0033237610000000002 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354715+0000", 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.1208000000000002e-05 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:41:30.354757+0000", 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.2179999999999999e-05 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.705 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:147218 14.f 14:f23d7e87:::benchmark_data_smithi107_590976_object46445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e522)", 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 220.24289172100001, 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.27446428099999998, 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.706 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 147218 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.707 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330378+0000", 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330376+0000", 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999986 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330384+0000", 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.2069999999999995e-06 2023-10-09T06:45:51.708 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330400+0000", 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.5277999999999998e-05 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330406+0000", 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.6150000000000002e-06 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.709 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330460+0000", 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.3185e-05 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330559+0000", 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 9.9193999999999999e-05 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.710 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.330638+0000", 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.9406999999999994e-05 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.337385+0000", 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0067469380000000001 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.337410+0000", 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.4589000000000001e-05 2023-10-09T06:45:51.711 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604752+0000", 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.26734247799999999 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604800+0000", 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.8211000000000003e-05 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.712 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:11.604842+0000", 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.1588000000000002e-05 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "osd_op(client.47051.0:166259 14.f 14:fe8f635d:::benchmark_data_smithi107_590976_object65486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e526)", 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 195.335609095, 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.22879331999999999, 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.713 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "client_info": { 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "client": "client.47051", 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "client_addr": "172.21.15.107:0/1377603410", 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "tid": 166259 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.714 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237660+0000", 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.715 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237664+0000", 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.9009999999999996e-06 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237670+0000", 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 6.8870000000000003e-06 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.237675+0000", 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.7380000000000002e-06 2023-10-09T06:45:51.716 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241213+0000", 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0035377899999999999 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241367+0000", 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000153726 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.717 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "waiting for subops from 1,3", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.241498+0000", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00013085899999999999 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op_commit", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.244495+0000", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.002997132 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.247740+0000", 2023-10-09T06:45:51.718 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0032456429999999999 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "sub_op_commit_rec", 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466399+0000", 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.218658507 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "commit_sent", 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466430+0000", 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 3.1588000000000003e-05 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.719 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.720 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.720 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:42:36.466454+0000", 2023-10-09T06:45:51.720 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.3125999999999998e-05 2023-10-09T06:45:51.720 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDPGRecoveryDeleteReply(2.0 e579,578 2:094b5210:::rbd_object_map.e05c96dbd989.000000000000000a:head,571'25)", 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:44:52.067201+0000", 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 59.506068526, 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.00048845200000000001, 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "reached pg", 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.721 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067201+0000", 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067201+0000", 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.722 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067200+0000", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.999999 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067207+0000", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 7.0890000000000002e-06 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067229+0000", 2023-10-09T06:45:51.723 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.1593000000000001e-05 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067237+0000", 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 8.5900000000000008e-06 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067365+0000", 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000127265 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.724 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:44:52.067689+0000", 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.000324791 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "description": "MOSDScrubReserve(1.1a RELEASE e595)", 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "initiated_at": "2023-10-09T06:45:35.666019+0000", 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "age": 15.907250168000001, 2023-10-09T06:45:51.725 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0028259650000000002, 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "type_data": { 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "flag_point": "started", 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "events": [ 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "initiated", 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666019+0000", 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "throttled", 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666019+0000", 2023-10-09T06:45:51.726 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "header_read", 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666019+0000", 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4294967295.9999995 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "all_read", 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666023+0000", 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3930000000000001e-06 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.727 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "dispatched", 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666035+0000", 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2268e-05 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "queued_for_pg", 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666041+0000", 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.9009999999999999e-06 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.728 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666099+0000", 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 5.7883000000000001e-05 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "op must wait for map", 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.666103+0000", 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 4.3120000000000004e-06 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "reached_pg", 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.668810+0000", 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 0.0027066490000000002 2023-10-09T06:45:51.729 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "started", 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.668823+0000", 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 1.2563e-05 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: { 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "event": "done", 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "time": "2023-10-09T06:45:35.668845+0000", 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: "duration": 2.2629e-05 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.730 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.731 INFO:teuthology.orchestra.run.smithi150.stdout: } 2023-10-09T06:45:51.731 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2023-10-09T06:45:51.731 INFO:teuthology.orchestra.run.smithi150.stdout:} 2023-10-09T06:45:52.322 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:45:52 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:45:52] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.43.0" 2023-10-09T06:45:52.989 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[212750]: pgmap v1804: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1139552031' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[220921]: pgmap v1804: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1139552031' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:52 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:52 smithi150 ceph-mon[194178]: pgmap v1804: 103 pgs: 14 peering, 89 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:52 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1139552031' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:52 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T06:45:53.660 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T06:45:54.011 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:54.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:45:54.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:54.269 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[212750]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T06:45:54.270 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:45:54.270 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[220921]: failed to encode map e602 with expected crc 2023-10-09T06:45:54.270 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:54.270 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[220921]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T06:45:54.271 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:53 smithi107 ceph-mon[220921]: failed to encode map e602 with expected crc 2023-10-09T06:45:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:53 smithi150 ceph-mon[194178]: failed to encode map e602 with expected crc 2023-10-09T06:45:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:53 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T06:45:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:53 smithi150 ceph-mon[194178]: osdmap e602: 8 total, 8 up, 8 in 2023-10-09T06:45:54.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:53 smithi150 ceph-mon[194178]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.304 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.304 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[212750]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.304 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[212750]: pgmap v1806: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:55.304 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[212750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 12 pgs peering) 2023-10-09T06:45:55.304 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[212750]: Cluster is now healthy 2023-10-09T06:45:55.305 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[220921]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.305 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[220921]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.305 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[220921]: pgmap v1806: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:55.305 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[220921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 12 pgs peering) 2023-10-09T06:45:55.305 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:54 smithi107 ceph-mon[220921]: Cluster is now healthy 2023-10-09T06:45:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:54 smithi150 ceph-mon[194178]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:54 smithi150 ceph-mon[194178]: failed to encode map e602 with expected crc 2023-10-09T06:45:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:54 smithi150 ceph-mon[194178]: pgmap v1806: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:54 smithi150 ceph-mon[194178]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 12 pgs peering) 2023-10-09T06:45:55.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:54 smithi150 ceph-mon[194178]: Cluster is now healthy 2023-10-09T06:45:56.012 INFO:teuthology.orchestra.run.smithi107.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T06:45:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:55 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1609446189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:55 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:55 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1609446189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:55 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:55 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1609446189' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:55 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:56.684 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T06:45:57.034 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:45:57.277 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: pgmap v1807: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: pgmap v1807: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:45:57.278 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T06:45:57.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.279 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:57 smithi107 ceph-mon[220921]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: pgmap v1807: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:45:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: osdmap e603: 8 total, 8 up, 8 in 2023-10-09T06:45:57.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: failed to encode map e603 with expected crc 2023-10-09T06:45:57.405 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:57 smithi150 ceph-mon[194178]: failed to encode map e603 with expected crc 2023-10-09T06:45:58.287 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:58 smithi107 ceph-mon[212750]: failed to encode map e603 with expected crc 2023-10-09T06:45:58.287 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:58 smithi107 ceph-mon[220921]: failed to encode map e603 with expected crc 2023-10-09T06:45:58.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:58 smithi150 ceph-mon[194178]: failed to encode map e603 with expected crc 2023-10-09T06:45:59.026 INFO:teuthology.orchestra.run.smithi107.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T06:45:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:59 smithi107 ceph-mon[212750]: pgmap v1809: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:59.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:45:59 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:59.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:59 smithi107 ceph-mon[220921]: pgmap v1809: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:59.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:45:59 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:59 smithi150 ceph-mon[194178]: pgmap v1809: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:45:59.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:45:59 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T06:45:59.726 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T06:45:59.726 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T06:45:59.726 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T06:45:59.726 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T06:45:59.726 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T06:45:59.726 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:46:00.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[212750]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.033 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:46:00.034 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[212750]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T06:46:00.073 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:00.303 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[220921]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[220921]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:46:00.304 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:00 smithi107 ceph-mon[220921]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T06:46:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:00 smithi150 ceph-mon[194178]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:00 smithi150 ceph-mon[194178]: failed to encode map e604 with expected crc 2023-10-09T06:46:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:00 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2312377794' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T06:46:00.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:00 smithi150 ceph-mon[194178]: osdmap e604: 8 total, 8 up, 8 in 2023-10-09T06:46:01.369 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:01 smithi107 ceph-mon[212750]: pgmap v1811: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:01.369 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:01 smithi107 ceph-mon[220921]: pgmap v1811: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:01.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:01 smithi150 ceph-mon[194178]: pgmap v1811: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:01.806 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:46:01.806 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":604,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T06:45:59.013515+0000","last_up_change":"2023-10-09T06:16:43.936496+0000","last_in_change":"2023-10-09T06:44:56.064543+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"383","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"381","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":1.809999942779541,"score_stable":1.809999942779541,"optimal_score":1,"raw_score_acting":1.809999942779541,"raw_score_stable":1.809999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"573","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":573,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":585,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6803","nonce":2817416168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6805","nonce":2817416168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6809","nonce":2817416168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6807","nonce":2817416168}]},"public_addr":"172.21.15.107:6803/2817416168","cluster_addr":"172.21.15.107:6805/2817416168","heartbeat_back_addr":"172.21.15.107:6809/2817416168","heartbeat_front_addr":"172.21.15.107:6807/2817416168","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":585,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6811","nonce":3281752293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6813","nonce":3281752293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6817","nonce":3281752293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6815","nonce":3281752293}]},"public_addr":"172.21.15.107:6811/3281752293","cluster_addr":"172.21.15.107:6813/3281752293","heartbeat_back_addr":"172.21.15.107:6817/3281752293","heartbeat_front_addr":"172.21.15.107:6815/3281752293","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":128,"up_thru":592,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6819","nonce":2613148714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6821","nonce":2613148714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6825","nonce":2613148714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6823","nonce":2613148714}]},"public_addr":"172.21.15.107:6819/2613148714","cluster_addr":"172.21.15.107:6821/2613148714","heartbeat_back_addr":"172.21.15.107:6825/2613148714","heartbeat_front_addr":"172.21.15.107:6823/2613148714","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":149,"up_thru":585,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6827","nonce":3856205119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6829","nonce":3856205119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6833","nonce":3856205119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6831","nonce":3856205119}]},"public_addr":"172.21.15.107:6827/3856205119","cluster_addr":"172.21.15.107:6829/3856205119","heartbeat_back_addr":"172.21.15.107:6833/3856205119","heartbeat_front_addr":"172.21.15.107:6831/3856205119","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":599,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.10","mappings":[{"from":5,"to":2}]},{"pgid":"1.1a","mappings":[{"from":4,"to":2}]},{"pgid":"3.f","mappings":[{"from":4,"to":2}]},{"pgid":"4.9","mappings":[{"from":3,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/443194359":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/3156803825":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/1608610913":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1657740329":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/700020267":"2023-10-10T06:02:08.994675+0000","172.21.15.150:0/3437069098":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/2371556112":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.150:6835/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6834/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/2681598821":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/1943625242":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/643879886":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/2216216350":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2681598821":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/3684818042":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3067456200":"2023-10-10T06:00:53.065448+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/3366444430":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6801/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/2888496999":"2023-10-10T06:00:53.065448+0000","172.21.15.150:0/1037268415":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/2930305484":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/2525813986":"2023-10-10T06:00:53.065448+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:46:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:02 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:02.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:02 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2194915390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:02.152 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:02 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:46:02] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.43.0" 2023-10-09T06:46:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:02 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:02.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:02 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2194915390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:02.391 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T06:46:02.391 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T06:46:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:02 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:02.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:02 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2194915390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:02.743 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:03.250 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:03 smithi107 ceph-mon[212750]: pgmap v1812: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:03.250 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:03 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:03.250 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:03 smithi107 ceph-mon[220921]: pgmap v1812: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:03.250 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:03 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:03 smithi150 ceph-mon[194178]: pgmap v1812: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:03.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:03 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:04.338 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:46:04.339 INFO:teuthology.orchestra.run.smithi107.stdout:{"epoch":604,"fsid":"b7a98308-6667-11ee-8db5-212e2dc638e7","created":"2023-10-09T05:51:17.416913+0000","modified":"2023-10-09T06:45:59.013515+0000","last_up_change":"2023-10-09T06:16:43.936496+0000","last_in_change":"2023-10-09T06:44:56.064543+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T05:54:18.659893+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"383","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"381","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":1.809999942779541,"score_stable":1.809999942779541,"optimal_score":1,"raw_score_acting":1.809999942779541,"raw_score_stable":1.809999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T05:56:57.206762+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"573","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":573,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T05:57:52.185629+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T05:57:52.499860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"61782add-fd90-440a-b489-da0bd6cdb972","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":585,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6802","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6803","nonce":2817416168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6804","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6805","nonce":2817416168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6808","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6809","nonce":2817416168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6806","nonce":2817416168},{"type":"v1","addr":"172.21.15.107:6807","nonce":2817416168}]},"public_addr":"172.21.15.107:6803/2817416168","cluster_addr":"172.21.15.107:6805/2817416168","heartbeat_back_addr":"172.21.15.107:6809/2817416168","heartbeat_front_addr":"172.21.15.107:6807/2817416168","state":["exists","up"]},{"osd":1,"uuid":"a0c947d5-3ca8-468c-9f6d-2859ed2275b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":585,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6810","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6811","nonce":3281752293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6812","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6813","nonce":3281752293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6816","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6817","nonce":3281752293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6814","nonce":3281752293},{"type":"v1","addr":"172.21.15.107:6815","nonce":3281752293}]},"public_addr":"172.21.15.107:6811/3281752293","cluster_addr":"172.21.15.107:6813/3281752293","heartbeat_back_addr":"172.21.15.107:6817/3281752293","heartbeat_front_addr":"172.21.15.107:6815/3281752293","state":["exists","up"]},{"osd":2,"uuid":"ba0a04dd-49a5-498f-a016-aff57d88ff33","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":128,"up_thru":592,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6818","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6819","nonce":2613148714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6820","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6821","nonce":2613148714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6824","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6825","nonce":2613148714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6822","nonce":2613148714},{"type":"v1","addr":"172.21.15.107:6823","nonce":2613148714}]},"public_addr":"172.21.15.107:6819/2613148714","cluster_addr":"172.21.15.107:6821/2613148714","heartbeat_back_addr":"172.21.15.107:6825/2613148714","heartbeat_front_addr":"172.21.15.107:6823/2613148714","state":["exists","up"]},{"osd":3,"uuid":"a3ca913f-7262-4d0a-8f15-07e70fe94239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":144,"up_from":149,"up_thru":585,"down_at":145,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6826","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6827","nonce":3856205119}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6828","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6829","nonce":3856205119}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6832","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6833","nonce":3856205119}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.107:6830","nonce":3856205119},{"type":"v1","addr":"172.21.15.107:6831","nonce":3856205119}]},"public_addr":"172.21.15.107:6827/3856205119","cluster_addr":"172.21.15.107:6829/3856205119","heartbeat_back_addr":"172.21.15.107:6833/3856205119","heartbeat_front_addr":"172.21.15.107:6831/3856205119","state":["exists","up"]},{"osd":4,"uuid":"c43fad0c-5200-44d6-b725-36b66a11482e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6801","nonce":2438956864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6803","nonce":2438956864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6807","nonce":2438956864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":2438956864},{"type":"v1","addr":"172.21.15.150:6805","nonce":2438956864}]},"public_addr":"172.21.15.150:6801/2438956864","cluster_addr":"172.21.15.150:6803/2438956864","heartbeat_back_addr":"172.21.15.150:6807/2438956864","heartbeat_front_addr":"172.21.15.150:6805/2438956864","state":["exists","up"]},{"osd":5,"uuid":"98904b1f-c066-4710-974a-288f9323551d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":599,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6809","nonce":2897644682}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6811","nonce":2897644682}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6815","nonce":2897644682}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":2897644682},{"type":"v1","addr":"172.21.15.150:6813","nonce":2897644682}]},"public_addr":"172.21.15.150:6809/2897644682","cluster_addr":"172.21.15.150:6811/2897644682","heartbeat_back_addr":"172.21.15.150:6815/2897644682","heartbeat_front_addr":"172.21.15.150:6813/2897644682","state":["exists","up"]},{"osd":6,"uuid":"d40d76a2-9d6d-41d9-91ed-dfa6f8228130","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6817","nonce":2964927495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6819","nonce":2964927495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6823","nonce":2964927495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":2964927495},{"type":"v1","addr":"172.21.15.150:6821","nonce":2964927495}]},"public_addr":"172.21.15.150:6817/2964927495","cluster_addr":"172.21.15.150:6819/2964927495","heartbeat_back_addr":"172.21.15.150:6823/2964927495","heartbeat_front_addr":"172.21.15.150:6821/2964927495","state":["exists","up"]},{"osd":7,"uuid":"4772509b-9d82-4565-86d2-557b830afc74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":585,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6825","nonce":274719784}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6827","nonce":274719784}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6831","nonce":274719784}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":274719784},{"type":"v1","addr":"172.21.15.150:6829","nonce":274719784}]},"public_addr":"172.21.15.150:6825/274719784","cluster_addr":"172.21.15.150:6827/274719784","heartbeat_back_addr":"172.21.15.150:6831/274719784","heartbeat_front_addr":"172.21.15.150:6829/274719784","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:28.083433+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:53:52.077048+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:16.379221+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:54:41.218969+0000","dead_epoch":144},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:03.359658+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:25.362010+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:55:47.802406+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T05:56:10.437852+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.10","mappings":[{"from":5,"to":2}]},{"pgid":"1.1a","mappings":[{"from":4,"to":2}]},{"pgid":"3.f","mappings":[{"from":4,"to":2}]},{"pgid":"4.9","mappings":[{"from":3,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.107:0/443194359":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3067456200":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/4121084552":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6800/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/2371556112":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4289301557":"2023-10-10T05:51:45.273080+0000","172.21.15.107:6801/1545922954":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6801/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1657740329":"2023-10-10T06:03:16.414020+0000","172.21.15.150:0/700020267":"2023-10-10T06:02:08.994675+0000","172.21.15.150:0/3437069098":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6835/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.150:6834/3571092793":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6800/2171634441":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1557369746":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/2216216350":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/3684818042":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1162645774":"2023-10-10T05:56:45.306041+0000","172.21.15.107:0/3008436255":"2023-10-10T05:52:10.327166+0000","172.21.15.107:0/1999847309":"2023-10-10T05:51:45.273080+0000","172.21.15.150:0/3156803825":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/2428731564":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/3366444430":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6801/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/2888496999":"2023-10-10T06:00:53.065448+0000","172.21.15.150:0/1037268415":"2023-10-10T06:02:08.994675+0000","172.21.15.107:0/283261379":"2023-10-10T05:56:45.306041+0000","172.21.15.150:0/643879886":"2023-10-10T06:02:08.994675+0000","172.21.15.107:6801/2437377245":"2023-10-10T05:51:45.273080+0000","172.21.15.107:0/1608610913":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/2930305484":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1830374370":"2023-10-10T05:56:45.306041+0000","172.21.15.107:6800/2525813986":"2023-10-10T06:00:53.065448+0000","172.21.15.107:0/1943625242":"2023-10-10T06:03:16.414020+0000","172.21.15.107:6800/2681598821":"2023-10-10T06:03:16.414020+0000","172.21.15.107:0/4293783841":"2023-10-10T05:52:10.327166+0000","172.21.15.107:6801/2681598821":"2023-10-10T06:03:16.414020+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T06:46:04.998 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T06:46:04.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T06:46:04.999 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T06:46:05.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T06:46:05.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T06:46:05.000 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T06:46:05.001 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T06:46:05.001 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T06:46:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:05 smithi107 ceph-mon[212750]: pgmap v1813: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:05.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:05 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/870058384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:05.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:05 smithi107 ceph-mon[220921]: pgmap v1813: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:05.402 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:05 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/870058384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:05 smithi150 ceph-mon[194178]: pgmap v1813: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:05.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:05 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/870058384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T06:46:06.325 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.375 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.435 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.484 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.580 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.646 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:06.902 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:07.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:07 smithi107 ceph-mon[212750]: pgmap v1814: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:07.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:07 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:46:07.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:07 smithi107 ceph-mon[220921]: pgmap v1814: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:07.152 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:07 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:46:07.318 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:07 smithi150 ceph-mon[194178]: pgmap v1814: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:07.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:07 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T06:46:09.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:09 smithi107 ceph-mon[212750]: pgmap v1815: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:09.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:09 smithi107 ceph-mon[220921]: pgmap v1815: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:09.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:09 smithi150 ceph-mon[194178]: pgmap v1815: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:10.606 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.606 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.606 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[212750]: pgmap v1816: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:10.607 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.607 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.607 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:10 smithi107 ceph-mon[220921]: pgmap v1816: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:10 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:10.904 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:10 smithi150 ceph-mon[194178]: pgmap v1816: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:12.401 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:12 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:46:12] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.43.0" 2023-10-09T06:46:12.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:12 smithi107 ceph-mon[212750]: pgmap v1817: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:12.902 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:12 smithi107 ceph-mon[220921]: pgmap v1817: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:13.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:12 smithi150 ceph-mon[194178]: pgmap v1817: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:13.250 INFO:teuthology.orchestra.run.smithi107.stdout:128849019496 2023-10-09T06:46:13.250 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T06:46:15.008 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:14 smithi107 ceph-mon[212750]: pgmap v1818: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:15.008 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:14 smithi107 ceph-mon[220921]: pgmap v1818: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:15.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:14 smithi150 ceph-mon[194178]: pgmap v1818: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:15.545 INFO:teuthology.orchestra.run.smithi107.stdout:197568496223 2023-10-09T06:46:15.546 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T06:46:15.895 INFO:teuthology.orchestra.run.smithi107.stdout:549755814263 2023-10-09T06:46:15.895 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T06:46:16.453 INFO:teuthology.orchestra.run.smithi107.stdout:390842024394 2023-10-09T06:46:16.453 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T06:46:17.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:16 smithi107 ceph-mon[212750]: pgmap v1819: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:17.022 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:16 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:17.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:16 smithi107 ceph-mon[220921]: pgmap v1819: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:17.023 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:16 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:17.045 INFO:teuthology.orchestra.run.smithi107.stdout:163208757862 2023-10-09T06:46:17.046 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T06:46:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:16 smithi150 ceph-mon[194178]: pgmap v1819: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:17.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:16 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:17.835 INFO:teuthology.orchestra.run.smithi107.stdout:231928234588 2023-10-09T06:46:17.835 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T06:46:18.077 INFO:teuthology.orchestra.run.smithi107.stdout:639950127460 2023-10-09T06:46:18.077 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T06:46:18.163 INFO:teuthology.orchestra.run.smithi107.stdout:472446402961 2023-10-09T06:46:18.164 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T06:46:18.536 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:18.587 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:18.655 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:18.766 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:19.060 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:18 smithi107 ceph-mon[212750]: pgmap v1820: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:19.061 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:18 smithi107 ceph-mon[220921]: pgmap v1820: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:19.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:18 smithi150 ceph-mon[194178]: pgmap v1820: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:19.176 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:19.276 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:21.044 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:20 smithi107 ceph-mon[212750]: pgmap v1821: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:21.045 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:20 smithi107 ceph-mon[220921]: pgmap v1821: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:21.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:20 smithi150 ceph-mon[194178]: pgmap v1821: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:21.792 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:22.379 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:22 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:46:22] "GET /metrics HTTP/1.1" 200 34102 "" "Prometheus/2.43.0" 2023-10-09T06:46:23.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:22 smithi107 ceph-mon[212750]: pgmap v1822: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:23.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:22 smithi107 ceph-mon[220921]: pgmap v1822: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:23.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:22 smithi150 ceph-mon[194178]: pgmap v1822: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:23.570 INFO:teuthology.orchestra.run.smithi107.stdout:128849019498 2023-10-09T06:46:23.834 INFO:teuthology.orchestra.run.smithi107.stdout:231928234590 2023-10-09T06:46:24.078 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:23 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1411069081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:46:24.079 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:23 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1411069081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:46:24.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:23 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1411069081' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T06:46:24.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[212750]: pgmap v1823: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:24.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2691944558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:46:24.884 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2916597053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:46:24.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[220921]: pgmap v1823: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:24.884 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2691944558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:46:24.885 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:24 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2916597053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:46:24.886 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019496 got 128849019498 for osd.4 2023-10-09T06:46:24.886 INFO:teuthology.orchestra.run.smithi107.stdout:197568496226 2023-10-09T06:46:24.886 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:24 smithi150 ceph-mon[194178]: pgmap v1823: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2691944558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T06:46:25.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:24 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2916597053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T06:46:25.192 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234588 got 231928234590 for osd.7 2023-10-09T06:46:25.192 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:25.282 INFO:teuthology.orchestra.run.smithi107.stdout:549755814265 2023-10-09T06:46:25.309 INFO:teuthology.orchestra.run.smithi107.stdout:163208757864 2023-10-09T06:46:25.807 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:25.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3796294617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:46:25.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:25 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/3039997930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:46:25.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3796294617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:46:25.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:25 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/3039997930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:46:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3796294617' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T06:46:26.155 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:25 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/3039997930' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T06:46:26.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496223 got 197568496226 for osd.6 2023-10-09T06:46:26.635 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:26.901 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:26 smithi107 ceph-mon[212750]: pgmap v1824: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:26.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:26 smithi107 ceph-mon[220921]: pgmap v1824: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:27.123 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757862 got 163208757864 for osd.5 2023-10-09T06:46:27.123 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:27.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:26 smithi150 ceph-mon[194178]: pgmap v1824: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:27.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 549755814263 got 549755814265 for osd.2 2023-10-09T06:46:27.391 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:27.559 INFO:teuthology.orchestra.run.smithi107.stdout:390842024397 2023-10-09T06:46:28.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:27 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/2388170547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:46:28.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:27 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/2388170547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:46:28.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:27 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/2388170547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T06:46:28.593 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024394 got 390842024397 for osd.0 2023-10-09T06:46:28.593 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:28.930 INFO:teuthology.orchestra.run.smithi107.stdout:472446402964 2023-10-09T06:46:29.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:28 smithi107 ceph-mon[212750]: pgmap v1825: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:29.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:28 smithi107 ceph-mon[220921]: pgmap v1825: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:29.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:28 smithi150 ceph-mon[194178]: pgmap v1825: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:29.848 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402961 got 472446402964 for osd.1 2023-10-09T06:46:29.848 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:29.936 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:29 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/1998746447' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:46:29.936 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:29 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/1998746447' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:46:30.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:29 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/1998746447' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T06:46:30.636 INFO:teuthology.orchestra.run.smithi107.stdout:639950127464 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[212750]: pgmap v1826: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[212750]: from='client.? 172.21.15.107:0/761698650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[220921]: pgmap v1826: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:30 smithi107 ceph-mon[220921]: from='client.? 172.21.15.107:0/761698650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:46:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:30 smithi150 ceph-mon[194178]: pgmap v1826: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:30 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:31.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:30 smithi150 ceph-mon[194178]: from='client.? 172.21.15.107:0/761698650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T06:46:31.450 INFO:tasks.cephadm.ceph_manager.ceph:need seq 639950127460 got 639950127464 for osd.3 2023-10-09T06:46:31.450 DEBUG:teuthology.parallel:result is None 2023-10-09T06:46:31.450 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T06:46:31.450 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:46:31.799 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:32.064 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:32 smithi107 conmon[200038]: ::ffff:172.21.15.150 - - [09/Oct/2023:06:46:32] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[212750]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:32.065 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:32.066 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:31 smithi107 ceph-mon[220921]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T06:46:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T06:46:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' 2023-10-09T06:46:32.154 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:31 smithi150 ceph-mon[194178]: from='mgr.34098 172.21.15.107:0/3503976480' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T06:46:33.562 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:46:33.564 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:46:33.582 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:33 smithi107 ceph-mon[212750]: pgmap v1827: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:33.582 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:33 smithi107 ceph-mon[220921]: pgmap v1827: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:33.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:33 smithi150 ceph-mon[194178]: pgmap v1827: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:34.217 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1827,"stamp":"2023-10-09T06:46:31.708359+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5976,"num_read_kb":13193,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":5535728,"num_keys_recovered":23,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6023,"ondisk_log_size":6023,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10367992,"kb_used_data":65656,"kb_used_omap":20,"kb_used_meta":10302251,"kb_avail":739593224,"statfs":{"total":767960285184,"available":757343461376,"internally_reserved":0,"allocated":67231744,"data_stored":52437253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10549505678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002752"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":848,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010133+0000","last_change":"2023-10-09T06:45:47.993498+0000","last_active":"2023-10-09T06:45:56.010133+0000","last_peered":"2023-10-09T06:45:56.010133+0000","last_clean":"2023-10-09T06:45:56.010133+0000","last_became_active":"2023-10-09T06:45:47.993103+0000","last_became_peered":"2023-10-09T06:45:47.993103+0000","last_unstale":"2023-10-09T06:45:56.010133+0000","last_undegraded":"2023-10-09T06:45:56.010133+0000","last_fullsized":"2023-10-09T06:45:56.010133+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:37.631015+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:37.631015+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:06.971585+0000","scrub_duration":0.002097706,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":583,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427278+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:45:56.427278+0000","last_peered":"2023-10-09T06:45:56.427278+0000","last_clean":"2023-10-09T06:45:56.427278+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:45:56.427278+0000","last_undegraded":"2023-10-09T06:45:56.427278+0000","last_fullsized":"2023-10-09T06:45:56.427278+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":308,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426140+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:45:56.426140+0000","last_peered":"2023-10-09T06:45:56.426140+0000","last_clean":"2023-10-09T06:45:56.426140+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:45:56.426140+0000","last_undegraded":"2023-10-09T06:45:56.426140+0000","last_fullsized":"2023-10-09T06:45:56.426140+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":805,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010599+0000","last_change":"2023-10-09T06:45:47.995274+0000","last_active":"2023-10-09T06:45:56.010599+0000","last_peered":"2023-10-09T06:45:56.010599+0000","last_clean":"2023-10-09T06:45:56.010599+0000","last_became_active":"2023-10-09T06:45:47.994500+0000","last_became_peered":"2023-10-09T06:45:47.994500+0000","last_unstale":"2023-10-09T06:45:56.010599+0000","last_undegraded":"2023-10-09T06:45:56.010599+0000","last_fullsized":"2023-10-09T06:45:56.010599+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:09.307580+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":602,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007389+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:45:56.007389+0000","last_peered":"2023-10-09T06:45:56.007389+0000","last_clean":"2023-10-09T06:45:56.007389+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:45:56.007389+0000","last_undegraded":"2023-10-09T06:45:56.007389+0000","last_fullsized":"2023-10-09T06:45:56.007389+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006659+0000","last_change":"2023-10-09T06:44:51.049695+0000","last_active":"2023-10-09T06:45:56.006659+0000","last_peered":"2023-10-09T06:45:56.006659+0000","last_clean":"2023-10-09T06:45:56.006659+0000","last_became_active":"2023-10-09T06:44:51.049267+0000","last_became_peered":"2023-10-09T06:44:51.049267+0000","last_unstale":"2023-10-09T06:45:56.006659+0000","last_undegraded":"2023-10-09T06:45:56.006659+0000","last_fullsized":"2023-10-09T06:45:56.006659+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:44.496671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":652,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425228+0000","last_change":"2023-10-09T06:45:35.665965+0000","last_active":"2023-10-09T06:45:56.425228+0000","last_peered":"2023-10-09T06:45:56.425228+0000","last_clean":"2023-10-09T06:45:56.425228+0000","last_became_active":"2023-10-09T06:45:29.894620+0000","last_became_peered":"2023-10-09T06:45:29.894620+0000","last_unstale":"2023-10-09T06:45:56.425228+0000","last_undegraded":"2023-10-09T06:45:56.425228+0000","last_fullsized":"2023-10-09T06:45:56.425228+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:35.665866+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:45:35.665866+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:08.662278+0000","scrub_duration":0.0019991689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":344,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425458+0000","last_change":"2023-10-09T06:44:51.056390+0000","last_active":"2023-10-09T06:45:56.425458+0000","last_peered":"2023-10-09T06:45:56.425458+0000","last_clean":"2023-10-09T06:45:56.425458+0000","last_became_active":"2023-10-09T06:44:51.055835+0000","last_became_peered":"2023-10-09T06:44:51.055835+0000","last_unstale":"2023-10-09T06:45:56.425458+0000","last_undegraded":"2023-10-09T06:45:56.425458+0000","last_fullsized":"2023-10-09T06:45:56.425458+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:55.171015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141151+0000","last_change":"2023-10-09T06:44:57.095295+0000","last_active":"2023-10-09T06:45:56.141151+0000","last_peered":"2023-10-09T06:45:56.141151+0000","last_clean":"2023-10-09T06:45:56.141151+0000","last_became_active":"2023-10-09T06:44:57.094946+0000","last_became_peered":"2023-10-09T06:44:57.094946+0000","last_unstale":"2023-10-09T06:45:56.141151+0000","last_undegraded":"2023-10-09T06:45:56.141151+0000","last_fullsized":"2023-10-09T06:45:56.141151+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:20.863235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":712,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427800+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:45:56.427800+0000","last_peered":"2023-10-09T06:45:56.427800+0000","last_clean":"2023-10-09T06:45:56.427800+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:45:56.427800+0000","last_undegraded":"2023-10-09T06:45:56.427800+0000","last_fullsized":"2023-10-09T06:45:56.427800+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006937+0000","last_change":"2023-10-09T06:44:51.045302+0000","last_active":"2023-10-09T06:45:56.006937+0000","last_peered":"2023-10-09T06:45:56.006937+0000","last_clean":"2023-10-09T06:45:56.006937+0000","last_became_active":"2023-10-09T06:44:51.044895+0000","last_became_peered":"2023-10-09T06:44:51.044895+0000","last_unstale":"2023-10-09T06:45:56.006937+0000","last_undegraded":"2023-10-09T06:45:56.006937+0000","last_fullsized":"2023-10-09T06:45:56.006937+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:10.438707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":327,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.214582+0000","last_change":"2023-10-09T06:44:57.093911+0000","last_active":"2023-10-09T06:45:59.214582+0000","last_peered":"2023-10-09T06:45:59.214582+0000","last_clean":"2023-10-09T06:45:59.214582+0000","last_became_active":"2023-10-09T06:44:57.093575+0000","last_became_peered":"2023-10-09T06:44:57.093575+0000","last_unstale":"2023-10-09T06:45:59.214582+0000","last_undegraded":"2023-10-09T06:45:59.214582+0000","last_fullsized":"2023-10-09T06:45:59.214582+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:32.176495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":530,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425260+0000","last_change":"2023-10-09T06:45:34.673437+0000","last_active":"2023-10-09T06:45:56.425260+0000","last_peered":"2023-10-09T06:45:56.425260+0000","last_clean":"2023-10-09T06:45:56.425260+0000","last_became_active":"2023-10-09T06:45:29.896909+0000","last_became_peered":"2023-10-09T06:45:29.896909+0000","last_unstale":"2023-10-09T06:45:56.425260+0000","last_undegraded":"2023-10-09T06:45:56.425260+0000","last_fullsized":"2023-10-09T06:45:56.425260+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:34.673334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:34.673334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:48.787338+0000","scrub_duration":0.001614814,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010471+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:45:56.010471+0000","last_peered":"2023-10-09T06:45:56.010471+0000","last_clean":"2023-10-09T06:45:56.010471+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:45:56.010471+0000","last_undegraded":"2023-10-09T06:45:56.010471+0000","last_fullsized":"2023-10-09T06:45:56.010471+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":327,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215102+0000","last_change":"2023-10-09T06:44:57.094186+0000","last_active":"2023-10-09T06:45:59.215102+0000","last_peered":"2023-10-09T06:45:59.215102+0000","last_clean":"2023-10-09T06:45:59.215102+0000","last_became_active":"2023-10-09T06:44:57.093864+0000","last_became_peered":"2023-10-09T06:44:57.093864+0000","last_unstale":"2023-10-09T06:45:59.215102+0000","last_undegraded":"2023-10-09T06:45:59.215102+0000","last_fullsized":"2023-10-09T06:45:59.215102+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:59.236931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":682,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010493+0000","last_change":"2023-10-09T06:45:47.994449+0000","last_active":"2023-10-09T06:45:56.010493+0000","last_peered":"2023-10-09T06:45:56.010493+0000","last_clean":"2023-10-09T06:45:56.010493+0000","last_became_active":"2023-10-09T06:45:47.994131+0000","last_became_peered":"2023-10-09T06:45:47.994131+0000","last_unstale":"2023-10-09T06:45:56.010493+0000","last_undegraded":"2023-10-09T06:45:56.010493+0000","last_fullsized":"2023-10-09T06:45:56.010493+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:48.403216+0000","scrub_duration":0.0017668409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":485,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428348+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:45:56.428348+0000","last_peered":"2023-10-09T06:45:56.428348+0000","last_clean":"2023-10-09T06:45:56.428348+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:45:56.428348+0000","last_undegraded":"2023-10-09T06:45:56.428348+0000","last_fullsized":"2023-10-09T06:45:56.428348+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":580,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008308+0000","last_change":"2023-10-09T06:44:57.100121+0000","last_active":"2023-10-09T06:45:56.008308+0000","last_peered":"2023-10-09T06:45:56.008308+0000","last_clean":"2023-10-09T06:45:56.008308+0000","last_became_active":"2023-10-09T06:44:57.099823+0000","last_became_peered":"2023-10-09T06:44:57.099823+0000","last_unstale":"2023-10-09T06:45:56.008308+0000","last_undegraded":"2023-10-09T06:45:56.008308+0000","last_fullsized":"2023-10-09T06:45:56.008308+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:27.144157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":749,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007882+0000","last_change":"2023-10-09T06:44:53.741205+0000","last_active":"2023-10-09T06:45:56.007882+0000","last_peered":"2023-10-09T06:45:56.007882+0000","last_clean":"2023-10-09T06:45:56.007882+0000","last_became_active":"2023-10-09T06:44:53.740849+0000","last_became_peered":"2023-10-09T06:44:53.740849+0000","last_unstale":"2023-10-09T06:45:56.007882+0000","last_undegraded":"2023-10-09T06:45:56.007882+0000","last_fullsized":"2023-10-09T06:45:56.007882+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:50.967855+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":650,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428070+0000","last_change":"2023-10-09T06:44:57.112092+0000","last_active":"2023-10-09T06:45:56.428070+0000","last_peered":"2023-10-09T06:45:56.428070+0000","last_clean":"2023-10-09T06:45:56.428070+0000","last_became_active":"2023-10-09T06:44:57.102041+0000","last_became_peered":"2023-10-09T06:44:57.102041+0000","last_unstale":"2023-10-09T06:45:56.428070+0000","last_undegraded":"2023-10-09T06:45:56.428070+0000","last_fullsized":"2023-10-09T06:45:56.428070+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:31.390225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":594,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428132+0000","last_change":"2023-10-09T06:44:57.095258+0000","last_active":"2023-10-09T06:45:56.428132+0000","last_peered":"2023-10-09T06:45:56.428132+0000","last_clean":"2023-10-09T06:45:56.428132+0000","last_became_active":"2023-10-09T06:44:57.094515+0000","last_became_peered":"2023-10-09T06:44:57.094515+0000","last_unstale":"2023-10-09T06:45:56.428132+0000","last_undegraded":"2023-10-09T06:45:56.428132+0000","last_fullsized":"2023-10-09T06:45:56.428132+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:07.381588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":764,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006480+0000","last_change":"2023-10-09T06:44:58.524760+0000","last_active":"2023-10-09T06:45:56.006480+0000","last_peered":"2023-10-09T06:45:56.006480+0000","last_clean":"2023-10-09T06:45:56.006480+0000","last_became_active":"2023-10-09T06:44:53.745772+0000","last_became_peered":"2023-10-09T06:44:53.745772+0000","last_unstale":"2023-10-09T06:45:56.006480+0000","last_undegraded":"2023-10-09T06:45:56.006480+0000","last_fullsized":"2023-10-09T06:45:56.006480+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:58.524664+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:44:58.524664+0000","last_clean_scrub_stamp":"2023-10-09T06:44:58.524664+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:16.128021+0000","scrub_duration":0.0022087320000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":453,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427314+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:45:56.427314+0000","last_peered":"2023-10-09T06:45:56.427314+0000","last_clean":"2023-10-09T06:45:56.427314+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:45:56.427314+0000","last_undegraded":"2023-10-09T06:45:56.427314+0000","last_fullsized":"2023-10-09T06:45:56.427314+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":309,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215562+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:45:59.215562+0000","last_peered":"2023-10-09T06:45:59.215562+0000","last_clean":"2023-10-09T06:45:59.215562+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:45:59.215562+0000","last_undegraded":"2023-10-09T06:45:59.215562+0000","last_fullsized":"2023-10-09T06:45:59.215562+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":876,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425003+0000","last_change":"2023-10-09T06:45:29.895178+0000","last_active":"2023-10-09T06:45:56.425003+0000","last_peered":"2023-10-09T06:45:56.425003+0000","last_clean":"2023-10-09T06:45:56.425003+0000","last_became_active":"2023-10-09T06:45:29.894775+0000","last_became_peered":"2023-10-09T06:45:29.894775+0000","last_unstale":"2023-10-09T06:45:56.425003+0000","last_undegraded":"2023-10-09T06:45:56.425003+0000","last_fullsized":"2023-10-09T06:45:56.425003+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:07.386362+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006521+0000","last_change":"2023-10-09T06:44:51.049108+0000","last_active":"2023-10-09T06:45:56.006521+0000","last_peered":"2023-10-09T06:45:56.006521+0000","last_clean":"2023-10-09T06:45:56.006521+0000","last_became_active":"2023-10-09T06:44:51.048886+0000","last_became_peered":"2023-10-09T06:44:51.048886+0000","last_unstale":"2023-10-09T06:45:56.006521+0000","last_undegraded":"2023-10-09T06:45:56.006521+0000","last_fullsized":"2023-10-09T06:45:56.006521+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:55.701046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":288,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010346+0000","last_change":"2023-10-09T06:45:47.995720+0000","last_active":"2023-10-09T06:45:56.010346+0000","last_peered":"2023-10-09T06:45:56.010346+0000","last_clean":"2023-10-09T06:45:56.010346+0000","last_became_active":"2023-10-09T06:45:47.994870+0000","last_became_peered":"2023-10-09T06:45:47.994870+0000","last_unstale":"2023-10-09T06:45:56.010346+0000","last_undegraded":"2023-10-09T06:45:56.010346+0000","last_fullsized":"2023-10-09T06:45:56.010346+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:09.324884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":934,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007676+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:45:56.007676+0000","last_peered":"2023-10-09T06:45:56.007676+0000","last_clean":"2023-10-09T06:45:56.007676+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:45:56.007676+0000","last_undegraded":"2023-10-09T06:45:56.007676+0000","last_fullsized":"2023-10-09T06:45:56.007676+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":487,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427214+0000","last_change":"2023-10-09T06:44:57.102182+0000","last_active":"2023-10-09T06:45:56.427214+0000","last_peered":"2023-10-09T06:45:56.427214+0000","last_clean":"2023-10-09T06:45:56.427214+0000","last_became_active":"2023-10-09T06:44:57.101856+0000","last_became_peered":"2023-10-09T06:44:57.101856+0000","last_unstale":"2023-10-09T06:45:56.427214+0000","last_undegraded":"2023-10-09T06:45:56.427214+0000","last_fullsized":"2023-10-09T06:45:56.427214+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:42.716993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006866+0000","last_change":"2023-10-09T06:44:51.052808+0000","last_active":"2023-10-09T06:45:56.006866+0000","last_peered":"2023-10-09T06:45:56.006866+0000","last_clean":"2023-10-09T06:45:56.006866+0000","last_became_active":"2023-10-09T06:44:51.052459+0000","last_became_peered":"2023-10-09T06:44:51.052459+0000","last_unstale":"2023-10-09T06:45:56.006866+0000","last_undegraded":"2023-10-09T06:45:56.006866+0000","last_fullsized":"2023-10-09T06:45:56.006866+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:27.222371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":758,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010399+0000","last_change":"2023-10-09T06:45:00.113440+0000","last_active":"2023-10-09T06:45:56.010399+0000","last_peered":"2023-10-09T06:45:56.010399+0000","last_clean":"2023-10-09T06:45:56.010399+0000","last_became_active":"2023-10-09T06:45:00.113082+0000","last_became_peered":"2023-10-09T06:45:00.113082+0000","last_unstale":"2023-10-09T06:45:56.010399+0000","last_undegraded":"2023-10-09T06:45:56.010399+0000","last_fullsized":"2023-10-09T06:45:56.010399+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:16.522169+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140736+0000","last_change":"2023-10-09T06:44:57.118343+0000","last_active":"2023-10-09T06:45:56.140736+0000","last_peered":"2023-10-09T06:45:56.140736+0000","last_clean":"2023-10-09T06:45:56.140736+0000","last_became_active":"2023-10-09T06:44:57.095708+0000","last_became_peered":"2023-10-09T06:44:57.095708+0000","last_unstale":"2023-10-09T06:45:56.140736+0000","last_undegraded":"2023-10-09T06:45:56.140736+0000","last_fullsized":"2023-10-09T06:45:56.140736+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:34.643625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":308,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426301+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:45:56.426301+0000","last_peered":"2023-10-09T06:45:56.426301+0000","last_clean":"2023-10-09T06:45:56.426301+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:45:56.426301+0000","last_undegraded":"2023-10-09T06:45:56.426301+0000","last_fullsized":"2023-10-09T06:45:56.426301+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":731,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427920+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:45:56.427920+0000","last_peered":"2023-10-09T06:45:56.427920+0000","last_clean":"2023-10-09T06:45:56.427920+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:45:56.427920+0000","last_undegraded":"2023-10-09T06:45:56.427920+0000","last_fullsized":"2023-10-09T06:45:56.427920+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":517,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428456+0000","last_change":"2023-10-09T06:44:51.062738+0000","last_active":"2023-10-09T06:45:56.428456+0000","last_peered":"2023-10-09T06:45:56.428456+0000","last_clean":"2023-10-09T06:45:56.428456+0000","last_became_active":"2023-10-09T06:44:51.055897+0000","last_became_peered":"2023-10-09T06:44:51.055897+0000","last_unstale":"2023-10-09T06:45:56.428456+0000","last_undegraded":"2023-10-09T06:45:56.428456+0000","last_fullsized":"2023-10-09T06:45:56.428456+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:02.436082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":394,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010027+0000","last_change":"2023-10-09T06:45:47.996770+0000","last_active":"2023-10-09T06:45:56.010027+0000","last_peered":"2023-10-09T06:45:56.010027+0000","last_clean":"2023-10-09T06:45:56.010027+0000","last_became_active":"2023-10-09T06:45:47.996439+0000","last_became_peered":"2023-10-09T06:45:47.996439+0000","last_unstale":"2023-10-09T06:45:56.010027+0000","last_undegraded":"2023-10-09T06:45:56.010027+0000","last_fullsized":"2023-10-09T06:45:56.010027+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.474695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1077,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425177+0000","last_change":"2023-10-09T06:45:00.108744+0000","last_active":"2023-10-09T06:45:56.425177+0000","last_peered":"2023-10-09T06:45:56.425177+0000","last_clean":"2023-10-09T06:45:56.425177+0000","last_became_active":"2023-10-09T06:45:00.108419+0000","last_became_peered":"2023-10-09T06:45:00.108419+0000","last_unstale":"2023-10-09T06:45:56.425177+0000","last_undegraded":"2023-10-09T06:45:56.425177+0000","last_fullsized":"2023-10-09T06:45:56.425177+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:42.093677+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":451,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428107+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:45:56.428107+0000","last_peered":"2023-10-09T06:45:56.428107+0000","last_clean":"2023-10-09T06:45:56.428107+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:45:56.428107+0000","last_undegraded":"2023-10-09T06:45:56.428107+0000","last_fullsized":"2023-10-09T06:45:56.428107+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":611,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427524+0000","last_change":"2023-10-09T06:44:18.735683+0000","last_active":"2023-10-09T06:45:56.427524+0000","last_peered":"2023-10-09T06:45:56.427524+0000","last_clean":"2023-10-09T06:45:56.427524+0000","last_became_active":"2023-10-09T06:44:18.735354+0000","last_became_peered":"2023-10-09T06:44:18.735354+0000","last_unstale":"2023-10-09T06:45:56.427524+0000","last_undegraded":"2023-10-09T06:45:56.427524+0000","last_fullsized":"2023-10-09T06:45:56.427524+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.979649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":702,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140523+0000","last_change":"2023-10-09T06:45:10.332573+0000","last_active":"2023-10-09T06:45:56.140523+0000","last_peered":"2023-10-09T06:45:56.140523+0000","last_clean":"2023-10-09T06:45:56.140523+0000","last_became_active":"2023-10-09T06:45:00.110237+0000","last_became_peered":"2023-10-09T06:45:00.110237+0000","last_unstale":"2023-10-09T06:45:56.140523+0000","last_undegraded":"2023-10-09T06:45:56.140523+0000","last_fullsized":"2023-10-09T06:45:56.140523+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:10.332488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:10.332488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:12.917946+0000","scrub_duration":0.001695691,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":535,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428098+0000","last_change":"2023-10-09T06:44:57.113953+0000","last_active":"2023-10-09T06:45:56.428098+0000","last_peered":"2023-10-09T06:45:56.428098+0000","last_clean":"2023-10-09T06:45:56.428098+0000","last_became_active":"2023-10-09T06:44:57.099932+0000","last_became_peered":"2023-10-09T06:44:57.099932+0000","last_unstale":"2023-10-09T06:45:56.428098+0000","last_undegraded":"2023-10-09T06:45:56.428098+0000","last_fullsized":"2023-10-09T06:45:56.428098+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:51.434120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140708+0000","last_change":"2023-10-09T06:44:57.098608+0000","last_active":"2023-10-09T06:45:56.140708+0000","last_peered":"2023-10-09T06:45:56.140708+0000","last_clean":"2023-10-09T06:45:56.140708+0000","last_became_active":"2023-10-09T06:44:57.098275+0000","last_became_peered":"2023-10-09T06:44:57.098275+0000","last_unstale":"2023-10-09T06:45:56.140708+0000","last_undegraded":"2023-10-09T06:45:56.140708+0000","last_fullsized":"2023-10-09T06:45:56.140708+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:48.449595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":728,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428254+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:45:56.428254+0000","last_peered":"2023-10-09T06:45:56.428254+0000","last_clean":"2023-10-09T06:45:56.428254+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:45:56.428254+0000","last_undegraded":"2023-10-09T06:45:56.428254+0000","last_fullsized":"2023-10-09T06:45:56.428254+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010315+0000","last_change":"2023-10-09T06:45:47.995081+0000","last_active":"2023-10-09T06:45:56.010315+0000","last_peered":"2023-10-09T06:45:56.010315+0000","last_clean":"2023-10-09T06:45:56.010315+0000","last_became_active":"2023-10-09T06:45:47.994801+0000","last_became_peered":"2023-10-09T06:45:47.994801+0000","last_unstale":"2023-10-09T06:45:56.010315+0000","last_undegraded":"2023-10-09T06:45:56.010315+0000","last_fullsized":"2023-10-09T06:45:56.010315+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:12.460282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007013+0000","last_change":"2023-10-09T06:44:51.050701+0000","last_active":"2023-10-09T06:45:56.007013+0000","last_peered":"2023-10-09T06:45:56.007013+0000","last_clean":"2023-10-09T06:45:56.007013+0000","last_became_active":"2023-10-09T06:44:51.050372+0000","last_became_peered":"2023-10-09T06:44:51.050372+0000","last_unstale":"2023-10-09T06:45:56.007013+0000","last_undegraded":"2023-10-09T06:45:56.007013+0000","last_fullsized":"2023-10-09T06:45:56.007013+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:33.153489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":956,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427114+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:45:56.427114+0000","last_peered":"2023-10-09T06:45:56.427114+0000","last_clean":"2023-10-09T06:45:56.427114+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:45:56.427114+0000","last_undegraded":"2023-10-09T06:45:56.427114+0000","last_fullsized":"2023-10-09T06:45:56.427114+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":361,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425910+0000","last_change":"2023-10-09T06:44:51.064784+0000","last_active":"2023-10-09T06:45:56.425910+0000","last_peered":"2023-10-09T06:45:56.425910+0000","last_clean":"2023-10-09T06:45:56.425910+0000","last_became_active":"2023-10-09T06:44:51.057499+0000","last_became_peered":"2023-10-09T06:44:51.057499+0000","last_unstale":"2023-10-09T06:45:56.425910+0000","last_undegraded":"2023-10-09T06:45:56.425910+0000","last_fullsized":"2023-10-09T06:45:56.425910+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:52.148891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":604,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427900+0000","last_change":"2023-10-09T06:44:57.099550+0000","last_active":"2023-10-09T06:45:56.427900+0000","last_peered":"2023-10-09T06:45:56.427900+0000","last_clean":"2023-10-09T06:45:56.427900+0000","last_became_active":"2023-10-09T06:44:57.099246+0000","last_became_peered":"2023-10-09T06:44:57.099246+0000","last_unstale":"2023-10-09T06:45:56.427900+0000","last_undegraded":"2023-10-09T06:45:56.427900+0000","last_fullsized":"2023-10-09T06:45:56.427900+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:22.898067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":806,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010105+0000","last_change":"2023-10-09T06:45:47.995383+0000","last_active":"2023-10-09T06:45:56.010105+0000","last_peered":"2023-10-09T06:45:56.010105+0000","last_clean":"2023-10-09T06:45:56.010105+0000","last_became_active":"2023-10-09T06:45:47.995061+0000","last_became_peered":"2023-10-09T06:45:47.995061+0000","last_unstale":"2023-10-09T06:45:56.010105+0000","last_undegraded":"2023-10-09T06:45:56.010105+0000","last_fullsized":"2023-10-09T06:45:56.010105+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:32.625098+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:45:32.625098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:40.864954+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140732+0000","last_change":"2023-10-09T06:44:57.101594+0000","last_active":"2023-10-09T06:45:56.140732+0000","last_peered":"2023-10-09T06:45:56.140732+0000","last_clean":"2023-10-09T06:45:56.140732+0000","last_became_active":"2023-10-09T06:44:57.101280+0000","last_became_peered":"2023-10-09T06:44:57.101280+0000","last_unstale":"2023-10-09T06:45:56.140732+0000","last_undegraded":"2023-10-09T06:45:56.140732+0000","last_fullsized":"2023-10-09T06:45:56.140732+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:27.578039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":581,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428351+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:45:56.428351+0000","last_peered":"2023-10-09T06:45:56.428351+0000","last_clean":"2023-10-09T06:45:56.428351+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:45:56.428351+0000","last_undegraded":"2023-10-09T06:45:56.428351+0000","last_fullsized":"2023-10-09T06:45:56.428351+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":977,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007767+0000","last_change":"2023-10-09T06:44:53.743777+0000","last_active":"2023-10-09T06:45:56.007767+0000","last_peered":"2023-10-09T06:45:56.007767+0000","last_clean":"2023-10-09T06:45:56.007767+0000","last_became_active":"2023-10-09T06:44:53.743495+0000","last_became_peered":"2023-10-09T06:44:53.743495+0000","last_unstale":"2023-10-09T06:45:56.007767+0000","last_undegraded":"2023-10-09T06:45:56.007767+0000","last_fullsized":"2023-10-09T06:45:56.007767+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:03.319999+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":585,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427262+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:45:56.427262+0000","last_peered":"2023-10-09T06:45:56.427262+0000","last_clean":"2023-10-09T06:45:56.427262+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:45:56.427262+0000","last_undegraded":"2023-10-09T06:45:56.427262+0000","last_fullsized":"2023-10-09T06:45:56.427262+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":584,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428441+0000","last_change":"2023-10-09T06:44:57.100399+0000","last_active":"2023-10-09T06:45:56.428441+0000","last_peered":"2023-10-09T06:45:56.428441+0000","last_clean":"2023-10-09T06:45:56.428441+0000","last_became_active":"2023-10-09T06:44:57.099999+0000","last_became_peered":"2023-10-09T06:44:57.099999+0000","last_unstale":"2023-10-09T06:45:56.428441+0000","last_undegraded":"2023-10-09T06:45:56.428441+0000","last_fullsized":"2023-10-09T06:45:56.428441+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:32.627021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":784,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010281+0000","last_change":"2023-10-09T06:45:47.994732+0000","last_active":"2023-10-09T06:45:56.010281+0000","last_peered":"2023-10-09T06:45:56.010281+0000","last_clean":"2023-10-09T06:45:56.010281+0000","last_became_active":"2023-10-09T06:45:47.994373+0000","last_became_peered":"2023-10-09T06:45:47.994373+0000","last_unstale":"2023-10-09T06:45:56.010281+0000","last_undegraded":"2023-10-09T06:45:56.010281+0000","last_fullsized":"2023-10-09T06:45:56.010281+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:30.591268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:30.591268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:54.272606+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":309,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425546+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:45:56.425546+0000","last_peered":"2023-10-09T06:45:56.425546+0000","last_clean":"2023-10-09T06:45:56.425546+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:45:56.425546+0000","last_undegraded":"2023-10-09T06:45:56.425546+0000","last_fullsized":"2023-10-09T06:45:56.425546+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":562,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007857+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:45:56.007857+0000","last_peered":"2023-10-09T06:45:56.007857+0000","last_clean":"2023-10-09T06:45:56.007857+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:45:56.007857+0000","last_undegraded":"2023-10-09T06:45:56.007857+0000","last_fullsized":"2023-10-09T06:45:56.007857+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":659,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141137+0000","last_change":"2023-10-09T06:45:00.109875+0000","last_active":"2023-10-09T06:45:56.141137+0000","last_peered":"2023-10-09T06:45:56.141137+0000","last_clean":"2023-10-09T06:45:56.141137+0000","last_became_active":"2023-10-09T06:45:00.109495+0000","last_became_peered":"2023-10-09T06:45:00.109495+0000","last_unstale":"2023-10-09T06:45:56.141137+0000","last_undegraded":"2023-10-09T06:45:56.141137+0000","last_fullsized":"2023-10-09T06:45:56.141137+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:03.276039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:03.276039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:20.714787+0000","scrub_duration":0.0023411270000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"573'136","reported_seq":1132,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216917+0000","last_change":"2023-10-09T06:44:53.745637+0000","last_active":"2023-10-09T06:45:59.216917+0000","last_peered":"2023-10-09T06:45:59.216917+0000","last_clean":"2023-10-09T06:45:59.216917+0000","last_became_active":"2023-10-09T06:44:53.745062+0000","last_became_peered":"2023-10-09T06:44:53.745062+0000","last_unstale":"2023-10-09T06:45:59.216917+0000","last_undegraded":"2023-10-09T06:45:59.216917+0000","last_fullsized":"2023-10-09T06:45:59.216917+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":136,"log_dups_size":0,"ondisk_log_size":136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:44.644929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035738799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":627,"num_read_kb":2028,"num_write":136,"num_write_kb":1324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":2,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010361+0000","last_change":"2023-10-09T06:45:47.995934+0000","last_active":"2023-10-09T06:45:56.010361+0000","last_peered":"2023-10-09T06:45:56.010361+0000","last_clean":"2023-10-09T06:45:56.010361+0000","last_became_active":"2023-10-09T06:45:47.995425+0000","last_became_peered":"2023-10-09T06:45:47.995425+0000","last_unstale":"2023-10-09T06:45:56.010361+0000","last_undegraded":"2023-10-09T06:45:56.010361+0000","last_fullsized":"2023-10-09T06:45:56.010361+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:35.354961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":549,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216534+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:45:59.216534+0000","last_peered":"2023-10-09T06:45:59.216534+0000","last_clean":"2023-10-09T06:45:59.216534+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:45:59.216534+0000","last_undegraded":"2023-10-09T06:45:59.216534+0000","last_fullsized":"2023-10-09T06:45:59.216534+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":820,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010186+0000","last_change":"2023-10-09T06:45:47.987993+0000","last_active":"2023-10-09T06:45:56.010186+0000","last_peered":"2023-10-09T06:45:56.010186+0000","last_clean":"2023-10-09T06:45:56.010186+0000","last_became_active":"2023-10-09T06:45:47.987676+0000","last_became_peered":"2023-10-09T06:45:47.987676+0000","last_unstale":"2023-10-09T06:45:56.010186+0000","last_undegraded":"2023-10-09T06:45:56.010186+0000","last_fullsized":"2023-10-09T06:45:56.010186+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:55.451401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:55.451401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:58.778079+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"573'90","reported_seq":835,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010177+0000","last_change":"2023-10-09T06:45:47.998091+0000","last_active":"2023-10-09T06:45:56.010177+0000","last_peered":"2023-10-09T06:45:56.010177+0000","last_clean":"2023-10-09T06:45:56.010177+0000","last_became_active":"2023-10-09T06:45:47.997699+0000","last_became_peered":"2023-10-09T06:45:47.997699+0000","last_unstale":"2023-10-09T06:45:56.010177+0000","last_undegraded":"2023-10-09T06:45:56.010177+0000","last_fullsized":"2023-10-09T06:45:56.010177+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:48.882045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000225527,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":662,"num_read_kb":1785,"num_write":84,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":625,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428028+0000","last_change":"2023-10-09T06:44:57.095443+0000","last_active":"2023-10-09T06:45:56.428028+0000","last_peered":"2023-10-09T06:45:56.428028+0000","last_clean":"2023-10-09T06:45:56.428028+0000","last_became_active":"2023-10-09T06:44:57.094944+0000","last_became_peered":"2023-10-09T06:44:57.094944+0000","last_unstale":"2023-10-09T06:45:56.428028+0000","last_undegraded":"2023-10-09T06:45:56.428028+0000","last_fullsized":"2023-10-09T06:45:56.428028+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:16.144698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":579,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216879+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:45:59.216879+0000","last_peered":"2023-10-09T06:45:59.216879+0000","last_clean":"2023-10-09T06:45:59.216879+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:45:59.216879+0000","last_undegraded":"2023-10-09T06:45:59.216879+0000","last_fullsized":"2023-10-09T06:45:59.216879+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":986,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427455+0000","last_change":"2023-10-09T06:44:53.748889+0000","last_active":"2023-10-09T06:45:56.427455+0000","last_peered":"2023-10-09T06:45:56.427455+0000","last_clean":"2023-10-09T06:45:56.427455+0000","last_became_active":"2023-10-09T06:44:53.747630+0000","last_became_peered":"2023-10-09T06:44:53.747630+0000","last_unstale":"2023-10-09T06:45:56.427455+0000","last_undegraded":"2023-10-09T06:45:56.427455+0000","last_fullsized":"2023-10-09T06:45:56.427455+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:01.511893+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"573'157","reported_seq":1201,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425079+0000","last_change":"2023-10-09T06:45:29.893515+0000","last_active":"2023-10-09T06:45:56.425079+0000","last_peered":"2023-10-09T06:45:56.425079+0000","last_clean":"2023-10-09T06:45:56.425079+0000","last_became_active":"2023-10-09T06:45:29.892852+0000","last_became_peered":"2023-10-09T06:45:29.892852+0000","last_unstale":"2023-10-09T06:45:56.425079+0000","last_undegraded":"2023-10-09T06:45:56.425079+0000","last_fullsized":"2023-10-09T06:45:56.425079+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:56.372743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033522800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1047,"num_read_kb":2896,"num_write":141,"num_write_kb":1616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":16,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":623,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007363+0000","last_change":"2023-10-09T06:44:57.091601+0000","last_active":"2023-10-09T06:45:56.007363+0000","last_peered":"2023-10-09T06:45:56.007363+0000","last_clean":"2023-10-09T06:45:56.007363+0000","last_became_active":"2023-10-09T06:44:57.091261+0000","last_became_peered":"2023-10-09T06:44:57.091261+0000","last_unstale":"2023-10-09T06:45:56.007363+0000","last_undegraded":"2023-10-09T06:45:56.007363+0000","last_fullsized":"2023-10-09T06:45:56.007363+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:01.004445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":627,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008409+0000","last_change":"2023-10-09T06:44:51.051148+0000","last_active":"2023-10-09T06:45:56.008409+0000","last_peered":"2023-10-09T06:45:56.008409+0000","last_clean":"2023-10-09T06:45:56.008409+0000","last_became_active":"2023-10-09T06:44:51.050863+0000","last_became_peered":"2023-10-09T06:44:51.050863+0000","last_unstale":"2023-10-09T06:45:56.008409+0000","last_undegraded":"2023-10-09T06:45:56.008409+0000","last_fullsized":"2023-10-09T06:45:56.008409+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:21.910235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"573'99","reported_seq":617,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010249+0000","last_change":"2023-10-09T06:45:47.994011+0000","last_active":"2023-10-09T06:45:56.010249+0000","last_peered":"2023-10-09T06:45:56.010249+0000","last_clean":"2023-10-09T06:45:56.010249+0000","last_became_active":"2023-10-09T06:45:47.993560+0000","last_became_peered":"2023-10-09T06:45:47.993560+0000","last_unstale":"2023-10-09T06:45:56.010249+0000","last_undegraded":"2023-10-09T06:45:56.010249+0000","last_fullsized":"2023-10-09T06:45:56.010249+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:13.026969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076790500000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":163,"num_read_kb":437,"num_write":99,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":1007,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426911+0000","last_change":"2023-10-09T06:45:00.109701+0000","last_active":"2023-10-09T06:45:56.426911+0000","last_peered":"2023-10-09T06:45:56.426911+0000","last_clean":"2023-10-09T06:45:56.426911+0000","last_became_active":"2023-10-09T06:45:00.109209+0000","last_became_peered":"2023-10-09T06:45:00.109209+0000","last_unstale":"2023-10-09T06:45:56.426911+0000","last_undegraded":"2023-10-09T06:45:56.426911+0000","last_fullsized":"2023-10-09T06:45:56.426911+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:37.845254+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":337,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.214421+0000","last_change":"2023-10-09T06:44:57.099304+0000","last_active":"2023-10-09T06:45:59.214421+0000","last_peered":"2023-10-09T06:45:59.214421+0000","last_clean":"2023-10-09T06:45:59.214421+0000","last_became_active":"2023-10-09T06:44:57.098921+0000","last_became_peered":"2023-10-09T06:44:57.098921+0000","last_unstale":"2023-10-09T06:45:59.214421+0000","last_undegraded":"2023-10-09T06:45:59.214421+0000","last_fullsized":"2023-10-09T06:45:59.214421+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:02.502847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141055+0000","last_change":"2023-10-09T06:44:57.094991+0000","last_active":"2023-10-09T06:45:56.141055+0000","last_peered":"2023-10-09T06:45:56.141055+0000","last_clean":"2023-10-09T06:45:56.141055+0000","last_became_active":"2023-10-09T06:44:57.094638+0000","last_became_peered":"2023-10-09T06:44:57.094638+0000","last_unstale":"2023-10-09T06:45:56.141055+0000","last_undegraded":"2023-10-09T06:45:56.141055+0000","last_fullsized":"2023-10-09T06:45:56.141055+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:29.219217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":952,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216729+0000","last_change":"2023-10-09T06:45:00.111537+0000","last_active":"2023-10-09T06:45:59.216729+0000","last_peered":"2023-10-09T06:45:59.216729+0000","last_clean":"2023-10-09T06:45:59.216729+0000","last_became_active":"2023-10-09T06:45:00.111166+0000","last_became_peered":"2023-10-09T06:45:00.111166+0000","last_unstale":"2023-10-09T06:45:59.216729+0000","last_undegraded":"2023-10-09T06:45:59.216729+0000","last_fullsized":"2023-10-09T06:45:59.216729+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:51.491231+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"573'110","reported_seq":944,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427418+0000","last_change":"2023-10-09T06:44:53.748832+0000","last_active":"2023-10-09T06:45:56.427418+0000","last_peered":"2023-10-09T06:45:56.427418+0000","last_clean":"2023-10-09T06:45:56.427418+0000","last_became_active":"2023-10-09T06:44:53.748345+0000","last_became_peered":"2023-10-09T06:44:53.748345+0000","last_unstale":"2023-10-09T06:45:56.427418+0000","last_undegraded":"2023-10-09T06:45:56.427418+0000","last_fullsized":"2023-10-09T06:45:56.427418+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:08.273007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001926374,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":826,"num_read_kb":718,"num_write":101,"num_write_kb":58,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":10240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006380+0000","last_change":"2023-10-09T06:44:51.056475+0000","last_active":"2023-10-09T06:45:56.006380+0000","last_peered":"2023-10-09T06:45:56.006380+0000","last_clean":"2023-10-09T06:45:56.006380+0000","last_became_active":"2023-10-09T06:44:51.056165+0000","last_became_peered":"2023-10-09T06:44:51.056165+0000","last_unstale":"2023-10-09T06:45:56.006380+0000","last_undegraded":"2023-10-09T06:45:56.006380+0000","last_fullsized":"2023-10-09T06:45:56.006380+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:30.834127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006769+0000","last_change":"2023-10-09T06:44:51.051163+0000","last_active":"2023-10-09T06:45:56.006769+0000","last_peered":"2023-10-09T06:45:56.006769+0000","last_clean":"2023-10-09T06:45:56.006769+0000","last_became_active":"2023-10-09T06:44:51.050878+0000","last_became_peered":"2023-10-09T06:44:51.050878+0000","last_unstale":"2023-10-09T06:45:56.006769+0000","last_undegraded":"2023-10-09T06:45:56.006769+0000","last_fullsized":"2023-10-09T06:45:56.006769+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:18.212567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":650,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140283+0000","last_change":"2023-10-09T06:45:00.111587+0000","last_active":"2023-10-09T06:45:56.140283+0000","last_peered":"2023-10-09T06:45:56.140283+0000","last_clean":"2023-10-09T06:45:56.140283+0000","last_became_active":"2023-10-09T06:45:00.111294+0000","last_became_peered":"2023-10-09T06:45:00.111294+0000","last_unstale":"2023-10-09T06:45:56.140283+0000","last_undegraded":"2023-10-09T06:45:56.140283+0000","last_fullsized":"2023-10-09T06:45:56.140283+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:43.314983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:43.314983+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:03.930013+0000","scrub_duration":0.0021392379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"573'201","reported_seq":1226,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428252+0000","last_change":"2023-10-09T06:44:53.748266+0000","last_active":"2023-10-09T06:45:56.428252+0000","last_peered":"2023-10-09T06:45:56.428252+0000","last_clean":"2023-10-09T06:45:56.428252+0000","last_became_active":"2023-10-09T06:44:53.746859+0000","last_became_peered":"2023-10-09T06:44:53.746859+0000","last_unstale":"2023-10-09T06:45:56.428252+0000","last_undegraded":"2023-10-09T06:45:56.428252+0000","last_fullsized":"2023-10-09T06:45:56.428252+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":201,"log_dups_size":0,"ondisk_log_size":201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:12.464291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088493999999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1244,"num_read_kb":4126,"num_write":180,"num_write_kb":3151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":35,"num_keys_recovered":11,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":605,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216847+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:45:59.216847+0000","last_peered":"2023-10-09T06:45:59.216847+0000","last_clean":"2023-10-09T06:45:59.216847+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:45:59.216847+0000","last_undegraded":"2023-10-09T06:45:59.216847+0000","last_fullsized":"2023-10-09T06:45:59.216847+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006852+0000","last_change":"2023-10-09T06:44:51.048050+0000","last_active":"2023-10-09T06:45:56.006852+0000","last_peered":"2023-10-09T06:45:56.006852+0000","last_clean":"2023-10-09T06:45:56.006852+0000","last_became_active":"2023-10-09T06:44:51.047782+0000","last_became_peered":"2023-10-09T06:44:51.047782+0000","last_unstale":"2023-10-09T06:45:56.006852+0000","last_undegraded":"2023-10-09T06:45:56.006852+0000","last_fullsized":"2023-10-09T06:45:56.006852+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:57.140425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":967,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428261+0000","last_change":"2023-10-09T06:45:00.111875+0000","last_active":"2023-10-09T06:45:56.428261+0000","last_peered":"2023-10-09T06:45:56.428261+0000","last_clean":"2023-10-09T06:45:56.428261+0000","last_became_active":"2023-10-09T06:45:00.111520+0000","last_became_peered":"2023-10-09T06:45:00.111520+0000","last_unstale":"2023-10-09T06:45:56.428261+0000","last_undegraded":"2023-10-09T06:45:56.428261+0000","last_fullsized":"2023-10-09T06:45:56.428261+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:33.954026+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":365,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426369+0000","last_change":"2023-10-09T06:44:51.067000+0000","last_active":"2023-10-09T06:45:56.426369+0000","last_peered":"2023-10-09T06:45:56.426369+0000","last_clean":"2023-10-09T06:45:56.426369+0000","last_became_active":"2023-10-09T06:44:51.057845+0000","last_became_peered":"2023-10-09T06:44:51.057845+0000","last_unstale":"2023-10-09T06:45:56.426369+0000","last_undegraded":"2023-10-09T06:45:56.426369+0000","last_fullsized":"2023-10-09T06:45:56.426369+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:22.125999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"573'130","reported_seq":1177,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008092+0000","last_change":"2023-10-09T06:45:00.111169+0000","last_active":"2023-10-09T06:45:56.008092+0000","last_peered":"2023-10-09T06:45:56.008092+0000","last_clean":"2023-10-09T06:45:56.008092+0000","last_became_active":"2023-10-09T06:45:00.110746+0000","last_became_peered":"2023-10-09T06:45:00.110746+0000","last_unstale":"2023-10-09T06:45:56.008092+0000","last_undegraded":"2023-10-09T06:45:56.008092+0000","last_fullsized":"2023-10-09T06:45:56.008092+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:45.413094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058176600000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":817,"num_write":79,"num_write_kb":5162,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":346,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425491+0000","last_change":"2023-10-09T06:44:51.056174+0000","last_active":"2023-10-09T06:45:56.425491+0000","last_peered":"2023-10-09T06:45:56.425491+0000","last_clean":"2023-10-09T06:45:56.425491+0000","last_became_active":"2023-10-09T06:44:51.055160+0000","last_became_peered":"2023-10-09T06:44:51.055160+0000","last_unstale":"2023-10-09T06:45:56.425491+0000","last_undegraded":"2023-10-09T06:45:56.425491+0000","last_fullsized":"2023-10-09T06:45:56.425491+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:19.799793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":962,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215639+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:45:59.215639+0000","last_peered":"2023-10-09T06:45:59.215639+0000","last_clean":"2023-10-09T06:45:59.215639+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:45:59.215639+0000","last_undegraded":"2023-10-09T06:45:59.215639+0000","last_fullsized":"2023-10-09T06:45:59.215639+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"565'54","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141210+0000","last_change":"2023-10-09T06:44:57.106119+0000","last_active":"2023-10-09T06:45:56.141210+0000","last_peered":"2023-10-09T06:45:56.141210+0000","last_clean":"2023-10-09T06:45:56.141210+0000","last_became_active":"2023-10-09T06:44:57.094050+0000","last_became_peered":"2023-10-09T06:44:57.094050+0000","last_unstale":"2023-10-09T06:45:56.141210+0000","last_undegraded":"2023-10-09T06:45:56.141210+0000","last_fullsized":"2023-10-09T06:45:56.141210+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:20.716304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000476019,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":438,"num_read_kb":386,"num_write":26,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006679+0000","last_change":"2023-10-09T06:44:51.078337+0000","last_active":"2023-10-09T06:45:56.006679+0000","last_peered":"2023-10-09T06:45:56.006679+0000","last_clean":"2023-10-09T06:45:56.006679+0000","last_became_active":"2023-10-09T06:44:51.049587+0000","last_became_peered":"2023-10-09T06:44:51.049587+0000","last_unstale":"2023-10-09T06:45:56.006679+0000","last_undegraded":"2023-10-09T06:45:56.006679+0000","last_fullsized":"2023-10-09T06:45:56.006679+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:19.138342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":607,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427422+0000","last_change":"2023-10-09T06:44:57.097133+0000","last_active":"2023-10-09T06:45:56.427422+0000","last_peered":"2023-10-09T06:45:56.427422+0000","last_clean":"2023-10-09T06:45:56.427422+0000","last_became_active":"2023-10-09T06:44:57.096732+0000","last_became_peered":"2023-10-09T06:44:57.096732+0000","last_unstale":"2023-10-09T06:45:56.427422+0000","last_undegraded":"2023-10-09T06:45:56.427422+0000","last_fullsized":"2023-10-09T06:45:56.427422+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:03.096466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":972,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425176+0000","last_change":"2023-10-09T06:45:00.108827+0000","last_active":"2023-10-09T06:45:56.425176+0000","last_peered":"2023-10-09T06:45:56.425176+0000","last_clean":"2023-10-09T06:45:56.425176+0000","last_became_active":"2023-10-09T06:45:00.108469+0000","last_became_peered":"2023-10-09T06:45:00.108469+0000","last_unstale":"2023-10-09T06:45:56.425176+0000","last_undegraded":"2023-10-09T06:45:56.425176+0000","last_fullsized":"2023-10-09T06:45:56.425176+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:19.934169+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006939+0000","last_change":"2023-10-09T06:44:51.047675+0000","last_active":"2023-10-09T06:45:56.006939+0000","last_peered":"2023-10-09T06:45:56.006939+0000","last_clean":"2023-10-09T06:45:56.006939+0000","last_became_active":"2023-10-09T06:44:51.047410+0000","last_became_peered":"2023-10-09T06:44:51.047410+0000","last_unstale":"2023-10-09T06:45:56.006939+0000","last_undegraded":"2023-10-09T06:45:56.006939+0000","last_fullsized":"2023-10-09T06:45:56.006939+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:51.476378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":352,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425439+0000","last_change":"2023-10-09T06:45:29.893646+0000","last_active":"2023-10-09T06:45:56.425439+0000","last_peered":"2023-10-09T06:45:56.425439+0000","last_clean":"2023-10-09T06:45:56.425439+0000","last_became_active":"2023-10-09T06:45:29.893405+0000","last_became_peered":"2023-10-09T06:45:29.893405+0000","last_unstale":"2023-10-09T06:45:56.425439+0000","last_undegraded":"2023-10-09T06:45:56.425439+0000","last_fullsized":"2023-10-09T06:45:56.425439+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:21.383885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":853,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215485+0000","last_change":"2023-10-09T06:45:00.109405+0000","last_active":"2023-10-09T06:45:59.215485+0000","last_peered":"2023-10-09T06:45:59.215485+0000","last_clean":"2023-10-09T06:45:59.215485+0000","last_became_active":"2023-10-09T06:45:00.109109+0000","last_became_peered":"2023-10-09T06:45:00.109109+0000","last_unstale":"2023-10-09T06:45:59.215485+0000","last_undegraded":"2023-10-09T06:45:59.215485+0000","last_fullsized":"2023-10-09T06:45:59.215485+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:09.455204+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":165,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010415+0000","last_change":"2023-10-09T06:45:47.998014+0000","last_active":"2023-10-09T06:45:56.010415+0000","last_peered":"2023-10-09T06:45:56.010415+0000","last_clean":"2023-10-09T06:45:56.010415+0000","last_became_active":"2023-10-09T06:45:47.997684+0000","last_became_peered":"2023-10-09T06:45:47.997684+0000","last_unstale":"2023-10-09T06:45:56.010415+0000","last_undegraded":"2023-10-09T06:45:56.010415+0000","last_fullsized":"2023-10-09T06:45:56.010415+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:11.858074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":611,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427961+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:45:56.427961+0000","last_peered":"2023-10-09T06:45:56.427961+0000","last_clean":"2023-10-09T06:45:56.427961+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:45:56.427961+0000","last_undegraded":"2023-10-09T06:45:56.427961+0000","last_fullsized":"2023-10-09T06:45:56.427961+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":165,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010511+0000","last_change":"2023-10-09T06:45:47.995652+0000","last_active":"2023-10-09T06:45:56.010511+0000","last_peered":"2023-10-09T06:45:56.010511+0000","last_clean":"2023-10-09T06:45:56.010511+0000","last_became_active":"2023-10-09T06:45:47.994380+0000","last_became_peered":"2023-10-09T06:45:47.994380+0000","last_unstale":"2023-10-09T06:45:56.010511+0000","last_undegraded":"2023-10-09T06:45:56.010511+0000","last_fullsized":"2023-10-09T06:45:56.010511+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:12.797577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":641,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427336+0000","last_change":"2023-10-09T06:44:51.046177+0000","last_active":"2023-10-09T06:45:56.427336+0000","last_peered":"2023-10-09T06:45:56.427336+0000","last_clean":"2023-10-09T06:45:56.427336+0000","last_became_active":"2023-10-09T06:44:51.045909+0000","last_became_peered":"2023-10-09T06:44:51.045909+0000","last_unstale":"2023-10-09T06:45:56.427336+0000","last_undegraded":"2023-10-09T06:45:56.427336+0000","last_fullsized":"2023-10-09T06:45:56.427336+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:06.637385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":1000,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426926+0000","last_change":"2023-10-09T06:45:00.110332+0000","last_active":"2023-10-09T06:45:56.426926+0000","last_peered":"2023-10-09T06:45:56.426926+0000","last_clean":"2023-10-09T06:45:56.426926+0000","last_became_active":"2023-10-09T06:45:00.109980+0000","last_became_peered":"2023-10-09T06:45:00.109980+0000","last_unstale":"2023-10-09T06:45:56.426926+0000","last_undegraded":"2023-10-09T06:45:56.426926+0000","last_fullsized":"2023-10-09T06:45:56.426926+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:04.079134+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":328,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426255+0000","last_change":"2023-10-09T06:44:57.093892+0000","last_active":"2023-10-09T06:45:56.426255+0000","last_peered":"2023-10-09T06:45:56.426255+0000","last_clean":"2023-10-09T06:45:56.426255+0000","last_became_active":"2023-10-09T06:44:57.093537+0000","last_became_peered":"2023-10-09T06:44:57.093537+0000","last_unstale":"2023-10-09T06:45:56.426255+0000","last_undegraded":"2023-10-09T06:45:56.426255+0000","last_fullsized":"2023-10-09T06:45:56.426255+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:06.715364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":344,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425699+0000","last_change":"2023-10-09T06:44:51.058622+0000","last_active":"2023-10-09T06:45:56.425699+0000","last_peered":"2023-10-09T06:45:56.425699+0000","last_clean":"2023-10-09T06:45:56.425699+0000","last_became_active":"2023-10-09T06:44:51.058147+0000","last_became_peered":"2023-10-09T06:44:51.058147+0000","last_unstale":"2023-10-09T06:45:56.425699+0000","last_undegraded":"2023-10-09T06:45:56.425699+0000","last_fullsized":"2023-10-09T06:45:56.425699+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:04.760476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":729,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007916+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:45:56.007916+0000","last_peered":"2023-10-09T06:45:56.007916+0000","last_clean":"2023-10-09T06:45:56.007916+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:45:56.007916+0000","last_undegraded":"2023-10-09T06:45:56.007916+0000","last_fullsized":"2023-10-09T06:45:56.007916+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":43,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007088+0000","last_change":"2023-10-09T06:44:51.072116+0000","last_active":"2023-10-09T06:45:56.007088+0000","last_peered":"2023-10-09T06:45:56.007088+0000","last_clean":"2023-10-09T06:45:56.007088+0000","last_became_active":"2023-10-09T06:44:51.047845+0000","last_became_peered":"2023-10-09T06:44:51.047845+0000","last_unstale":"2023-10-09T06:45:56.007088+0000","last_undegraded":"2023-10-09T06:45:56.007088+0000","last_fullsized":"2023-10-09T06:45:56.007088+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:24.262662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":584,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427538+0000","last_change":"2023-10-09T06:44:57.097289+0000","last_active":"2023-10-09T06:45:56.427538+0000","last_peered":"2023-10-09T06:45:56.427538+0000","last_clean":"2023-10-09T06:45:56.427538+0000","last_became_active":"2023-10-09T06:44:57.096952+0000","last_became_peered":"2023-10-09T06:44:57.096952+0000","last_unstale":"2023-10-09T06:45:56.427538+0000","last_undegraded":"2023-10-09T06:45:56.427538+0000","last_fullsized":"2023-10-09T06:45:56.427538+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:45.990363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":20394,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5976,"num_read_kb":13193,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":4204614,"num_keys_recovered":13,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234591,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498744,"kb_used_data":5936,"kb_used_omap":0,"kb_used_meta":1492800,"kb_avail":92246408,"statfs":{"total":95995035648,"available":94460321792,"internally_reserved":0,"allocated":6078464,"data_stored":4929714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1528627200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":2.9510000000000001,"15min":3.0150000000000001},"min":{"1min":0.28299999999999997,"5min":0.216,"15min":0.216},"max":{"1min":0.70499999999999996,"5min":76.727999999999994,"15min":196.28899999999999},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":2.9900000000000002,"15min":3.0739999999999998},"min":{"1min":0.34200000000000003,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":0.71899999999999997,"5min":76.712999999999994,"15min":196.333},"last":0.629}]},{"osd":1,"last update":"Mon Oct 9 06:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":3.113,"15min":4.1719999999999997},"min":{"1min":0.36599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.627,"5min":169.51900000000001,"15min":208.626},"last":0.504},{"interface":"front","average":{"1min":0.53400000000000003,"5min":3.1629999999999998,"15min":3.5099999999999998},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":0.98399999999999999,"5min":169.101,"15min":181.24199999999999},"last":0.60999999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.252,"15min":4.5949999999999998},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.627,"5min":30.117000000000001,"15min":228.24000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.597,"15min":4.6980000000000004},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.30299999999999999},"max":{"1min":1.0189999999999999,"5min":30.245999999999999,"15min":228.291},"last":0.52400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":2.5579999999999998,"15min":3.8239999999999998},"min":{"1min":0.379,"5min":0.308,"15min":0.27000000000000002},"max":{"1min":0.81100000000000005,"5min":59.847000000000001,"15min":175.40600000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.6800000000000002,"15min":3.7970000000000002},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":0.77400000000000002,"5min":59.698999999999998,"15min":175.30099999999999},"last":0.67700000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":2.8359999999999999,"15min":2.746},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":0.79400000000000004,"5min":130.774,"15min":225.041},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":2.6669999999999998,"15min":2.5779999999999998},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.84599999999999997,"5min":130.72999999999999,"15min":225.244},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.59699999999999998,"15min":2.1949999999999998},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26700000000000002},"max":{"1min":0.67700000000000005,"5min":4.8520000000000003,"15min":219.923},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.747,"15min":2.2200000000000002},"min":{"1min":0.39700000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.751,"5min":3.4409999999999998,"15min":219.86000000000001},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.859,"15min":3.0110000000000001},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":0.82599999999999996,"5min":138.477,"15min":222.179},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":2.8809999999999998,"15min":2.944},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81299999999999994,"5min":135.244,"15min":223.261},"last":0.40699999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496227,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518500,"kb_used_data":10076,"kb_used_omap":0,"kb_used_meta":1508416,"kb_avail":92226652,"statfs":{"total":95995035648,"available":94440091648,"internally_reserved":0,"allocated":10317824,"data_stored":9188295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1544617984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":2.1160000000000001,"15min":2.5990000000000002},"min":{"1min":0.34999999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.77800000000000002,"5min":45.613999999999997,"15min":75.061999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.2679999999999998,"15min":3.5569999999999999},"min":{"1min":0.36699999999999999,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":0.89200000000000002,"5min":45.646999999999998,"15min":212.096},"last":0.625}]},{"osd":1,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.458,"15min":3.1779999999999999},"min":{"1min":0.34599999999999997,"5min":0.27000000000000002,"15min":0.20899999999999999},"max":{"1min":0.94399999999999995,"5min":40.067,"15min":219.93199999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.505,"5min":1.3149999999999999,"15min":3.157},"min":{"1min":0.36499999999999999,"5min":0.27800000000000002,"15min":0.222},"max":{"1min":0.65100000000000002,"5min":40.085999999999999,"15min":219.94800000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.2170000000000001,"15min":3.4500000000000002},"min":{"1min":0.379,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":0.97199999999999998,"5min":7.4980000000000002,"15min":185.654},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.1870000000000001,"15min":3.3679999999999999},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":7.4829999999999997,"15min":185.67699999999999},"last":0.55300000000000005}]},{"osd":3,"last update":"Mon Oct 9 06:45:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":3.609,"15min":3.7160000000000002},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.8450000000000002,"15min":3.6930000000000001},"min":{"1min":0.34499999999999997,"5min":0.34100000000000003,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.67300000000000004,"15min":1.282},"min":{"1min":0.32500000000000001,"5min":0.25800000000000001,"15min":0.23799999999999999},"max":{"1min":0.77200000000000002,"5min":5.6890000000000001,"15min":86.171999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.72499999999999998,"15min":2.0419999999999998},"min":{"1min":0.36399999999999999,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":0.81499999999999995,"5min":8.2989999999999995,"15min":212.03399999999999},"last":0.53400000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.876,"15min":1.155},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.27000000000000002},"max":{"1min":0.79600000000000004,"5min":10.435,"15min":10.435},"last":0.441},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.76400000000000001,"15min":1.282},"min":{"1min":0.34899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.78600000000000003,"5min":8.3369999999999997,"15min":12.244999999999999},"last":0.498}]},{"osd":7,"last update":"Mon Oct 9 06:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.887,"15min":1.9610000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.27000000000000002},"max":{"1min":0.76900000000000002,"5min":109.83199999999999,"15min":165.37},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.9710000000000001,"15min":2.202},"min":{"1min":0.46899999999999997,"5min":0.34300000000000003,"15min":0.29099999999999998},"max":{"1min":0.76900000000000002,"5min":109.911,"15min":165.40199999999999},"last":0.58599999999999997}]}]},{"osd":1,"up_from":110,"seq":472446402965,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143720,"kb_used_data":11296,"kb_used_omap":4,"kb_used_meta":1132411,"kb_avail":92601432,"statfs":{"total":95995035648,"available":94823866368,"internally_reserved":0,"allocated":11567104,"data_stored":9043533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1159589774},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":4.1879999999999997,"15min":3.6629999999999998},"min":{"1min":0.221,"5min":0.21299999999999999,"15min":0.182},"max":{"1min":0.45300000000000001,"5min":114.97799999999999,"15min":143.465},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.34399999999999997,"5min":4.7789999999999999,"15min":4.048},"min":{"1min":0.26100000000000001,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":0.51100000000000001,"5min":114.934,"15min":143.66399999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.88900000000000001,"15min":2.0350000000000001},"min":{"1min":0.25600000000000001,"5min":0.193,"15min":0.184},"max":{"1min":0.52800000000000002,"5min":8.0129999999999999,"15min":138.452},"last":0.312},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1240000000000001,"15min":2.6720000000000002},"min":{"1min":0.29699999999999999,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":1.147,"5min":20.085000000000001,"15min":138.47499999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:45:40 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":2.2850000000000001,"15min":2.4369999999999998},"min":{"1min":0.253,"5min":0.25,"15min":0.219},"max":{"1min":0.48199999999999998,"5min":27.655999999999999,"15min":27.655999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.38,"5min":1.9239999999999999,"15min":1.7629999999999999},"min":{"1min":0.30199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.55600000000000005,"5min":27.745999999999999,"15min":27.745999999999999},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":2.7850000000000001,"15min":2.9630000000000001},"min":{"1min":0.40100000000000002,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":0.91600000000000004,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.504},{"interface":"front","average":{"1min":0.621,"5min":2.7370000000000001,"15min":2.7090000000000001},"min":{"1min":0.44500000000000001,"5min":0.28799999999999998,"15min":0.28499999999999998},"max":{"1min":1.0489999999999999,"5min":180.70599999999999,"15min":180.70599999999999},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.93899999999999995,"15min":2.177},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":0.747,"5min":7.9420000000000002,"15min":119.408},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.2310000000000001,"15min":2.3959999999999999},"min":{"1min":0.39800000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.89000000000000001,"5min":9.6820000000000004,"15min":119.483},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":4.242,"15min":3.4049999999999998},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.84099999999999997,"5min":183.12100000000001,"15min":183.12100000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":4.2919999999999998,"15min":3.7069999999999999},"min":{"1min":0.371,"5min":0.36699999999999999,"15min":0.316},"max":{"1min":0.83299999999999996,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":2.8210000000000002,"15min":3.4359999999999999},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.34300000000000003},"max":{"1min":0.879,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":2.4449999999999998,"15min":2.948},"min":{"1min":0.434,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":0.875,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.51200000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024398,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1201040,"kb_used_data":8008,"kb_used_omap":5,"kb_used_meta":1193018,"kb_avail":92544112,"statfs":{"total":95995035648,"available":94765170688,"internally_reserved":0,"allocated":8200192,"data_stored":5236884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1221651066},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":6.0910000000000002,"15min":4.867},"min":{"1min":0.248,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":0.48199999999999998,"5min":263.86799999999999,"15min":263.86799999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.41499999999999998,"5min":5.8929999999999998,"15min":5.0590000000000002},"min":{"1min":0.27400000000000002,"5min":0.20200000000000001,"15min":0.187},"max":{"1min":0.57599999999999996,"5min":263.51999999999998,"15min":263.51999999999998},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":3.2250000000000001,"15min":3.8039999999999998},"min":{"1min":0.28100000000000003,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":0.52600000000000002,"5min":147.61099999999999,"15min":147.61099999999999},"last":0.312},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.4369999999999998,"15min":4.04},"min":{"1min":0.27300000000000002,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":0.502,"5min":150.12700000000001,"15min":150.12700000000001},"last":0.379}]},{"osd":3,"last update":"Mon Oct 9 06:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.8360000000000001,"15min":3.5720000000000001},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.53800000000000003,"5min":48.216000000000001,"15min":187.172},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.9690000000000001,"15min":3.6240000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.19800000000000001},"max":{"1min":0.64500000000000002,"5min":48.261000000000003,"15min":194.851},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.502,"15min":2.9809999999999999},"min":{"1min":0.432,"5min":0.316,"15min":0.28199999999999997},"max":{"1min":4.3140000000000001,"5min":14.183,"15min":200.654},"last":0.5},{"interface":"front","average":{"1min":0.78300000000000003,"5min":1.405,"15min":3.3010000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":4.3010000000000002,"5min":8.2850000000000001,"15min":200.86600000000001},"last":0.46800000000000003}]},{"osd":5,"last update":"Mon Oct 9 06:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.587,"15min":1.9630000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.24299999999999999},"max":{"1min":4.2510000000000003,"5min":14.263,"15min":80.885999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.411,"15min":2.0129999999999999},"min":{"1min":0.42299999999999999,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":4.2610000000000001,"5min":8.4019999999999992,"15min":80.438999999999993},"last":0.65200000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.407,"15min":2.262},"min":{"1min":0.36799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.1609999999999996,"5min":8.4309999999999992,"15min":118.056},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.571,"15min":2.1419999999999999},"min":{"1min":0.45300000000000001,"5min":0.35799999999999998,"15min":0.29899999999999999},"max":{"1min":4.1900000000000004,"5min":14.308999999999999,"15min":118.274},"last":0.47399999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.4910000000000001,"15min":1.8779999999999999},"min":{"1min":0.42899999999999999,"5min":0.40400000000000003,"15min":0.35699999999999998},"max":{"1min":4.2050000000000001,"5min":8.2490000000000006,"15min":22.879000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.83399999999999996,"5min":1.5529999999999999,"15min":2.1469999999999998},"min":{"1min":0.443,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.2270000000000003,"5min":8.4570000000000007,"15min":22.683},"last":0.67000000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814267,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078796,"kb_used_data":7364,"kb_used_omap":7,"kb_used_meta":1071416,"kb_avail":92666356,"statfs":{"total":95995035648,"available":94890348544,"internally_reserved":0,"allocated":7540736,"data_stored":5099456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1097130065},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":3.4169999999999998,"15min":6.0629999999999997},"min":{"1min":0.19600000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.46700000000000003,"5min":89.287000000000006,"15min":214.595},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.374,"5min":3.206,"15min":5.6859999999999999},"min":{"1min":0.27900000000000003,"5min":0.221,"15min":0.17000000000000001},"max":{"1min":0.502,"5min":89.225999999999999,"15min":214.94300000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":6.4500000000000002,"15min":5.218},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.41999999999999998,"5min":299.988,"15min":299.988},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.35999999999999999,"5min":6.694,"15min":5.1849999999999996},"min":{"1min":0.245,"5min":0.219,"15min":0.183},"max":{"1min":0.46700000000000003,"5min":300.30500000000001,"15min":300.30500000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":3.569,"15min":3.4460000000000002},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.53000000000000003,"5min":74.317999999999998,"15min":79.566000000000003},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":3.468,"15min":3.5129999999999999},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.997,"5min":74.355999999999995,"15min":79.614000000000004},"last":0.378}]},{"osd":4,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85699999999999998,"15min":1.9730000000000001},"min":{"1min":0.45900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.216,"5min":9.5679999999999996,"15min":79.643000000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.94299999999999995,"15min":2.1640000000000001},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":1.228,"5min":7.0910000000000002,"15min":82.872},"last":0.81799999999999995}]},{"osd":5,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.93400000000000005,"15min":1.9690000000000001},"min":{"1min":0.42699999999999999,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":0.73599999999999999,"5min":10.914999999999999,"15min":82.834999999999994},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.018,"15min":2.1030000000000002},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":0.65100000000000002,"5min":11.481999999999999,"15min":82.921999999999997},"last":0.59799999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.95699999999999996,"15min":2.7589999999999999},"min":{"1min":0.42399999999999999,"5min":0.34000000000000002,"15min":0.313},"max":{"1min":1.367,"5min":7.1669999999999998,"15min":123.47799999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.056,"15min":2.9169999999999998},"min":{"1min":0.41299999999999998,"5min":0.36799999999999999,"15min":0.313},"max":{"1min":1.2030000000000001,"5min":6.3689999999999998,"15min":122.517},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":3.0800000000000001,"15min":3.1299999999999999},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":1.143,"5min":132.06399999999999,"15min":132.06399999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":3.056,"15min":2.7589999999999999},"min":{"1min":0.38800000000000001,"5min":0.35899999999999999,"15min":0.35499999999999998},"max":{"1min":0.66800000000000004,"5min":132.196,"15min":132.196},"last":0.755}]}]},{"osd":3,"up_from":149,"seq":639950127464,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":948364,"kb_used_data":7300,"kb_used_omap":2,"kb_used_meta":941053,"kb_avail":92796788,"statfs":{"total":95995035648,"available":95023910912,"internally_reserved":0,"allocated":7475200,"data_stored":5153314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":963638325},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":3.431,"15min":4.0439999999999996},"min":{"1min":0.24199999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.58399999999999996,"5min":95.281999999999996,"15min":161.821},"last":0.318},{"interface":"front","average":{"1min":0.35799999999999998,"5min":4.1920000000000002,"15min":4.1369999999999996},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":0.47999999999999998,"5min":98.572000000000003,"15min":163.52000000000001},"last":0.33400000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":3.5419999999999998,"15min":4.1440000000000001},"min":{"1min":0.27000000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.54600000000000004,"5min":102.468,"15min":184.012},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.379,"5min":3.5390000000000001,"15min":4.532},"min":{"1min":0.255,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.66800000000000004,"5min":102.58,"15min":184.703},"last":0.36399999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.597,"15min":3.6909999999999998},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.59499999999999997,"5min":15.807,"15min":182.44900000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2829999999999999,"15min":3.79},"min":{"1min":0.27700000000000002,"5min":0.22,"15min":0.219},"max":{"1min":0.70399999999999996,"5min":14.9,"15min":182.56399999999999},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.081,"15min":2.9900000000000002},"min":{"1min":0.44400000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79500000000000004,"5min":9.7390000000000008,"15min":178.22200000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.248,"15min":3.0630000000000002},"min":{"1min":0.39400000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.79400000000000004,"5min":7.7999999999999998,"15min":180.43600000000001},"last":0.59299999999999997}]},{"osd":5,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.3440000000000001,"15min":3.4220000000000002},"min":{"1min":0.433,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80600000000000005,"5min":7.8380000000000001,"15min":177.56299999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.4890000000000001,"15min":2.9790000000000001},"min":{"1min":0.45800000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.83599999999999997,"5min":15.743,"15min":177.59399999999999},"last":0.67800000000000005}]},{"osd":6,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.3959999999999999,"15min":1.9870000000000001},"min":{"1min":0.41499999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82499999999999996,"5min":7.3159999999999998,"15min":48.270000000000003},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4910000000000001,"15min":1.804},"min":{"1min":0.434,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":0.76000000000000001,"5min":15.69,"15min":48.155000000000001},"last":0.72099999999999997}]},{"osd":7,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.9750000000000001,"15min":2.1110000000000002},"min":{"1min":0.45400000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.85099999999999998,"5min":99.248000000000005,"15min":99.248000000000005},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.1859999999999999,"15min":2.5579999999999998},"min":{"1min":0.39900000000000002,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":0.86299999999999999,"5min":99.457999999999998,"15min":99.457999999999998},"last":0.66600000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019500,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493416,"kb_used_data":9760,"kb_used_omap":0,"kb_used_meta":1483648,"kb_avail":92251736,"statfs":{"total":95995035648,"available":94465777664,"internally_reserved":0,"allocated":9994240,"data_stored":8860973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519255552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":4.0090000000000003,"15min":6.7800000000000002},"min":{"1min":0.35299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.75600000000000001,"5min":108.01300000000001,"15min":287.58800000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":4.04,"15min":6.9470000000000001},"min":{"1min":0.32600000000000001,"5min":0.28199999999999997,"15min":0.23200000000000001},"max":{"1min":0.72499999999999998,"5min":107.96299999999999,"15min":287.57100000000003},"last":0.57299999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":1.8999999999999999,"15min":3.0129999999999999},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.82099999999999995,"5min":69.644000000000005,"15min":163.392},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":2.0430000000000001,"15min":3.1070000000000002},"min":{"1min":0.35099999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.92800000000000005,"5min":69.572000000000003,"15min":163.42400000000001},"last":0.50600000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.98899999999999999,"15min":3.968},"min":{"1min":0.35199999999999998,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":0.77300000000000002,"5min":6.6879999999999997,"15min":175.97900000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.099,"15min":4.0839999999999996},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77100000000000002,"5min":6.0110000000000001,"15min":175.875},"last":0.64400000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.839,"15min":4.3360000000000003},"min":{"1min":0.41899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.745,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.624,"5min":2.8050000000000002,"15min":4.1600000000000001},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.79800000000000004,"5min":112.21899999999999,"15min":161.934},"last":0.63100000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.627,"15min":0.84699999999999998},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.252},"max":{"1min":0.78900000000000003,"5min":6.7869999999999999,"15min":9.3979999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77700000000000002,"15min":0.97999999999999998},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":0.91200000000000003,"5min":5.9820000000000002,"15min":9.5009999999999994},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Oct 9 06:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.6100000000000001,"15min":1.679},"min":{"1min":0.439,"5min":0.27300000000000002,"15min":0.26500000000000001},"max":{"1min":0.78900000000000003,"5min":80.549000000000007,"15min":115.70999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.6040000000000001,"15min":1.7949999999999999},"min":{"1min":0.41299999999999998,"5min":0.32200000000000001,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":83.218000000000004,"15min":115.68600000000001},"last":0.66100000000000003}]},{"osd":7,"last update":"Mon Oct 9 06:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.514,"15min":1.8300000000000001},"min":{"1min":0.443,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.80400000000000005,"5min":76.081000000000003,"15min":106.801},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.526,"15min":1.944},"min":{"1min":0.49099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82599999999999996,"5min":76.283000000000001,"15min":106.827},"last":0.56100000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757865,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485412,"kb_used_data":5916,"kb_used_omap":0,"kb_used_meta":1479488,"kb_avail":92259740,"statfs":{"total":95995035648,"available":94473973760,"internally_reserved":0,"allocated":6057984,"data_stored":4925084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514995712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.7769999999999999,"15min":5.2519999999999998},"min":{"1min":0.34000000000000002,"5min":0.23200000000000001,"15min":0.20799999999999999},"max":{"1min":0.70899999999999996,"5min":62.884999999999998,"15min":294.89699999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.905,"15min":5.5250000000000004},"min":{"1min":0.35099999999999998,"5min":0.25900000000000001,"15min":0.215},"max":{"1min":0.78500000000000003,"5min":62.195,"15min":294.30399999999997},"last":0.48099999999999998}]},{"osd":1,"last update":"Mon Oct 9 06:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.7430000000000001,"15min":6.1150000000000002},"min":{"1min":0.379,"5min":0.253,"15min":0.22},"max":{"1min":0.78800000000000003,"5min":69.287999999999997,"15min":188.34100000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.499,"5min":1.7490000000000001,"15min":6.0490000000000004},"min":{"1min":0.371,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":67.245999999999995,"15min":188.839},"last":0.60099999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":4.1500000000000004,"15min":4.5970000000000004},"min":{"1min":0.375,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":0.70899999999999996,"5min":145.20400000000001,"15min":145.20400000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":4.1660000000000004,"15min":4.5289999999999999},"min":{"1min":0.41699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.72299999999999998,"5min":145.71899999999999,"15min":145.71899999999999},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":2.1829999999999998,"15min":4.2380000000000004},"min":{"1min":0.39400000000000002,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":0.754,"5min":63.578000000000003,"15min":220.41800000000001},"last":0.754},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.9890000000000001,"15min":3.9929999999999999},"min":{"1min":0.5,"5min":0.39700000000000002,"15min":0.29299999999999998},"max":{"1min":0.81799999999999995,"5min":62.113999999999997,"15min":220.45500000000001},"last":0.68500000000000005}]},{"osd":4,"last update":"Mon Oct 9 06:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.5860000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.71799999999999997,"5min":74.725999999999999,"15min":79.043000000000006},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.6040000000000001,"15min":1.613},"min":{"1min":0.36599999999999999,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":1.0880000000000001,"5min":74.777000000000001,"15min":79.165999999999997},"last":0.58699999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.788,"15min":1.6459999999999999},"min":{"1min":0.45900000000000002,"5min":0.29999999999999999,"15min":0.251},"max":{"1min":0.76900000000000002,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.754,"15min":1.6890000000000001},"min":{"1min":0.31900000000000001,"5min":0.29899999999999999,"15min":0.26600000000000001},"max":{"1min":0.76000000000000001,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.85499999999999998,"15min":1.3759999999999999},"min":{"1min":0.44500000000000001,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":0.84199999999999997,"5min":5.4729999999999999,"15min":78.890000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.79500000000000004,"15min":1.2370000000000001},"min":{"1min":0.41799999999999998,"5min":0.317,"15min":0.28599999999999998},"max":{"1min":1.167,"5min":3.359,"15min":79.138999999999996},"last":0.72599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:46:34.226 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T06:46:34.606 INFO:teuthology.orchestra.run.smithi107.stderr:Inferring config /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T06:46:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:35 smithi107 ceph-mon[212750]: from='client.38370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:35.401 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:35 smithi107 ceph-mon[212750]: pgmap v1828: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:35 smithi107 ceph-mon[220921]: from='client.38370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:35.401 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:35 smithi107 ceph-mon[220921]: pgmap v1828: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:35 smithi150 ceph-mon[194178]: from='client.38370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:35.654 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:35 smithi150 ceph-mon[194178]: pgmap v1828: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:36.294 INFO:teuthology.orchestra.run.smithi107.stdout: 2023-10-09T06:46:36.296 INFO:teuthology.orchestra.run.smithi107.stderr:dumped all 2023-10-09T06:46:36.933 INFO:teuthology.orchestra.run.smithi107.stdout:{"pg_ready":true,"pg_map":{"version":1829,"stamp":"2023-10-09T06:46:35.709212+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5976,"num_read_kb":13193,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":5535728,"num_keys_recovered":23,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6023,"ondisk_log_size":6023,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10367992,"kb_used_data":65656,"kb_used_omap":20,"kb_used_meta":10302251,"kb_avail":739593224,"statfs":{"total":767960285184,"available":757343461376,"internally_reserved":0,"allocated":67231744,"data_stored":52437253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":10549505678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002616"},"pg_stats":[{"pgid":"1.18","version":"77'161","reported_seq":848,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010133+0000","last_change":"2023-10-09T06:45:47.993498+0000","last_active":"2023-10-09T06:45:56.010133+0000","last_peered":"2023-10-09T06:45:56.010133+0000","last_clean":"2023-10-09T06:45:56.010133+0000","last_became_active":"2023-10-09T06:45:47.993103+0000","last_became_peered":"2023-10-09T06:45:47.993103+0000","last_unstale":"2023-10-09T06:45:56.010133+0000","last_undegraded":"2023-10-09T06:45:56.010133+0000","last_fullsized":"2023-10-09T06:45:56.010133+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:37.631015+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:37.631015+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:40:06.971585+0000","scrub_duration":0.002097706,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":583,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427278+0000","last_change":"2023-10-09T06:15:11.882200+0000","last_active":"2023-10-09T06:45:56.427278+0000","last_peered":"2023-10-09T06:45:56.427278+0000","last_clean":"2023-10-09T06:45:56.427278+0000","last_became_active":"2023-10-09T06:15:11.881972+0000","last_became_peered":"2023-10-09T06:15:11.881972+0000","last_unstale":"2023-10-09T06:45:56.427278+0000","last_undegraded":"2023-10-09T06:45:56.427278+0000","last_fullsized":"2023-10-09T06:45:56.427278+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:03:03.864224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":308,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426140+0000","last_change":"2023-10-09T06:27:36.661148+0000","last_active":"2023-10-09T06:45:56.426140+0000","last_peered":"2023-10-09T06:45:56.426140+0000","last_clean":"2023-10-09T06:45:56.426140+0000","last_became_active":"2023-10-09T06:27:36.660848+0000","last_became_peered":"2023-10-09T06:27:36.660848+0000","last_unstale":"2023-10-09T06:45:56.426140+0000","last_undegraded":"2023-10-09T06:45:56.426140+0000","last_fullsized":"2023-10-09T06:45:56.426140+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:35:56.147072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"77'161","reported_seq":805,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010599+0000","last_change":"2023-10-09T06:45:47.995274+0000","last_active":"2023-10-09T06:45:56.010599+0000","last_peered":"2023-10-09T06:45:56.010599+0000","last_clean":"2023-10-09T06:45:56.010599+0000","last_became_active":"2023-10-09T06:45:47.994500+0000","last_became_peered":"2023-10-09T06:45:47.994500+0000","last_unstale":"2023-10-09T06:45:56.010599+0000","last_undegraded":"2023-10-09T06:45:56.010599+0000","last_fullsized":"2023-10-09T06:45:56.010599+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:27.203434+0000","last_clean_scrub_stamp":"2023-10-09T06:39:27.203434+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:33:09.307580+0000","scrub_duration":0.0052775840000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":602,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007389+0000","last_change":"2023-10-09T06:27:36.704379+0000","last_active":"2023-10-09T06:45:56.007389+0000","last_peered":"2023-10-09T06:45:56.007389+0000","last_clean":"2023-10-09T06:45:56.007389+0000","last_became_active":"2023-10-09T06:27:36.668726+0000","last_became_peered":"2023-10-09T06:27:36.668726+0000","last_unstale":"2023-10-09T06:45:56.007389+0000","last_undegraded":"2023-10-09T06:45:56.007389+0000","last_fullsized":"2023-10-09T06:45:56.007389+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:27:58.705102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006659+0000","last_change":"2023-10-09T06:44:51.049695+0000","last_active":"2023-10-09T06:45:56.006659+0000","last_peered":"2023-10-09T06:45:56.006659+0000","last_clean":"2023-10-09T06:45:56.006659+0000","last_became_active":"2023-10-09T06:44:51.049267+0000","last_became_peered":"2023-10-09T06:44:51.049267+0000","last_unstale":"2023-10-09T06:45:56.006659+0000","last_undegraded":"2023-10-09T06:45:56.006659+0000","last_fullsized":"2023-10-09T06:45:56.006659+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:12:44.496671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"77'161","reported_seq":652,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425228+0000","last_change":"2023-10-09T06:45:35.665965+0000","last_active":"2023-10-09T06:45:56.425228+0000","last_peered":"2023-10-09T06:45:56.425228+0000","last_clean":"2023-10-09T06:45:56.425228+0000","last_became_active":"2023-10-09T06:45:29.894620+0000","last_became_peered":"2023-10-09T06:45:29.894620+0000","last_unstale":"2023-10-09T06:45:56.425228+0000","last_undegraded":"2023-10-09T06:45:56.425228+0000","last_fullsized":"2023-10-09T06:45:56.425228+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:35.665866+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:26.333195+0000","last_clean_scrub_stamp":"2023-10-09T06:45:35.665866+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:08.662278+0000","scrub_duration":0.0019991689999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":344,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425458+0000","last_change":"2023-10-09T06:44:51.056390+0000","last_active":"2023-10-09T06:45:56.425458+0000","last_peered":"2023-10-09T06:45:56.425458+0000","last_clean":"2023-10-09T06:45:56.425458+0000","last_became_active":"2023-10-09T06:44:51.055835+0000","last_became_peered":"2023-10-09T06:44:51.055835+0000","last_unstale":"2023-10-09T06:45:56.425458+0000","last_undegraded":"2023-10-09T06:45:56.425458+0000","last_fullsized":"2023-10-09T06:45:56.425458+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:09:55.171015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141151+0000","last_change":"2023-10-09T06:44:57.095295+0000","last_active":"2023-10-09T06:45:56.141151+0000","last_peered":"2023-10-09T06:45:56.141151+0000","last_clean":"2023-10-09T06:45:56.141151+0000","last_became_active":"2023-10-09T06:44:57.094946+0000","last_became_peered":"2023-10-09T06:44:57.094946+0000","last_unstale":"2023-10-09T06:45:56.141151+0000","last_undegraded":"2023-10-09T06:45:56.141151+0000","last_fullsized":"2023-10-09T06:45:56.141151+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:20.863235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"77'161","reported_seq":712,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427800+0000","last_change":"2023-10-09T06:32:24.598991+0000","last_active":"2023-10-09T06:45:56.427800+0000","last_peered":"2023-10-09T06:45:56.427800+0000","last_clean":"2023-10-09T06:45:56.427800+0000","last_became_active":"2023-10-09T06:32:19.747451+0000","last_became_peered":"2023-10-09T06:32:19.747451+0000","last_unstale":"2023-10-09T06:45:56.427800+0000","last_undegraded":"2023-10-09T06:45:56.427800+0000","last_fullsized":"2023-10-09T06:45:56.427800+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:24.598898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:24.598898+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.193898+0000","scrub_duration":0.013731133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006937+0000","last_change":"2023-10-09T06:44:51.045302+0000","last_active":"2023-10-09T06:45:56.006937+0000","last_peered":"2023-10-09T06:45:56.006937+0000","last_clean":"2023-10-09T06:45:56.006937+0000","last_became_active":"2023-10-09T06:44:51.044895+0000","last_became_peered":"2023-10-09T06:44:51.044895+0000","last_unstale":"2023-10-09T06:45:56.006937+0000","last_undegraded":"2023-10-09T06:45:56.006937+0000","last_fullsized":"2023-10-09T06:45:56.006937+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:10.438707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":327,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.214582+0000","last_change":"2023-10-09T06:44:57.093911+0000","last_active":"2023-10-09T06:45:59.214582+0000","last_peered":"2023-10-09T06:45:59.214582+0000","last_clean":"2023-10-09T06:45:59.214582+0000","last_became_active":"2023-10-09T06:44:57.093575+0000","last_became_peered":"2023-10-09T06:44:57.093575+0000","last_unstale":"2023-10-09T06:45:59.214582+0000","last_undegraded":"2023-10-09T06:45:59.214582+0000","last_fullsized":"2023-10-09T06:45:59.214582+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:32.176495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"77'161","reported_seq":530,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425260+0000","last_change":"2023-10-09T06:45:34.673437+0000","last_active":"2023-10-09T06:45:56.425260+0000","last_peered":"2023-10-09T06:45:56.425260+0000","last_clean":"2023-10-09T06:45:56.425260+0000","last_became_active":"2023-10-09T06:45:29.896909+0000","last_became_peered":"2023-10-09T06:45:29.896909+0000","last_unstale":"2023-10-09T06:45:56.425260+0000","last_undegraded":"2023-10-09T06:45:56.425260+0000","last_fullsized":"2023-10-09T06:45:56.425260+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:34.673334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:34.673334+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:48.787338+0000","scrub_duration":0.001614814,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010471+0000","last_change":"2023-10-09T06:28:11.703930+0000","last_active":"2023-10-09T06:45:56.010471+0000","last_peered":"2023-10-09T06:45:56.010471+0000","last_clean":"2023-10-09T06:45:56.010471+0000","last_became_active":"2023-10-09T06:28:11.703564+0000","last_became_peered":"2023-10-09T06:28:11.703564+0000","last_unstale":"2023-10-09T06:45:56.010471+0000","last_undegraded":"2023-10-09T06:45:56.010471+0000","last_fullsized":"2023-10-09T06:45:56.010471+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:56:22.743005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":327,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215102+0000","last_change":"2023-10-09T06:44:57.094186+0000","last_active":"2023-10-09T06:45:59.215102+0000","last_peered":"2023-10-09T06:45:59.215102+0000","last_clean":"2023-10-09T06:45:59.215102+0000","last_became_active":"2023-10-09T06:44:57.093864+0000","last_became_peered":"2023-10-09T06:44:57.093864+0000","last_unstale":"2023-10-09T06:45:59.215102+0000","last_undegraded":"2023-10-09T06:45:59.215102+0000","last_fullsized":"2023-10-09T06:45:59.215102+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:59.236931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"77'161","reported_seq":682,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010493+0000","last_change":"2023-10-09T06:45:47.994449+0000","last_active":"2023-10-09T06:45:56.010493+0000","last_peered":"2023-10-09T06:45:56.010493+0000","last_clean":"2023-10-09T06:45:56.010493+0000","last_became_active":"2023-10-09T06:45:47.994131+0000","last_became_peered":"2023-10-09T06:45:47.994131+0000","last_unstale":"2023-10-09T06:45:56.010493+0000","last_undegraded":"2023-10-09T06:45:56.010493+0000","last_fullsized":"2023-10-09T06:45:56.010493+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:30.150873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:30.150873+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:31:48.403216+0000","scrub_duration":0.0017668409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":485,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428348+0000","last_change":"2023-10-09T06:27:36.689039+0000","last_active":"2023-10-09T06:45:56.428348+0000","last_peered":"2023-10-09T06:45:56.428348+0000","last_clean":"2023-10-09T06:45:56.428348+0000","last_became_active":"2023-10-09T06:27:36.633366+0000","last_became_peered":"2023-10-09T06:27:36.633366+0000","last_unstale":"2023-10-09T06:45:56.428348+0000","last_undegraded":"2023-10-09T06:45:56.428348+0000","last_fullsized":"2023-10-09T06:45:56.428348+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:07:31.552611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":580,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008308+0000","last_change":"2023-10-09T06:44:57.100121+0000","last_active":"2023-10-09T06:45:56.008308+0000","last_peered":"2023-10-09T06:45:56.008308+0000","last_clean":"2023-10-09T06:45:56.008308+0000","last_became_active":"2023-10-09T06:44:57.099823+0000","last_became_peered":"2023-10-09T06:44:57.099823+0000","last_unstale":"2023-10-09T06:45:56.008308+0000","last_undegraded":"2023-10-09T06:45:56.008308+0000","last_fullsized":"2023-10-09T06:45:56.008308+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:27.144157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"77'161","reported_seq":749,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007882+0000","last_change":"2023-10-09T06:44:53.741205+0000","last_active":"2023-10-09T06:45:56.007882+0000","last_peered":"2023-10-09T06:45:56.007882+0000","last_clean":"2023-10-09T06:45:56.007882+0000","last_became_active":"2023-10-09T06:44:53.740849+0000","last_became_peered":"2023-10-09T06:44:53.740849+0000","last_unstale":"2023-10-09T06:45:56.007882+0000","last_undegraded":"2023-10-09T06:45:56.007882+0000","last_fullsized":"2023-10-09T06:45:56.007882+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:26.181810+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:26.181810+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:49:50.967855+0000","scrub_duration":0.028002856999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":650,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428070+0000","last_change":"2023-10-09T06:44:57.112092+0000","last_active":"2023-10-09T06:45:56.428070+0000","last_peered":"2023-10-09T06:45:56.428070+0000","last_clean":"2023-10-09T06:45:56.428070+0000","last_became_active":"2023-10-09T06:44:57.102041+0000","last_became_peered":"2023-10-09T06:44:57.102041+0000","last_unstale":"2023-10-09T06:45:56.428070+0000","last_undegraded":"2023-10-09T06:45:56.428070+0000","last_fullsized":"2023-10-09T06:45:56.428070+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:31.390225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":594,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428132+0000","last_change":"2023-10-09T06:44:57.095258+0000","last_active":"2023-10-09T06:45:56.428132+0000","last_peered":"2023-10-09T06:45:56.428132+0000","last_clean":"2023-10-09T06:45:56.428132+0000","last_became_active":"2023-10-09T06:44:57.094515+0000","last_became_peered":"2023-10-09T06:44:57.094515+0000","last_unstale":"2023-10-09T06:45:56.428132+0000","last_undegraded":"2023-10-09T06:45:56.428132+0000","last_fullsized":"2023-10-09T06:45:56.428132+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:07.381588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"77'161","reported_seq":764,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006480+0000","last_change":"2023-10-09T06:44:58.524760+0000","last_active":"2023-10-09T06:45:56.006480+0000","last_peered":"2023-10-09T06:45:56.006480+0000","last_clean":"2023-10-09T06:45:56.006480+0000","last_became_active":"2023-10-09T06:44:53.745772+0000","last_became_peered":"2023-10-09T06:44:53.745772+0000","last_unstale":"2023-10-09T06:45:56.006480+0000","last_undegraded":"2023-10-09T06:45:56.006480+0000","last_fullsized":"2023-10-09T06:45:56.006480+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:58.524664+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:44:58.524664+0000","last_clean_scrub_stamp":"2023-10-09T06:44:58.524664+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:16.128021+0000","scrub_duration":0.0022087320000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":453,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427314+0000","last_change":"2023-10-09T06:19:06.721874+0000","last_active":"2023-10-09T06:45:56.427314+0000","last_peered":"2023-10-09T06:45:56.427314+0000","last_clean":"2023-10-09T06:45:56.427314+0000","last_became_active":"2023-10-09T06:19:06.665451+0000","last_became_peered":"2023-10-09T06:19:06.665451+0000","last_unstale":"2023-10-09T06:45:56.427314+0000","last_undegraded":"2023-10-09T06:45:56.427314+0000","last_fullsized":"2023-10-09T06:45:56.427314+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:54.682873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":309,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215562+0000","last_change":"2023-10-09T06:27:36.645773+0000","last_active":"2023-10-09T06:45:59.215562+0000","last_peered":"2023-10-09T06:45:59.215562+0000","last_clean":"2023-10-09T06:45:59.215562+0000","last_became_active":"2023-10-09T06:27:36.645299+0000","last_became_peered":"2023-10-09T06:27:36.645299+0000","last_unstale":"2023-10-09T06:45:59.215562+0000","last_undegraded":"2023-10-09T06:45:59.215562+0000","last_fullsized":"2023-10-09T06:45:59.215562+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:58.334635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"77'161","reported_seq":876,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425003+0000","last_change":"2023-10-09T06:45:29.895178+0000","last_active":"2023-10-09T06:45:56.425003+0000","last_peered":"2023-10-09T06:45:56.425003+0000","last_clean":"2023-10-09T06:45:56.425003+0000","last_became_active":"2023-10-09T06:45:29.894775+0000","last_became_peered":"2023-10-09T06:45:29.894775+0000","last_unstale":"2023-10-09T06:45:56.425003+0000","last_undegraded":"2023-10-09T06:45:56.425003+0000","last_fullsized":"2023-10-09T06:45:56.425003+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:39:10.128116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:39:10.128116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:07.386362+0000","scrub_duration":0.013709687,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006521+0000","last_change":"2023-10-09T06:44:51.049108+0000","last_active":"2023-10-09T06:45:56.006521+0000","last_peered":"2023-10-09T06:45:56.006521+0000","last_clean":"2023-10-09T06:45:56.006521+0000","last_became_active":"2023-10-09T06:44:51.048886+0000","last_became_peered":"2023-10-09T06:44:51.048886+0000","last_unstale":"2023-10-09T06:45:56.006521+0000","last_undegraded":"2023-10-09T06:45:56.006521+0000","last_fullsized":"2023-10-09T06:45:56.006521+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:10:55.701046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":288,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010346+0000","last_change":"2023-10-09T06:45:47.995720+0000","last_active":"2023-10-09T06:45:56.010346+0000","last_peered":"2023-10-09T06:45:56.010346+0000","last_clean":"2023-10-09T06:45:56.010346+0000","last_became_active":"2023-10-09T06:45:47.994870+0000","last_became_peered":"2023-10-09T06:45:47.994870+0000","last_unstale":"2023-10-09T06:45:56.010346+0000","last_undegraded":"2023-10-09T06:45:56.010346+0000","last_fullsized":"2023-10-09T06:45:56.010346+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:09.324884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"77'161","reported_seq":934,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007676+0000","last_change":"2023-10-09T06:32:19.134405+0000","last_active":"2023-10-09T06:45:56.007676+0000","last_peered":"2023-10-09T06:45:56.007676+0000","last_clean":"2023-10-09T06:45:56.007676+0000","last_became_active":"2023-10-09T06:32:03.479061+0000","last_became_peered":"2023-10-09T06:32:03.479061+0000","last_unstale":"2023-10-09T06:45:56.007676+0000","last_undegraded":"2023-10-09T06:45:56.007676+0000","last_fullsized":"2023-10-09T06:45:56.007676+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.134228+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.134228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:04:10.561830+0000","scrub_duration":0.17134993600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":487,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427214+0000","last_change":"2023-10-09T06:44:57.102182+0000","last_active":"2023-10-09T06:45:56.427214+0000","last_peered":"2023-10-09T06:45:56.427214+0000","last_clean":"2023-10-09T06:45:56.427214+0000","last_became_active":"2023-10-09T06:44:57.101856+0000","last_became_peered":"2023-10-09T06:44:57.101856+0000","last_unstale":"2023-10-09T06:45:56.427214+0000","last_undegraded":"2023-10-09T06:45:56.427214+0000","last_fullsized":"2023-10-09T06:45:56.427214+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:42.716993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006866+0000","last_change":"2023-10-09T06:44:51.052808+0000","last_active":"2023-10-09T06:45:56.006866+0000","last_peered":"2023-10-09T06:45:56.006866+0000","last_clean":"2023-10-09T06:45:56.006866+0000","last_became_active":"2023-10-09T06:44:51.052459+0000","last_became_peered":"2023-10-09T06:44:51.052459+0000","last_unstale":"2023-10-09T06:45:56.006866+0000","last_undegraded":"2023-10-09T06:45:56.006866+0000","last_fullsized":"2023-10-09T06:45:56.006866+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:27.222371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"77'161","reported_seq":758,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010399+0000","last_change":"2023-10-09T06:45:00.113440+0000","last_active":"2023-10-09T06:45:56.010399+0000","last_peered":"2023-10-09T06:45:56.010399+0000","last_clean":"2023-10-09T06:45:56.010399+0000","last_became_active":"2023-10-09T06:45:00.113082+0000","last_became_peered":"2023-10-09T06:45:00.113082+0000","last_unstale":"2023-10-09T06:45:56.010399+0000","last_undegraded":"2023-10-09T06:45:56.010399+0000","last_fullsized":"2023-10-09T06:45:56.010399+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.142430+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.142430+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:18:16.522169+0000","scrub_duration":0.0053330900000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140736+0000","last_change":"2023-10-09T06:44:57.118343+0000","last_active":"2023-10-09T06:45:56.140736+0000","last_peered":"2023-10-09T06:45:56.140736+0000","last_clean":"2023-10-09T06:45:56.140736+0000","last_became_active":"2023-10-09T06:44:57.095708+0000","last_became_peered":"2023-10-09T06:44:57.095708+0000","last_unstale":"2023-10-09T06:45:56.140736+0000","last_undegraded":"2023-10-09T06:45:56.140736+0000","last_fullsized":"2023-10-09T06:45:56.140736+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:34.643625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":308,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426301+0000","last_change":"2023-10-09T06:27:36.647879+0000","last_active":"2023-10-09T06:45:56.426301+0000","last_peered":"2023-10-09T06:45:56.426301+0000","last_clean":"2023-10-09T06:45:56.426301+0000","last_became_active":"2023-10-09T06:27:36.647456+0000","last_became_peered":"2023-10-09T06:27:36.647456+0000","last_unstale":"2023-10-09T06:45:56.426301+0000","last_undegraded":"2023-10-09T06:45:56.426301+0000","last_fullsized":"2023-10-09T06:45:56.426301+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:18.998994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"77'161","reported_seq":731,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427920+0000","last_change":"2023-10-09T06:32:16.690393+0000","last_active":"2023-10-09T06:45:56.427920+0000","last_peered":"2023-10-09T06:45:56.427920+0000","last_clean":"2023-10-09T06:45:56.427920+0000","last_became_active":"2023-10-09T06:32:10.598346+0000","last_became_peered":"2023-10-09T06:32:10.598346+0000","last_unstale":"2023-10-09T06:45:56.427920+0000","last_undegraded":"2023-10-09T06:45:56.427920+0000","last_fullsized":"2023-10-09T06:45:56.427920+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.690282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.690282+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:16:16.490340+0000","scrub_duration":0.001865893,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":517,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428456+0000","last_change":"2023-10-09T06:44:51.062738+0000","last_active":"2023-10-09T06:45:56.428456+0000","last_peered":"2023-10-09T06:45:56.428456+0000","last_clean":"2023-10-09T06:45:56.428456+0000","last_became_active":"2023-10-09T06:44:51.055897+0000","last_became_peered":"2023-10-09T06:44:51.055897+0000","last_unstale":"2023-10-09T06:45:56.428456+0000","last_undegraded":"2023-10-09T06:45:56.428456+0000","last_fullsized":"2023-10-09T06:45:56.428456+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:02.436082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":394,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010027+0000","last_change":"2023-10-09T06:45:47.996770+0000","last_active":"2023-10-09T06:45:56.010027+0000","last_peered":"2023-10-09T06:45:56.010027+0000","last_clean":"2023-10-09T06:45:56.010027+0000","last_became_active":"2023-10-09T06:45:47.996439+0000","last_became_peered":"2023-10-09T06:45:47.996439+0000","last_unstale":"2023-10-09T06:45:56.010027+0000","last_undegraded":"2023-10-09T06:45:56.010027+0000","last_fullsized":"2023-10-09T06:45:56.010027+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:22.474695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"77'161","reported_seq":1077,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425177+0000","last_change":"2023-10-09T06:45:00.108744+0000","last_active":"2023-10-09T06:45:56.425177+0000","last_peered":"2023-10-09T06:45:56.425177+0000","last_clean":"2023-10-09T06:45:56.425177+0000","last_became_active":"2023-10-09T06:45:00.108419+0000","last_became_peered":"2023-10-09T06:45:00.108419+0000","last_unstale":"2023-10-09T06:45:56.425177+0000","last_undegraded":"2023-10-09T06:45:56.425177+0000","last_fullsized":"2023-10-09T06:45:56.425177+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:12.100338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:12.100338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:42.093677+0000","scrub_duration":0.012873661999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":451,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428107+0000","last_change":"2023-10-09T06:19:06.774532+0000","last_active":"2023-10-09T06:45:56.428107+0000","last_peered":"2023-10-09T06:45:56.428107+0000","last_clean":"2023-10-09T06:45:56.428107+0000","last_became_active":"2023-10-09T06:19:06.753303+0000","last_became_peered":"2023-10-09T06:19:06.753303+0000","last_unstale":"2023-10-09T06:45:56.428107+0000","last_undegraded":"2023-10-09T06:45:56.428107+0000","last_fullsized":"2023-10-09T06:45:56.428107+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:35:34.128455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":611,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427524+0000","last_change":"2023-10-09T06:44:18.735683+0000","last_active":"2023-10-09T06:45:56.427524+0000","last_peered":"2023-10-09T06:45:56.427524+0000","last_clean":"2023-10-09T06:45:56.427524+0000","last_became_active":"2023-10-09T06:44:18.735354+0000","last_became_peered":"2023-10-09T06:44:18.735354+0000","last_unstale":"2023-10-09T06:45:56.427524+0000","last_undegraded":"2023-10-09T06:45:56.427524+0000","last_fullsized":"2023-10-09T06:45:56.427524+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:09.979649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"77'161","reported_seq":702,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140523+0000","last_change":"2023-10-09T06:45:10.332573+0000","last_active":"2023-10-09T06:45:56.140523+0000","last_peered":"2023-10-09T06:45:56.140523+0000","last_clean":"2023-10-09T06:45:56.140523+0000","last_became_active":"2023-10-09T06:45:00.110237+0000","last_became_peered":"2023-10-09T06:45:00.110237+0000","last_unstale":"2023-10-09T06:45:56.140523+0000","last_undegraded":"2023-10-09T06:45:56.140523+0000","last_fullsized":"2023-10-09T06:45:56.140523+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:10.332488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:10.332488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:12.917946+0000","scrub_duration":0.001695691,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":535,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428098+0000","last_change":"2023-10-09T06:44:57.113953+0000","last_active":"2023-10-09T06:45:56.428098+0000","last_peered":"2023-10-09T06:45:56.428098+0000","last_clean":"2023-10-09T06:45:56.428098+0000","last_became_active":"2023-10-09T06:44:57.099932+0000","last_became_peered":"2023-10-09T06:44:57.099932+0000","last_unstale":"2023-10-09T06:45:56.428098+0000","last_undegraded":"2023-10-09T06:45:56.428098+0000","last_fullsized":"2023-10-09T06:45:56.428098+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:51.434120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140708+0000","last_change":"2023-10-09T06:44:57.098608+0000","last_active":"2023-10-09T06:45:56.140708+0000","last_peered":"2023-10-09T06:45:56.140708+0000","last_clean":"2023-10-09T06:45:56.140708+0000","last_became_active":"2023-10-09T06:44:57.098275+0000","last_became_peered":"2023-10-09T06:44:57.098275+0000","last_unstale":"2023-10-09T06:45:56.140708+0000","last_undegraded":"2023-10-09T06:45:56.140708+0000","last_fullsized":"2023-10-09T06:45:56.140708+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:48.449595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"77'161","reported_seq":728,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428254+0000","last_change":"2023-10-09T06:32:16.253097+0000","last_active":"2023-10-09T06:45:56.428254+0000","last_peered":"2023-10-09T06:45:56.428254+0000","last_clean":"2023-10-09T06:45:56.428254+0000","last_became_active":"2023-10-09T06:32:10.595097+0000","last_became_peered":"2023-10-09T06:32:10.595097+0000","last_unstale":"2023-10-09T06:45:56.428254+0000","last_undegraded":"2023-10-09T06:45:56.428254+0000","last_fullsized":"2023-10-09T06:45:56.428254+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:16.253030+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:16.253030+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:43.626299+0000","scrub_duration":0.0066887969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010315+0000","last_change":"2023-10-09T06:45:47.995081+0000","last_active":"2023-10-09T06:45:56.010315+0000","last_peered":"2023-10-09T06:45:56.010315+0000","last_clean":"2023-10-09T06:45:56.010315+0000","last_became_active":"2023-10-09T06:45:47.994801+0000","last_became_peered":"2023-10-09T06:45:47.994801+0000","last_unstale":"2023-10-09T06:45:56.010315+0000","last_undegraded":"2023-10-09T06:45:56.010315+0000","last_fullsized":"2023-10-09T06:45:56.010315+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:50:12.460282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007013+0000","last_change":"2023-10-09T06:44:51.050701+0000","last_active":"2023-10-09T06:45:56.007013+0000","last_peered":"2023-10-09T06:45:56.007013+0000","last_clean":"2023-10-09T06:45:56.007013+0000","last_became_active":"2023-10-09T06:44:51.050372+0000","last_became_peered":"2023-10-09T06:44:51.050372+0000","last_unstale":"2023-10-09T06:45:56.007013+0000","last_undegraded":"2023-10-09T06:45:56.007013+0000","last_fullsized":"2023-10-09T06:45:56.007013+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:33.153489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"77'161","reported_seq":956,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427114+0000","last_change":"2023-10-09T06:32:15.726982+0000","last_active":"2023-10-09T06:45:56.427114+0000","last_peered":"2023-10-09T06:45:56.427114+0000","last_clean":"2023-10-09T06:45:56.427114+0000","last_became_active":"2023-10-09T06:32:10.603736+0000","last_became_peered":"2023-10-09T06:32:10.603736+0000","last_unstale":"2023-10-09T06:45:56.427114+0000","last_undegraded":"2023-10-09T06:45:56.427114+0000","last_fullsized":"2023-10-09T06:45:56.427114+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:15.726870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:15.726870+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T18:00:42.881333+0000","scrub_duration":0.010323339000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":361,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425910+0000","last_change":"2023-10-09T06:44:51.064784+0000","last_active":"2023-10-09T06:45:56.425910+0000","last_peered":"2023-10-09T06:45:56.425910+0000","last_clean":"2023-10-09T06:45:56.425910+0000","last_became_active":"2023-10-09T06:44:51.057499+0000","last_became_peered":"2023-10-09T06:44:51.057499+0000","last_unstale":"2023-10-09T06:45:56.425910+0000","last_undegraded":"2023-10-09T06:45:56.425910+0000","last_fullsized":"2023-10-09T06:45:56.425910+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:52.148891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":604,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427900+0000","last_change":"2023-10-09T06:44:57.099550+0000","last_active":"2023-10-09T06:45:56.427900+0000","last_peered":"2023-10-09T06:45:56.427900+0000","last_clean":"2023-10-09T06:45:56.427900+0000","last_became_active":"2023-10-09T06:44:57.099246+0000","last_became_peered":"2023-10-09T06:44:57.099246+0000","last_unstale":"2023-10-09T06:45:56.427900+0000","last_undegraded":"2023-10-09T06:45:56.427900+0000","last_fullsized":"2023-10-09T06:45:56.427900+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:22.898067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"77'161","reported_seq":806,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010105+0000","last_change":"2023-10-09T06:45:47.995383+0000","last_active":"2023-10-09T06:45:56.010105+0000","last_peered":"2023-10-09T06:45:56.010105+0000","last_clean":"2023-10-09T06:45:56.010105+0000","last_became_active":"2023-10-09T06:45:47.995061+0000","last_became_peered":"2023-10-09T06:45:47.995061+0000","last_unstale":"2023-10-09T06:45:56.010105+0000","last_undegraded":"2023-10-09T06:45:56.010105+0000","last_fullsized":"2023-10-09T06:45:56.010105+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:32.625098+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:39:26.873632+0000","last_clean_scrub_stamp":"2023-10-09T06:45:32.625098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:07:40.864954+0000","scrub_duration":0.0053873230000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140732+0000","last_change":"2023-10-09T06:44:57.101594+0000","last_active":"2023-10-09T06:45:56.140732+0000","last_peered":"2023-10-09T06:45:56.140732+0000","last_clean":"2023-10-09T06:45:56.140732+0000","last_became_active":"2023-10-09T06:44:57.101280+0000","last_became_peered":"2023-10-09T06:44:57.101280+0000","last_unstale":"2023-10-09T06:45:56.140732+0000","last_undegraded":"2023-10-09T06:45:56.140732+0000","last_fullsized":"2023-10-09T06:45:56.140732+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:27.578039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":581,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428351+0000","last_change":"2023-10-09T06:15:11.874998+0000","last_active":"2023-10-09T06:45:56.428351+0000","last_peered":"2023-10-09T06:45:56.428351+0000","last_clean":"2023-10-09T06:45:56.428351+0000","last_became_active":"2023-10-09T06:15:11.874715+0000","last_became_peered":"2023-10-09T06:15:11.874715+0000","last_unstale":"2023-10-09T06:45:56.428351+0000","last_undegraded":"2023-10-09T06:45:56.428351+0000","last_fullsized":"2023-10-09T06:45:56.428351+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:53.523485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"77'161","reported_seq":977,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007767+0000","last_change":"2023-10-09T06:44:53.743777+0000","last_active":"2023-10-09T06:45:56.007767+0000","last_peered":"2023-10-09T06:45:56.007767+0000","last_clean":"2023-10-09T06:45:56.007767+0000","last_became_active":"2023-10-09T06:44:53.743495+0000","last_became_peered":"2023-10-09T06:44:53.743495+0000","last_unstale":"2023-10-09T06:45:56.007767+0000","last_undegraded":"2023-10-09T06:45:56.007767+0000","last_fullsized":"2023-10-09T06:45:56.007767+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":372,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:19.978348+0000","last_clean_scrub_stamp":"2023-10-09T06:32:19.978348+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:03.319999+0000","scrub_duration":0.015057011,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":585,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427262+0000","last_change":"2023-10-09T06:19:06.675229+0000","last_active":"2023-10-09T06:45:56.427262+0000","last_peered":"2023-10-09T06:45:56.427262+0000","last_clean":"2023-10-09T06:45:56.427262+0000","last_became_active":"2023-10-09T06:19:06.674905+0000","last_became_peered":"2023-10-09T06:19:06.674905+0000","last_unstale":"2023-10-09T06:45:56.427262+0000","last_undegraded":"2023-10-09T06:45:56.427262+0000","last_fullsized":"2023-10-09T06:45:56.427262+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:16:42.205464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":584,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428441+0000","last_change":"2023-10-09T06:44:57.100399+0000","last_active":"2023-10-09T06:45:56.428441+0000","last_peered":"2023-10-09T06:45:56.428441+0000","last_clean":"2023-10-09T06:45:56.428441+0000","last_became_active":"2023-10-09T06:44:57.099999+0000","last_became_peered":"2023-10-09T06:44:57.099999+0000","last_unstale":"2023-10-09T06:45:56.428441+0000","last_undegraded":"2023-10-09T06:45:56.428441+0000","last_fullsized":"2023-10-09T06:45:56.428441+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:32.627021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"77'161","reported_seq":784,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010281+0000","last_change":"2023-10-09T06:45:47.994732+0000","last_active":"2023-10-09T06:45:56.010281+0000","last_peered":"2023-10-09T06:45:56.010281+0000","last_clean":"2023-10-09T06:45:56.010281+0000","last_became_active":"2023-10-09T06:45:47.994373+0000","last_became_peered":"2023-10-09T06:45:47.994373+0000","last_unstale":"2023-10-09T06:45:56.010281+0000","last_undegraded":"2023-10-09T06:45:56.010281+0000","last_fullsized":"2023-10-09T06:45:56.010281+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:45:30.591268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:45:30.591268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:54.272606+0000","scrub_duration":0.0030466540000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":309,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425546+0000","last_change":"2023-10-09T06:27:36.661315+0000","last_active":"2023-10-09T06:45:56.425546+0000","last_peered":"2023-10-09T06:45:56.425546+0000","last_clean":"2023-10-09T06:45:56.425546+0000","last_became_active":"2023-10-09T06:27:36.650722+0000","last_became_peered":"2023-10-09T06:27:36.650722+0000","last_unstale":"2023-10-09T06:45:56.425546+0000","last_undegraded":"2023-10-09T06:45:56.425546+0000","last_fullsized":"2023-10-09T06:45:56.425546+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:19.366281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":562,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007857+0000","last_change":"2023-10-09T06:27:36.631667+0000","last_active":"2023-10-09T06:45:56.007857+0000","last_peered":"2023-10-09T06:45:56.007857+0000","last_clean":"2023-10-09T06:45:56.007857+0000","last_became_active":"2023-10-09T06:27:36.631355+0000","last_became_peered":"2023-10-09T06:27:36.631355+0000","last_unstale":"2023-10-09T06:45:56.007857+0000","last_undegraded":"2023-10-09T06:45:56.007857+0000","last_fullsized":"2023-10-09T06:45:56.007857+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:55.714061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"77'161","reported_seq":659,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141137+0000","last_change":"2023-10-09T06:45:00.109875+0000","last_active":"2023-10-09T06:45:56.141137+0000","last_peered":"2023-10-09T06:45:56.141137+0000","last_clean":"2023-10-09T06:45:56.141137+0000","last_became_active":"2023-10-09T06:45:00.109495+0000","last_became_peered":"2023-10-09T06:45:00.109495+0000","last_unstale":"2023-10-09T06:45:56.141137+0000","last_undegraded":"2023-10-09T06:45:56.141137+0000","last_fullsized":"2023-10-09T06:45:56.141137+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:03.276039+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:03.276039+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:20.714787+0000","scrub_duration":0.0023411270000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"573'136","reported_seq":1132,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216917+0000","last_change":"2023-10-09T06:44:53.745637+0000","last_active":"2023-10-09T06:45:59.216917+0000","last_peered":"2023-10-09T06:45:59.216917+0000","last_clean":"2023-10-09T06:45:59.216917+0000","last_became_active":"2023-10-09T06:44:53.745062+0000","last_became_peered":"2023-10-09T06:44:53.745062+0000","last_unstale":"2023-10-09T06:45:59.216917+0000","last_undegraded":"2023-10-09T06:45:59.216917+0000","last_fullsized":"2023-10-09T06:45:59.216917+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":136,"log_dups_size":0,"ondisk_log_size":136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:04:44.644929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035738799999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":627,"num_read_kb":2028,"num_write":136,"num_write_kb":1324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":2,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010361+0000","last_change":"2023-10-09T06:45:47.995934+0000","last_active":"2023-10-09T06:45:56.010361+0000","last_peered":"2023-10-09T06:45:56.010361+0000","last_clean":"2023-10-09T06:45:56.010361+0000","last_became_active":"2023-10-09T06:45:47.995425+0000","last_became_peered":"2023-10-09T06:45:47.995425+0000","last_unstale":"2023-10-09T06:45:56.010361+0000","last_undegraded":"2023-10-09T06:45:56.010361+0000","last_fullsized":"2023-10-09T06:45:56.010361+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:35.354961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":549,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216534+0000","last_change":"2023-10-09T06:28:11.700144+0000","last_active":"2023-10-09T06:45:59.216534+0000","last_peered":"2023-10-09T06:45:59.216534+0000","last_clean":"2023-10-09T06:45:59.216534+0000","last_became_active":"2023-10-09T06:28:11.699791+0000","last_became_peered":"2023-10-09T06:28:11.699791+0000","last_unstale":"2023-10-09T06:45:59.216534+0000","last_undegraded":"2023-10-09T06:45:59.216534+0000","last_fullsized":"2023-10-09T06:45:59.216534+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.220199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"77'161","reported_seq":820,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010186+0000","last_change":"2023-10-09T06:45:47.987993+0000","last_active":"2023-10-09T06:45:56.010186+0000","last_peered":"2023-10-09T06:45:56.010186+0000","last_clean":"2023-10-09T06:45:56.010186+0000","last_became_active":"2023-10-09T06:45:47.987676+0000","last_became_peered":"2023-10-09T06:45:47.987676+0000","last_unstale":"2023-10-09T06:45:56.010186+0000","last_undegraded":"2023-10-09T06:45:56.010186+0000","last_fullsized":"2023-10-09T06:45:56.010186+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:55.451401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:55.451401+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:30:58.778079+0000","scrub_duration":0.011645910000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"573'90","reported_seq":835,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010177+0000","last_change":"2023-10-09T06:45:47.998091+0000","last_active":"2023-10-09T06:45:56.010177+0000","last_peered":"2023-10-09T06:45:56.010177+0000","last_clean":"2023-10-09T06:45:56.010177+0000","last_became_active":"2023-10-09T06:45:47.997699+0000","last_became_peered":"2023-10-09T06:45:47.997699+0000","last_unstale":"2023-10-09T06:45:56.010177+0000","last_undegraded":"2023-10-09T06:45:56.010177+0000","last_fullsized":"2023-10-09T06:45:56.010177+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:48.882045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000225527,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":662,"num_read_kb":1785,"num_write":84,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":625,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428028+0000","last_change":"2023-10-09T06:44:57.095443+0000","last_active":"2023-10-09T06:45:56.428028+0000","last_peered":"2023-10-09T06:45:56.428028+0000","last_clean":"2023-10-09T06:45:56.428028+0000","last_became_active":"2023-10-09T06:44:57.094944+0000","last_became_peered":"2023-10-09T06:44:57.094944+0000","last_unstale":"2023-10-09T06:45:56.428028+0000","last_undegraded":"2023-10-09T06:45:56.428028+0000","last_fullsized":"2023-10-09T06:45:56.428028+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:16.144698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":579,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216879+0000","last_change":"2023-10-09T06:27:36.631614+0000","last_active":"2023-10-09T06:45:59.216879+0000","last_peered":"2023-10-09T06:45:59.216879+0000","last_clean":"2023-10-09T06:45:59.216879+0000","last_became_active":"2023-10-09T06:27:36.631370+0000","last_became_peered":"2023-10-09T06:27:36.631370+0000","last_unstale":"2023-10-09T06:45:59.216879+0000","last_undegraded":"2023-10-09T06:45:59.216879+0000","last_fullsized":"2023-10-09T06:45:59.216879+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:10.786789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"77'161","reported_seq":986,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427455+0000","last_change":"2023-10-09T06:44:53.748889+0000","last_active":"2023-10-09T06:45:56.427455+0000","last_peered":"2023-10-09T06:45:56.427455+0000","last_clean":"2023-10-09T06:45:56.427455+0000","last_became_active":"2023-10-09T06:44:53.747630+0000","last_became_peered":"2023-10-09T06:44:53.747630+0000","last_unstale":"2023-10-09T06:45:56.427455+0000","last_undegraded":"2023-10-09T06:45:56.427455+0000","last_fullsized":"2023-10-09T06:45:56.427455+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:09.868704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:09.868704+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:21:01.511893+0000","scrub_duration":0.0037442669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"573'157","reported_seq":1201,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425079+0000","last_change":"2023-10-09T06:45:29.893515+0000","last_active":"2023-10-09T06:45:56.425079+0000","last_peered":"2023-10-09T06:45:56.425079+0000","last_clean":"2023-10-09T06:45:56.425079+0000","last_became_active":"2023-10-09T06:45:29.892852+0000","last_became_peered":"2023-10-09T06:45:29.892852+0000","last_unstale":"2023-10-09T06:45:56.425079+0000","last_undegraded":"2023-10-09T06:45:56.425079+0000","last_fullsized":"2023-10-09T06:45:56.425079+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:12:56.372743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033522800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1047,"num_read_kb":2896,"num_write":141,"num_write_kb":1616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":16,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":623,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007363+0000","last_change":"2023-10-09T06:44:57.091601+0000","last_active":"2023-10-09T06:45:56.007363+0000","last_peered":"2023-10-09T06:45:56.007363+0000","last_clean":"2023-10-09T06:45:56.007363+0000","last_became_active":"2023-10-09T06:44:57.091261+0000","last_became_peered":"2023-10-09T06:44:57.091261+0000","last_unstale":"2023-10-09T06:45:56.007363+0000","last_undegraded":"2023-10-09T06:45:56.007363+0000","last_fullsized":"2023-10-09T06:45:56.007363+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:01.004445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":627,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008409+0000","last_change":"2023-10-09T06:44:51.051148+0000","last_active":"2023-10-09T06:45:56.008409+0000","last_peered":"2023-10-09T06:45:56.008409+0000","last_clean":"2023-10-09T06:45:56.008409+0000","last_became_active":"2023-10-09T06:44:51.050863+0000","last_became_peered":"2023-10-09T06:44:51.050863+0000","last_unstale":"2023-10-09T06:45:56.008409+0000","last_undegraded":"2023-10-09T06:45:56.008409+0000","last_fullsized":"2023-10-09T06:45:56.008409+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:21.910235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"573'99","reported_seq":617,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010249+0000","last_change":"2023-10-09T06:45:47.994011+0000","last_active":"2023-10-09T06:45:56.010249+0000","last_peered":"2023-10-09T06:45:56.010249+0000","last_clean":"2023-10-09T06:45:56.010249+0000","last_became_active":"2023-10-09T06:45:47.993560+0000","last_became_peered":"2023-10-09T06:45:47.993560+0000","last_unstale":"2023-10-09T06:45:56.010249+0000","last_undegraded":"2023-10-09T06:45:56.010249+0000","last_fullsized":"2023-10-09T06:45:56.010249+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:13.026969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076790500000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":163,"num_read_kb":437,"num_write":99,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'161","reported_seq":1007,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426911+0000","last_change":"2023-10-09T06:45:00.109701+0000","last_active":"2023-10-09T06:45:56.426911+0000","last_peered":"2023-10-09T06:45:56.426911+0000","last_clean":"2023-10-09T06:45:56.426911+0000","last_became_active":"2023-10-09T06:45:00.109209+0000","last_became_peered":"2023-10-09T06:45:00.109209+0000","last_unstale":"2023-10-09T06:45:56.426911+0000","last_undegraded":"2023-10-09T06:45:56.426911+0000","last_fullsized":"2023-10-09T06:45:56.426911+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:10.851370+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:10.851370+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:22:37.845254+0000","scrub_duration":0.0052088550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":337,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.214421+0000","last_change":"2023-10-09T06:44:57.099304+0000","last_active":"2023-10-09T06:45:59.214421+0000","last_peered":"2023-10-09T06:45:59.214421+0000","last_clean":"2023-10-09T06:45:59.214421+0000","last_became_active":"2023-10-09T06:44:57.098921+0000","last_became_peered":"2023-10-09T06:44:57.098921+0000","last_unstale":"2023-10-09T06:45:59.214421+0000","last_undegraded":"2023-10-09T06:45:59.214421+0000","last_fullsized":"2023-10-09T06:45:59.214421+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:09:02.502847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":31,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141055+0000","last_change":"2023-10-09T06:44:57.094991+0000","last_active":"2023-10-09T06:45:56.141055+0000","last_peered":"2023-10-09T06:45:56.141055+0000","last_clean":"2023-10-09T06:45:56.141055+0000","last_became_active":"2023-10-09T06:44:57.094638+0000","last_became_peered":"2023-10-09T06:44:57.094638+0000","last_unstale":"2023-10-09T06:45:56.141055+0000","last_undegraded":"2023-10-09T06:45:56.141055+0000","last_fullsized":"2023-10-09T06:45:56.141055+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:29.219217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"77'161","reported_seq":952,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216729+0000","last_change":"2023-10-09T06:45:00.111537+0000","last_active":"2023-10-09T06:45:59.216729+0000","last_peered":"2023-10-09T06:45:59.216729+0000","last_clean":"2023-10-09T06:45:59.216729+0000","last_became_active":"2023-10-09T06:45:00.111166+0000","last_became_peered":"2023-10-09T06:45:00.111166+0000","last_unstale":"2023-10-09T06:45:59.216729+0000","last_undegraded":"2023-10-09T06:45:59.216729+0000","last_fullsized":"2023-10-09T06:45:59.216729+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:14.925938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:14.925938+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:51.491231+0000","scrub_duration":0.0028339469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"573'110","reported_seq":944,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427418+0000","last_change":"2023-10-09T06:44:53.748832+0000","last_active":"2023-10-09T06:45:56.427418+0000","last_peered":"2023-10-09T06:45:56.427418+0000","last_clean":"2023-10-09T06:45:56.427418+0000","last_became_active":"2023-10-09T06:44:53.748345+0000","last_became_peered":"2023-10-09T06:44:53.748345+0000","last_unstale":"2023-10-09T06:45:56.427418+0000","last_undegraded":"2023-10-09T06:45:56.427418+0000","last_fullsized":"2023-10-09T06:45:56.427418+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":110,"log_dups_size":0,"ondisk_log_size":110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:57:08.273007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001926374,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":826,"num_read_kb":718,"num_write":101,"num_write_kb":58,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":10240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006380+0000","last_change":"2023-10-09T06:44:51.056475+0000","last_active":"2023-10-09T06:45:56.006380+0000","last_peered":"2023-10-09T06:45:56.006380+0000","last_clean":"2023-10-09T06:45:56.006380+0000","last_became_active":"2023-10-09T06:44:51.056165+0000","last_became_peered":"2023-10-09T06:44:51.056165+0000","last_unstale":"2023-10-09T06:45:56.006380+0000","last_undegraded":"2023-10-09T06:45:56.006380+0000","last_fullsized":"2023-10-09T06:45:56.006380+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:06:30.834127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006769+0000","last_change":"2023-10-09T06:44:51.051163+0000","last_active":"2023-10-09T06:45:56.006769+0000","last_peered":"2023-10-09T06:45:56.006769+0000","last_clean":"2023-10-09T06:45:56.006769+0000","last_became_active":"2023-10-09T06:44:51.050878+0000","last_became_peered":"2023-10-09T06:44:51.050878+0000","last_unstale":"2023-10-09T06:45:56.006769+0000","last_undegraded":"2023-10-09T06:45:56.006769+0000","last_fullsized":"2023-10-09T06:45:56.006769+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:53:18.212567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"77'161","reported_seq":650,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.140283+0000","last_change":"2023-10-09T06:45:00.111587+0000","last_active":"2023-10-09T06:45:56.140283+0000","last_peered":"2023-10-09T06:45:56.140283+0000","last_clean":"2023-10-09T06:45:56.140283+0000","last_became_active":"2023-10-09T06:45:00.111294+0000","last_became_peered":"2023-10-09T06:45:00.111294+0000","last_unstale":"2023-10-09T06:45:56.140283+0000","last_undegraded":"2023-10-09T06:45:56.140283+0000","last_fullsized":"2023-10-09T06:45:56.140283+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:44:43.314983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:44:43.314983+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:06:03.930013+0000","scrub_duration":0.0021392379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"573'201","reported_seq":1226,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428252+0000","last_change":"2023-10-09T06:44:53.748266+0000","last_active":"2023-10-09T06:45:56.428252+0000","last_peered":"2023-10-09T06:45:56.428252+0000","last_clean":"2023-10-09T06:45:56.428252+0000","last_became_active":"2023-10-09T06:44:53.746859+0000","last_became_peered":"2023-10-09T06:44:53.746859+0000","last_unstale":"2023-10-09T06:45:56.428252+0000","last_undegraded":"2023-10-09T06:45:56.428252+0000","last_fullsized":"2023-10-09T06:45:56.428252+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":201,"log_dups_size":0,"ondisk_log_size":201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:39:12.464291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088493999999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1244,"num_read_kb":4126,"num_write":180,"num_write_kb":3151,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":35,"num_keys_recovered":11,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":605,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.216847+0000","last_change":"2023-10-09T06:28:11.698945+0000","last_active":"2023-10-09T06:45:59.216847+0000","last_peered":"2023-10-09T06:45:59.216847+0000","last_clean":"2023-10-09T06:45:59.216847+0000","last_became_active":"2023-10-09T06:28:11.698615+0000","last_became_peered":"2023-10-09T06:28:11.698615+0000","last_unstale":"2023-10-09T06:45:59.216847+0000","last_undegraded":"2023-10-09T06:45:59.216847+0000","last_fullsized":"2023-10-09T06:45:59.216847+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:01:09.338365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006852+0000","last_change":"2023-10-09T06:44:51.048050+0000","last_active":"2023-10-09T06:45:56.006852+0000","last_peered":"2023-10-09T06:45:56.006852+0000","last_clean":"2023-10-09T06:45:56.006852+0000","last_became_active":"2023-10-09T06:44:51.047782+0000","last_became_peered":"2023-10-09T06:44:51.047782+0000","last_unstale":"2023-10-09T06:45:56.006852+0000","last_undegraded":"2023-10-09T06:45:56.006852+0000","last_fullsized":"2023-10-09T06:45:56.006852+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:57.140425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"77'161","reported_seq":967,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.428261+0000","last_change":"2023-10-09T06:45:00.111875+0000","last_active":"2023-10-09T06:45:56.428261+0000","last_peered":"2023-10-09T06:45:56.428261+0000","last_clean":"2023-10-09T06:45:56.428261+0000","last_became_active":"2023-10-09T06:45:00.111520+0000","last_became_peered":"2023-10-09T06:45:00.111520+0000","last_unstale":"2023-10-09T06:45:56.428261+0000","last_undegraded":"2023-10-09T06:45:56.428261+0000","last_fullsized":"2023-10-09T06:45:56.428261+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:17.264138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:17.264138+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:33.954026+0000","scrub_duration":0.001429189,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":365,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426369+0000","last_change":"2023-10-09T06:44:51.067000+0000","last_active":"2023-10-09T06:45:56.426369+0000","last_peered":"2023-10-09T06:45:56.426369+0000","last_clean":"2023-10-09T06:45:56.426369+0000","last_became_active":"2023-10-09T06:44:51.057845+0000","last_became_peered":"2023-10-09T06:44:51.057845+0000","last_unstale":"2023-10-09T06:45:56.426369+0000","last_undegraded":"2023-10-09T06:45:56.426369+0000","last_fullsized":"2023-10-09T06:45:56.426369+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:22.125999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"573'130","reported_seq":1177,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.008092+0000","last_change":"2023-10-09T06:45:00.111169+0000","last_active":"2023-10-09T06:45:56.008092+0000","last_peered":"2023-10-09T06:45:56.008092+0000","last_clean":"2023-10-09T06:45:56.008092+0000","last_became_active":"2023-10-09T06:45:00.110746+0000","last_became_peered":"2023-10-09T06:45:00.110746+0000","last_unstale":"2023-10-09T06:45:56.008092+0000","last_undegraded":"2023-10-09T06:45:56.008092+0000","last_fullsized":"2023-10-09T06:45:56.008092+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:45.413094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058176600000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":969,"num_read_kb":817,"num_write":79,"num_write_kb":5162,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":346,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425491+0000","last_change":"2023-10-09T06:44:51.056174+0000","last_active":"2023-10-09T06:45:56.425491+0000","last_peered":"2023-10-09T06:45:56.425491+0000","last_clean":"2023-10-09T06:45:56.425491+0000","last_became_active":"2023-10-09T06:44:51.055160+0000","last_became_peered":"2023-10-09T06:44:51.055160+0000","last_unstale":"2023-10-09T06:45:56.425491+0000","last_undegraded":"2023-10-09T06:45:56.425491+0000","last_fullsized":"2023-10-09T06:45:56.425491+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:19.799793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"77'161","reported_seq":962,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215639+0000","last_change":"2023-10-09T06:32:11.078039+0000","last_active":"2023-10-09T06:45:59.215639+0000","last_peered":"2023-10-09T06:45:59.215639+0000","last_clean":"2023-10-09T06:45:59.215639+0000","last_became_active":"2023-10-09T06:31:52.454516+0000","last_became_peered":"2023-10-09T06:31:52.454516+0000","last_unstale":"2023-10-09T06:45:59.215639+0000","last_undegraded":"2023-10-09T06:45:59.215639+0000","last_fullsized":"2023-10-09T06:45:59.215639+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":207,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":3,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:11.077896+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:11.077896+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:30:02.798815+0000","scrub_duration":0.0096690070000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"565'54","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.141210+0000","last_change":"2023-10-09T06:44:57.106119+0000","last_active":"2023-10-09T06:45:56.141210+0000","last_peered":"2023-10-09T06:45:56.141210+0000","last_clean":"2023-10-09T06:45:56.141210+0000","last_became_active":"2023-10-09T06:44:57.094050+0000","last_became_peered":"2023-10-09T06:44:57.094050+0000","last_unstale":"2023-10-09T06:45:56.141210+0000","last_undegraded":"2023-10-09T06:45:56.141210+0000","last_fullsized":"2023-10-09T06:45:56.141210+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:56:57.372718+0000","last_clean_scrub_stamp":"2023-10-09T05:56:57.372718+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:32:20.716304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000476019,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":438,"num_read_kb":386,"num_write":26,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":41,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006679+0000","last_change":"2023-10-09T06:44:51.078337+0000","last_active":"2023-10-09T06:45:56.006679+0000","last_peered":"2023-10-09T06:45:56.006679+0000","last_clean":"2023-10-09T06:45:56.006679+0000","last_became_active":"2023-10-09T06:44:51.049587+0000","last_became_peered":"2023-10-09T06:44:51.049587+0000","last_unstale":"2023-10-09T06:45:56.006679+0000","last_undegraded":"2023-10-09T06:45:56.006679+0000","last_fullsized":"2023-10-09T06:45:56.006679+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:19.138342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":607,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427422+0000","last_change":"2023-10-09T06:44:57.097133+0000","last_active":"2023-10-09T06:45:56.427422+0000","last_peered":"2023-10-09T06:45:56.427422+0000","last_clean":"2023-10-09T06:45:56.427422+0000","last_became_active":"2023-10-09T06:44:57.096732+0000","last_became_peered":"2023-10-09T06:44:57.096732+0000","last_unstale":"2023-10-09T06:45:56.427422+0000","last_undegraded":"2023-10-09T06:45:56.427422+0000","last_fullsized":"2023-10-09T06:45:56.427422+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:03.096466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"77'161","reported_seq":972,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425176+0000","last_change":"2023-10-09T06:45:00.108827+0000","last_active":"2023-10-09T06:45:56.425176+0000","last_peered":"2023-10-09T06:45:56.425176+0000","last_clean":"2023-10-09T06:45:56.425176+0000","last_became_active":"2023-10-09T06:45:00.108469+0000","last_became_peered":"2023-10-09T06:45:00.108469+0000","last_unstale":"2023-10-09T06:45:56.425176+0000","last_undegraded":"2023-10-09T06:45:56.425176+0000","last_fullsized":"2023-10-09T06:45:56.425176+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:25.148268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:25.148268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:57:19.934169+0000","scrub_duration":0.010632556,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":37,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.006939+0000","last_change":"2023-10-09T06:44:51.047675+0000","last_active":"2023-10-09T06:45:56.006939+0000","last_peered":"2023-10-09T06:45:56.006939+0000","last_clean":"2023-10-09T06:45:56.006939+0000","last_became_active":"2023-10-09T06:44:51.047410+0000","last_became_peered":"2023-10-09T06:44:51.047410+0000","last_unstale":"2023-10-09T06:45:56.006939+0000","last_undegraded":"2023-10-09T06:45:56.006939+0000","last_fullsized":"2023-10-09T06:45:56.006939+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:51.476378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":352,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425439+0000","last_change":"2023-10-09T06:45:29.893646+0000","last_active":"2023-10-09T06:45:56.425439+0000","last_peered":"2023-10-09T06:45:56.425439+0000","last_clean":"2023-10-09T06:45:56.425439+0000","last_became_active":"2023-10-09T06:45:29.893405+0000","last_became_peered":"2023-10-09T06:45:29.893405+0000","last_unstale":"2023-10-09T06:45:56.425439+0000","last_undegraded":"2023-10-09T06:45:56.425439+0000","last_fullsized":"2023-10-09T06:45:56.425439+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:21.383885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"77'161","reported_seq":853,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-10-09T06:45:59.215485+0000","last_change":"2023-10-09T06:45:00.109405+0000","last_active":"2023-10-09T06:45:59.215485+0000","last_peered":"2023-10-09T06:45:59.215485+0000","last_clean":"2023-10-09T06:45:59.215485+0000","last_became_active":"2023-10-09T06:45:00.109109+0000","last_became_peered":"2023-10-09T06:45:00.109109+0000","last_unstale":"2023-10-09T06:45:59.215485+0000","last_undegraded":"2023-10-09T06:45:59.215485+0000","last_fullsized":"2023-10-09T06:45:59.215485+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:20.145384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:20.145384+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:09.455204+0000","scrub_duration":0.019809789000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":165,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010415+0000","last_change":"2023-10-09T06:45:47.998014+0000","last_active":"2023-10-09T06:45:56.010415+0000","last_peered":"2023-10-09T06:45:56.010415+0000","last_clean":"2023-10-09T06:45:56.010415+0000","last_became_active":"2023-10-09T06:45:47.997684+0000","last_became_peered":"2023-10-09T06:45:47.997684+0000","last_unstale":"2023-10-09T06:45:56.010415+0000","last_undegraded":"2023-10-09T06:45:56.010415+0000","last_fullsized":"2023-10-09T06:45:56.010415+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:11.858074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":611,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427961+0000","last_change":"2023-10-09T06:27:36.633841+0000","last_active":"2023-10-09T06:45:56.427961+0000","last_peered":"2023-10-09T06:45:56.427961+0000","last_clean":"2023-10-09T06:45:56.427961+0000","last_became_active":"2023-10-09T06:27:36.633549+0000","last_became_peered":"2023-10-09T06:27:36.633549+0000","last_unstale":"2023-10-09T06:45:56.427961+0000","last_undegraded":"2023-10-09T06:45:56.427961+0000","last_fullsized":"2023-10-09T06:45:56.427961+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.109205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":165,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.010511+0000","last_change":"2023-10-09T06:45:47.995652+0000","last_active":"2023-10-09T06:45:56.010511+0000","last_peered":"2023-10-09T06:45:56.010511+0000","last_clean":"2023-10-09T06:45:56.010511+0000","last_became_active":"2023-10-09T06:45:47.994380+0000","last_became_peered":"2023-10-09T06:45:47.994380+0000","last_unstale":"2023-10-09T06:45:56.010511+0000","last_undegraded":"2023-10-09T06:45:56.010511+0000","last_fullsized":"2023-10-09T06:45:56.010511+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:41:12.797577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":641,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427336+0000","last_change":"2023-10-09T06:44:51.046177+0000","last_active":"2023-10-09T06:45:56.427336+0000","last_peered":"2023-10-09T06:45:56.427336+0000","last_clean":"2023-10-09T06:45:56.427336+0000","last_became_active":"2023-10-09T06:44:51.045909+0000","last_became_peered":"2023-10-09T06:44:51.045909+0000","last_unstale":"2023-10-09T06:45:56.427336+0000","last_undegraded":"2023-10-09T06:45:56.427336+0000","last_fullsized":"2023-10-09T06:45:56.427336+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:06.637385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"77'161","reported_seq":1000,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426926+0000","last_change":"2023-10-09T06:45:00.110332+0000","last_active":"2023-10-09T06:45:56.426926+0000","last_peered":"2023-10-09T06:45:56.426926+0000","last_clean":"2023-10-09T06:45:56.426926+0000","last_became_active":"2023-10-09T06:45:00.109980+0000","last_became_peered":"2023-10-09T06:45:00.109980+0000","last_unstale":"2023-10-09T06:45:56.426926+0000","last_undegraded":"2023-10-09T06:45:56.426926+0000","last_fullsized":"2023-10-09T06:45:56.426926+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:22.597880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:54:19.242593+0000","last_clean_scrub_stamp":"2023-10-09T06:32:22.597880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:18:04.079134+0000","scrub_duration":0.010645063999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":328,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.426255+0000","last_change":"2023-10-09T06:44:57.093892+0000","last_active":"2023-10-09T06:45:56.426255+0000","last_peered":"2023-10-09T06:45:56.426255+0000","last_clean":"2023-10-09T06:45:56.426255+0000","last_became_active":"2023-10-09T06:44:57.093537+0000","last_became_peered":"2023-10-09T06:44:57.093537+0000","last_unstale":"2023-10-09T06:45:56.426255+0000","last_undegraded":"2023-10-09T06:45:56.426255+0000","last_fullsized":"2023-10-09T06:45:56.426255+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:06.715364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":344,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.425699+0000","last_change":"2023-10-09T06:44:51.058622+0000","last_active":"2023-10-09T06:45:56.425699+0000","last_peered":"2023-10-09T06:45:56.425699+0000","last_clean":"2023-10-09T06:45:56.425699+0000","last_became_active":"2023-10-09T06:44:51.058147+0000","last_became_peered":"2023-10-09T06:44:51.058147+0000","last_unstale":"2023-10-09T06:45:56.425699+0000","last_undegraded":"2023-10-09T06:45:56.425699+0000","last_fullsized":"2023-10-09T06:45:56.425699+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:04.760476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"77'161","reported_seq":729,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007916+0000","last_change":"2023-10-09T06:32:21.006571+0000","last_active":"2023-10-09T06:45:56.007916+0000","last_peered":"2023-10-09T06:45:56.007916+0000","last_clean":"2023-10-09T06:45:56.007916+0000","last_became_active":"2023-10-09T06:32:19.747013+0000","last_became_peered":"2023-10-09T06:32:19.747013+0000","last_unstale":"2023-10-09T06:45:56.007916+0000","last_undegraded":"2023-10-09T06:45:56.007916+0000","last_fullsized":"2023-10-09T06:45:56.007916+0000","mapping_epoch":386,"log_start":"0'0","ondisk_log_start":"0'0","created":381,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":5,"last_scrub":"77'161","last_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_deep_scrub":"77'161","last_deep_scrub_stamp":"2023-10-09T06:32:21.006440+0000","last_clean_scrub_stamp":"2023-10-09T06:32:21.006440+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:13.950084+0000","scrub_duration":0.013826222000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":43,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.007088+0000","last_change":"2023-10-09T06:44:51.072116+0000","last_active":"2023-10-09T06:45:56.007088+0000","last_peered":"2023-10-09T06:45:56.007088+0000","last_clean":"2023-10-09T06:45:56.007088+0000","last_became_active":"2023-10-09T06:44:51.047845+0000","last_became_peered":"2023-10-09T06:44:51.047845+0000","last_unstale":"2023-10-09T06:45:56.007088+0000","last_undegraded":"2023-10-09T06:45:56.007088+0000","last_fullsized":"2023-10-09T06:45:56.007088+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:52.235741+0000","last_clean_scrub_stamp":"2023-10-09T05:57:52.235741+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:24.262662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":584,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-10-09T06:45:56.427538+0000","last_change":"2023-10-09T06:44:57.097289+0000","last_active":"2023-10-09T06:45:56.427538+0000","last_peered":"2023-10-09T06:45:56.427538+0000","last_clean":"2023-10-09T06:45:56.427538+0000","last_became_active":"2023-10-09T06:44:57.096952+0000","last_became_peered":"2023-10-09T06:44:57.096952+0000","last_unstale":"2023-10-09T06:45:56.427538+0000","last_undegraded":"2023-10-09T06:45:56.427538+0000","last_fullsized":"2023-10-09T06:45:56.427538+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T05:57:53.237061+0000","last_clean_scrub_stamp":"2023-10-09T05:57:53.237061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:20:45.990363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":20394,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5976,"num_read_kb":13193,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":4204614,"num_keys_recovered":13,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234592,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1498744,"kb_used_data":5936,"kb_used_omap":0,"kb_used_meta":1492800,"kb_avail":92246408,"statfs":{"total":95995035648,"available":94460321792,"internally_reserved":0,"allocated":6078464,"data_stored":4929714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1528627200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":2.7679999999999998,"15min":2.9769999999999999},"min":{"1min":0.33300000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.72699999999999998,"5min":76.727999999999994,"15min":196.28899999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":2.7970000000000002,"15min":3.0379999999999998},"min":{"1min":0.30599999999999999,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":0.66200000000000003,"5min":76.712999999999994,"15min":196.333},"last":0.65300000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":3.113,"15min":4.1719999999999997},"min":{"1min":0.36599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.627,"5min":169.51900000000001,"15min":208.626},"last":0.63},{"interface":"front","average":{"1min":0.53400000000000003,"5min":3.1629999999999998,"15min":3.5099999999999998},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.26100000000000001},"max":{"1min":0.98399999999999999,"5min":169.101,"15min":181.24199999999999},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Oct 9 06:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.252,"15min":4.5949999999999998},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.627,"5min":30.117000000000001,"15min":228.24000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.597,"15min":4.6980000000000004},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.30299999999999999},"max":{"1min":1.0189999999999999,"5min":30.245999999999999,"15min":228.291},"last":0.499}]},{"osd":3,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":2.5579999999999998,"15min":3.8239999999999998},"min":{"1min":0.379,"5min":0.308,"15min":0.27000000000000002},"max":{"1min":0.81100000000000005,"5min":59.847000000000001,"15min":175.40600000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.6800000000000002,"15min":3.7970000000000002},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32100000000000001},"max":{"1min":0.77400000000000002,"5min":59.698999999999998,"15min":175.30099999999999},"last":0.73599999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":2.8359999999999999,"15min":2.746},"min":{"1min":0.31,"5min":0.26600000000000001,"15min":0.23499999999999999},"max":{"1min":0.79400000000000004,"5min":130.774,"15min":225.041},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.49399999999999999,"5min":2.6669999999999998,"15min":2.5779999999999998},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.84599999999999997,"5min":130.72999999999999,"15min":225.244},"last":0.70599999999999996}]},{"osd":5,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.59699999999999998,"15min":2.1949999999999998},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26700000000000002},"max":{"1min":0.67700000000000005,"5min":4.8520000000000003,"15min":219.923},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.747,"15min":2.2200000000000002},"min":{"1min":0.39700000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.751,"5min":3.4409999999999998,"15min":219.86000000000001},"last":0.61399999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.859,"15min":3.0110000000000001},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":0.82599999999999996,"5min":138.477,"15min":222.179},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.58699999999999997,"5min":2.8809999999999998,"15min":2.944},"min":{"1min":0.33700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.81299999999999994,"5min":135.244,"15min":223.261},"last":0.76200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496228,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518500,"kb_used_data":10076,"kb_used_omap":0,"kb_used_meta":1508416,"kb_avail":92226652,"statfs":{"total":95995035648,"available":94440091648,"internally_reserved":0,"allocated":10317824,"data_stored":9188295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1544617984},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":2.1160000000000001,"15min":2.5990000000000002},"min":{"1min":0.34999999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.77800000000000002,"5min":45.613999999999997,"15min":75.061999999999998},"last":0.622},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.2679999999999998,"15min":3.5569999999999999},"min":{"1min":0.36699999999999999,"5min":0.28000000000000003,"15min":0.23599999999999999},"max":{"1min":0.89200000000000002,"5min":45.646999999999998,"15min":212.096},"last":0.56100000000000005}]},{"osd":1,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":1.458,"15min":3.1779999999999999},"min":{"1min":0.34599999999999997,"5min":0.27000000000000002,"15min":0.20899999999999999},"max":{"1min":0.94399999999999995,"5min":40.067,"15min":219.93199999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.505,"5min":1.3149999999999999,"15min":3.157},"min":{"1min":0.36499999999999999,"5min":0.27800000000000002,"15min":0.222},"max":{"1min":0.65100000000000002,"5min":40.085999999999999,"15min":219.94800000000001},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.2170000000000001,"15min":3.4500000000000002},"min":{"1min":0.379,"5min":0.28899999999999998,"15min":0.25},"max":{"1min":0.97199999999999998,"5min":7.4980000000000002,"15min":185.654},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.1870000000000001,"15min":3.3679999999999999},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":0.72799999999999998,"5min":7.4829999999999997,"15min":185.67699999999999},"last":0.81899999999999995}]},{"osd":3,"last update":"Mon Oct 9 06:45:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":3.609,"15min":3.7160000000000002},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74299999999999999,"5min":161.60300000000001,"15min":161.60300000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.53800000000000003,"5min":3.8450000000000002,"15min":3.6930000000000001},"min":{"1min":0.34499999999999997,"5min":0.34100000000000003,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":161.56299999999999,"15min":161.56299999999999},"last":0.74199999999999999}]},{"osd":4,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.67300000000000004,"15min":1.282},"min":{"1min":0.32500000000000001,"5min":0.25800000000000001,"15min":0.23799999999999999},"max":{"1min":0.77200000000000002,"5min":5.6890000000000001,"15min":86.171999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.72499999999999998,"15min":2.0419999999999998},"min":{"1min":0.36399999999999999,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":0.81499999999999995,"5min":8.2989999999999995,"15min":212.03399999999999},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.876,"15min":1.155},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.27000000000000002},"max":{"1min":0.79600000000000004,"5min":10.435,"15min":10.435},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.76400000000000001,"15min":1.282},"min":{"1min":0.34899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.78600000000000003,"5min":8.3369999999999997,"15min":12.244999999999999},"last":0.64400000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.887,"15min":1.9610000000000001},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.27000000000000002},"max":{"1min":0.76900000000000002,"5min":109.83199999999999,"15min":165.37},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.9710000000000001,"15min":2.202},"min":{"1min":0.46899999999999997,"5min":0.34300000000000003,"15min":0.29099999999999998},"max":{"1min":0.76900000000000002,"5min":109.911,"15min":165.40199999999999},"last":0.52900000000000003}]}]},{"osd":1,"up_from":110,"seq":472446402965,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143720,"kb_used_data":11296,"kb_used_omap":4,"kb_used_meta":1132411,"kb_avail":92601432,"statfs":{"total":95995035648,"available":94823866368,"internally_reserved":0,"allocated":11567104,"data_stored":9043533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1159589774},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":4.1879999999999997,"15min":3.6629999999999998},"min":{"1min":0.221,"5min":0.21299999999999999,"15min":0.182},"max":{"1min":0.45300000000000001,"5min":114.97799999999999,"15min":143.465},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.34399999999999997,"5min":4.7789999999999999,"15min":4.048},"min":{"1min":0.26100000000000001,"5min":0.20200000000000001,"15min":0.186},"max":{"1min":0.51100000000000001,"5min":114.934,"15min":143.66399999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.88900000000000001,"15min":2.0350000000000001},"min":{"1min":0.25600000000000001,"5min":0.193,"15min":0.184},"max":{"1min":0.52800000000000002,"5min":8.0129999999999999,"15min":138.452},"last":0.312},{"interface":"front","average":{"1min":0.40999999999999998,"5min":2.1240000000000001,"15min":2.6720000000000002},"min":{"1min":0.29699999999999999,"5min":0.224,"15min":0.20599999999999999},"max":{"1min":1.147,"5min":20.085000000000001,"15min":138.47499999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:45:40 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":2.2850000000000001,"15min":2.4369999999999998},"min":{"1min":0.253,"5min":0.25,"15min":0.219},"max":{"1min":0.48199999999999998,"5min":27.655999999999999,"15min":27.655999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.38,"5min":1.9239999999999999,"15min":1.7629999999999999},"min":{"1min":0.30199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.55600000000000005,"5min":27.745999999999999,"15min":27.745999999999999},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":2.7850000000000001,"15min":2.9630000000000001},"min":{"1min":0.40100000000000002,"5min":0.316,"15min":0.28399999999999997},"max":{"1min":0.91600000000000004,"5min":180.59700000000001,"15min":180.59700000000001},"last":0.504},{"interface":"front","average":{"1min":0.621,"5min":2.7370000000000001,"15min":2.7090000000000001},"min":{"1min":0.44500000000000001,"5min":0.28799999999999998,"15min":0.28499999999999998},"max":{"1min":1.0489999999999999,"5min":180.70599999999999,"15min":180.70599999999999},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.93899999999999995,"15min":2.177},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.253},"max":{"1min":0.747,"5min":7.9420000000000002,"15min":119.408},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.2310000000000001,"15min":2.3959999999999999},"min":{"1min":0.39800000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.89000000000000001,"5min":9.6820000000000004,"15min":119.483},"last":0.42099999999999999}]},{"osd":6,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":4.242,"15min":3.4049999999999998},"min":{"1min":0.38700000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.84099999999999997,"5min":183.12100000000001,"15min":183.12100000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":4.2919999999999998,"15min":3.7069999999999999},"min":{"1min":0.371,"5min":0.36699999999999999,"15min":0.316},"max":{"1min":0.83299999999999996,"5min":183.13900000000001,"15min":183.13900000000001},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":2.8210000000000002,"15min":3.4359999999999999},"min":{"1min":0.41399999999999998,"5min":0.40699999999999997,"15min":0.34300000000000003},"max":{"1min":0.879,"5min":178.74000000000001,"15min":178.74000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":2.4449999999999998,"15min":2.948},"min":{"1min":0.434,"5min":0.371,"15min":0.33100000000000002},"max":{"1min":0.875,"5min":178.75299999999999,"15min":178.75299999999999},"last":0.51200000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024399,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1201040,"kb_used_data":8008,"kb_used_omap":5,"kb_used_meta":1193018,"kb_avail":92544112,"statfs":{"total":95995035648,"available":94765170688,"internally_reserved":0,"allocated":8200192,"data_stored":5236884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1221651066},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 06:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":6.0910000000000002,"15min":4.867},"min":{"1min":0.248,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":0.48199999999999998,"5min":263.86799999999999,"15min":263.86799999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":5.8929999999999998,"15min":5.0590000000000002},"min":{"1min":0.27400000000000002,"5min":0.20200000000000001,"15min":0.187},"max":{"1min":0.57599999999999996,"5min":263.51999999999998,"15min":263.51999999999998},"last":0.48299999999999998}]},{"osd":2,"last update":"Mon Oct 9 06:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":3.2250000000000001,"15min":3.8039999999999998},"min":{"1min":0.28100000000000003,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":0.52600000000000002,"5min":147.61099999999999,"15min":147.61099999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.4369999999999998,"15min":4.04},"min":{"1min":0.27300000000000002,"5min":0.223,"15min":0.19600000000000001},"max":{"1min":0.502,"5min":150.12700000000001,"15min":150.12700000000001},"last":0.438}]},{"osd":3,"last update":"Mon Oct 9 06:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.5089999999999999,"15min":2.7799999999999998},"min":{"1min":0.27400000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.81399999999999995,"5min":48.216000000000001,"15min":95.555000000000007},"last":0.496},{"interface":"front","average":{"1min":0.45300000000000001,"5min":1.645,"15min":2.8340000000000001},"min":{"1min":0.313,"5min":0.26300000000000001,"15min":0.19800000000000001},"max":{"1min":0.96599999999999997,"5min":48.261000000000003,"15min":97.605000000000004},"last":0.50800000000000001}]},{"osd":4,"last update":"Mon Oct 9 06:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.139,"15min":2.9249999999999998},"min":{"1min":0.40999999999999998,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":0.88800000000000001,"5min":7.9649999999999999,"15min":200.654},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.1399999999999999,"15min":3.1949999999999998},"min":{"1min":0.38600000000000001,"5min":0.317,"15min":0.29099999999999998},"max":{"1min":0.90700000000000003,"5min":8.2850000000000001,"15min":200.86600000000001},"last":0.80900000000000005}]},{"osd":5,"last update":"Mon Oct 9 06:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.2,"15min":1.905},"min":{"1min":0.40000000000000002,"5min":0.34799999999999998,"15min":0.24299999999999999},"max":{"1min":0.90800000000000003,"5min":8.0120000000000005,"15min":80.885999999999996},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.139,"15min":1.905},"min":{"1min":0.377,"5min":0.33600000000000002,"15min":0.31900000000000001},"max":{"1min":0.86299999999999999,"5min":8.4019999999999992,"15min":80.438999999999993},"last":0.82999999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":1.113,"15min":2.1600000000000001},"min":{"1min":0.41499999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.93000000000000005,"5min":8.4309999999999992,"15min":118.056},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.2050000000000001,"15min":2.0640000000000001},"min":{"1min":0.46899999999999997,"5min":0.41099999999999998,"15min":0.29899999999999999},"max":{"1min":0.84099999999999997,"5min":8.0449999999999999,"15min":118.274},"last":0.75800000000000001}]},{"osd":7,"last update":"Mon Oct 9 06:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.224,"15min":1.786},"min":{"1min":0.45800000000000002,"5min":0.40400000000000003,"15min":0.35699999999999998},"max":{"1min":0.871,"5min":8.2490000000000006,"15min":22.879000000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.216,"15min":2.0259999999999998},"min":{"1min":0.49299999999999999,"5min":0.34499999999999997,"15min":0.34200000000000003},"max":{"1min":0.89600000000000002,"5min":8.4570000000000007,"15min":22.683},"last":0.79200000000000004}]}]},{"osd":2,"up_from":128,"seq":549755814268,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078796,"kb_used_data":7364,"kb_used_omap":7,"kb_used_meta":1071416,"kb_avail":92666356,"statfs":{"total":95995035648,"available":94890348544,"internally_reserved":0,"allocated":7540736,"data_stored":5099456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1097130065},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":3.4169999999999998,"15min":6.0629999999999997},"min":{"1min":0.19600000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.46700000000000003,"5min":89.287000000000006,"15min":214.595},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.374,"5min":3.206,"15min":5.6859999999999999},"min":{"1min":0.27900000000000003,"5min":0.221,"15min":0.17000000000000001},"max":{"1min":0.502,"5min":89.225999999999999,"15min":214.94300000000001},"last":0.309}]},{"osd":1,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":6.4500000000000002,"15min":5.218},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.41999999999999998,"5min":299.988,"15min":299.988},"last":0.246},{"interface":"front","average":{"1min":0.35999999999999999,"5min":6.694,"15min":5.1849999999999996},"min":{"1min":0.245,"5min":0.219,"15min":0.183},"max":{"1min":0.46700000000000003,"5min":300.30500000000001,"15min":300.30500000000001},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Oct 9 06:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":3.569,"15min":3.4460000000000002},"min":{"1min":0.26100000000000001,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.53000000000000003,"5min":74.317999999999998,"15min":79.566000000000003},"last":0.372},{"interface":"front","average":{"1min":0.45300000000000001,"5min":3.468,"15min":3.5129999999999999},"min":{"1min":0.28499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.997,"5min":74.355999999999995,"15min":79.614000000000004},"last":0.35399999999999998}]},{"osd":4,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.85699999999999998,"15min":1.9730000000000001},"min":{"1min":0.45900000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.216,"5min":9.5679999999999996,"15min":79.643000000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.94299999999999995,"15min":2.1640000000000001},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.29799999999999999},"max":{"1min":1.228,"5min":7.0910000000000002,"15min":82.872},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.93400000000000005,"15min":1.9690000000000001},"min":{"1min":0.42699999999999999,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":0.73599999999999999,"5min":10.914999999999999,"15min":82.834999999999994},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.018,"15min":2.1030000000000002},"min":{"1min":0.39200000000000002,"5min":0.33200000000000002,"15min":0.32000000000000001},"max":{"1min":0.65100000000000002,"5min":11.481999999999999,"15min":82.921999999999997},"last":0.58899999999999997}]},{"osd":6,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.95699999999999996,"15min":2.7589999999999999},"min":{"1min":0.42399999999999999,"5min":0.34000000000000002,"15min":0.313},"max":{"1min":1.367,"5min":7.1669999999999998,"15min":123.47799999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.056,"15min":2.9169999999999998},"min":{"1min":0.41299999999999998,"5min":0.36799999999999999,"15min":0.313},"max":{"1min":1.2030000000000001,"5min":6.3689999999999998,"15min":122.517},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 06:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":3.0800000000000001,"15min":3.1299999999999999},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":1.143,"5min":132.06399999999999,"15min":132.06399999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":3.056,"15min":2.7589999999999999},"min":{"1min":0.38800000000000001,"5min":0.35899999999999999,"15min":0.35499999999999998},"max":{"1min":0.66800000000000004,"5min":132.196,"15min":132.196},"last":0.63800000000000001}]}]},{"osd":3,"up_from":149,"seq":639950127465,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":948364,"kb_used_data":7300,"kb_used_omap":2,"kb_used_meta":941053,"kb_avail":92796788,"statfs":{"total":95995035648,"available":95023910912,"internally_reserved":0,"allocated":7475200,"data_stored":5153314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":963638325},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":3.431,"15min":4.0439999999999996},"min":{"1min":0.24199999999999999,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.58399999999999996,"5min":95.281999999999996,"15min":161.821},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.35799999999999998,"5min":4.1920000000000002,"15min":4.1369999999999996},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.17799999999999999},"max":{"1min":0.47999999999999998,"5min":98.572000000000003,"15min":163.52000000000001},"last":0.27800000000000002}]},{"osd":1,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":3.5419999999999998,"15min":4.1440000000000001},"min":{"1min":0.27000000000000002,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.54600000000000004,"5min":102.468,"15min":184.012},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.379,"5min":3.5390000000000001,"15min":4.532},"min":{"1min":0.255,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.66800000000000004,"5min":102.58,"15min":184.703},"last":0.34300000000000003}]},{"osd":2,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":1.597,"15min":3.6909999999999998},"min":{"1min":0.26800000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.59499999999999997,"5min":15.807,"15min":182.44900000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2829999999999999,"15min":3.79},"min":{"1min":0.27700000000000002,"5min":0.22,"15min":0.219},"max":{"1min":0.70399999999999996,"5min":14.9,"15min":182.56399999999999},"last":0.40300000000000002}]},{"osd":4,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.081,"15min":2.9900000000000002},"min":{"1min":0.44400000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79500000000000004,"5min":9.7390000000000008,"15min":178.22200000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.248,"15min":3.0630000000000002},"min":{"1min":0.39400000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.79400000000000004,"5min":7.7999999999999998,"15min":180.43600000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.3440000000000001,"15min":3.4220000000000002},"min":{"1min":0.433,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80600000000000005,"5min":7.8380000000000001,"15min":177.56299999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.4890000000000001,"15min":2.9790000000000001},"min":{"1min":0.45800000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.83599999999999997,"5min":15.743,"15min":177.59399999999999},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.3959999999999999,"15min":1.9870000000000001},"min":{"1min":0.41499999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82499999999999996,"5min":7.3159999999999998,"15min":48.270000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4910000000000001,"15min":1.804},"min":{"1min":0.434,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":0.76000000000000001,"5min":15.69,"15min":48.155000000000001},"last":0.56699999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:45:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.9750000000000001,"15min":2.1110000000000002},"min":{"1min":0.45400000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.85099999999999998,"5min":99.248000000000005,"15min":99.248000000000005},"last":0.5},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.1859999999999999,"15min":2.5579999999999998},"min":{"1min":0.39900000000000002,"5min":0.38100000000000001,"15min":0.33400000000000002},"max":{"1min":0.86299999999999999,"5min":99.457999999999998,"15min":99.457999999999998},"last":0.51300000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019501,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1493416,"kb_used_data":9760,"kb_used_omap":0,"kb_used_meta":1483648,"kb_avail":92251736,"statfs":{"total":95995035648,"available":94465777664,"internally_reserved":0,"allocated":9994240,"data_stored":8860973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1519255552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.7599999999999998,"15min":6.6520000000000001},"min":{"1min":0.33500000000000002,"5min":0.23400000000000001,"15min":0.21199999999999999},"max":{"1min":0.84399999999999997,"5min":69.165000000000006,"15min":287.58800000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":2.839,"15min":6.8540000000000001},"min":{"1min":0.377,"5min":0.28199999999999997,"15min":0.23200000000000001},"max":{"1min":0.82999999999999996,"5min":69.147999999999996,"15min":287.57100000000003},"last":0.57299999999999995}]},{"osd":1,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":1.8999999999999999,"15min":3.0129999999999999},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.82099999999999995,"5min":69.644000000000005,"15min":163.392},"last":0.497},{"interface":"front","average":{"1min":0.53700000000000003,"5min":2.0430000000000001,"15min":3.1070000000000002},"min":{"1min":0.35099999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.92800000000000005,"5min":69.572000000000003,"15min":163.42400000000001},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.98899999999999999,"15min":3.968},"min":{"1min":0.35199999999999998,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":0.77300000000000002,"5min":6.6879999999999997,"15min":175.97900000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.099,"15min":4.0839999999999996},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77100000000000002,"5min":6.0110000000000001,"15min":175.875},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 06:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.839,"15min":4.3360000000000003},"min":{"1min":0.41899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.745,"5min":108.19799999999999,"15min":161.19300000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.624,"5min":2.8050000000000002,"15min":4.1600000000000001},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.79800000000000004,"5min":112.21899999999999,"15min":161.934},"last":0.45500000000000002}]},{"osd":5,"last update":"Mon Oct 9 06:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.627,"15min":0.84699999999999998},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.252},"max":{"1min":0.78900000000000003,"5min":6.7869999999999999,"15min":9.3979999999999997},"last":0.442},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.77700000000000002,"15min":0.97999999999999998},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":0.91200000000000003,"5min":5.9820000000000002,"15min":9.5009999999999994},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 06:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.581,"15min":1.655},"min":{"1min":0.33500000000000002,"5min":0.317,"15min":0.26500000000000001},"max":{"1min":0.79100000000000004,"5min":80.549000000000007,"15min":115.70999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.609,"15min":1.7649999999999999},"min":{"1min":0.35299999999999998,"5min":0.34399999999999997,"15min":0.32200000000000001},"max":{"1min":0.85099999999999998,"5min":83.218000000000004,"15min":115.68600000000001},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 06:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.514,"15min":1.8300000000000001},"min":{"1min":0.443,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.80400000000000005,"5min":76.081000000000003,"15min":106.801},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.526,"15min":1.944},"min":{"1min":0.49099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82599999999999996,"5min":76.283000000000001,"15min":106.827},"last":0.64500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757866,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485412,"kb_used_data":5916,"kb_used_omap":0,"kb_used_meta":1479488,"kb_avail":92259740,"statfs":{"total":95995035648,"available":94473973760,"internally_reserved":0,"allocated":6057984,"data_stored":4925084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514995712},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.7769999999999999,"15min":5.2519999999999998},"min":{"1min":0.34000000000000002,"5min":0.23200000000000001,"15min":0.20799999999999999},"max":{"1min":0.70899999999999996,"5min":62.884999999999998,"15min":294.89699999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.905,"15min":5.5250000000000004},"min":{"1min":0.35099999999999998,"5min":0.25900000000000001,"15min":0.215},"max":{"1min":0.78500000000000003,"5min":62.195,"15min":294.30399999999997},"last":0.53400000000000003}]},{"osd":1,"last update":"Mon Oct 9 06:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.7430000000000001,"15min":6.1150000000000002},"min":{"1min":0.379,"5min":0.253,"15min":0.22},"max":{"1min":0.78800000000000003,"5min":69.287999999999997,"15min":188.34100000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.499,"5min":1.7490000000000001,"15min":6.0490000000000004},"min":{"1min":0.371,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":67.245999999999995,"15min":188.839},"last":0.498}]},{"osd":2,"last update":"Mon Oct 9 06:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":4.1500000000000004,"15min":4.5970000000000004},"min":{"1min":0.375,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":0.70899999999999996,"5min":145.20400000000001,"15min":145.20400000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":4.1660000000000004,"15min":4.5289999999999999},"min":{"1min":0.41699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.72299999999999998,"5min":145.71899999999999,"15min":145.71899999999999},"last":0.61499999999999999}]},{"osd":3,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":2.1829999999999998,"15min":4.2380000000000004},"min":{"1min":0.39400000000000002,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":0.754,"5min":63.578000000000003,"15min":220.41800000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.9890000000000001,"15min":3.9929999999999999},"min":{"1min":0.5,"5min":0.39700000000000002,"15min":0.29299999999999998},"max":{"1min":0.81799999999999995,"5min":62.113999999999997,"15min":220.45500000000001},"last":0.434}]},{"osd":4,"last update":"Mon Oct 9 06:46:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":1.5860000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.26100000000000001,"15min":0.247},"max":{"1min":0.71799999999999997,"5min":74.725999999999999,"15min":79.043000000000006},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.6040000000000001,"15min":1.613},"min":{"1min":0.36599999999999999,"5min":0.25700000000000001,"15min":0.251},"max":{"1min":1.0880000000000001,"5min":74.777000000000001,"15min":79.165999999999997},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 06:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":1.788,"15min":1.6459999999999999},"min":{"1min":0.45900000000000002,"5min":0.29999999999999999,"15min":0.251},"max":{"1min":0.76900000000000002,"5min":81.254000000000005,"15min":81.254000000000005},"last":0.37},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.754,"15min":1.6890000000000001},"min":{"1min":0.31900000000000001,"5min":0.29899999999999999,"15min":0.26600000000000001},"max":{"1min":0.76000000000000001,"5min":79.216999999999999,"15min":80.781999999999996},"last":0.56499999999999995}]},{"osd":7,"last update":"Mon Oct 9 06:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.85499999999999998,"15min":1.3759999999999999},"min":{"1min":0.44500000000000001,"5min":0.311,"15min":0.29099999999999998},"max":{"1min":0.84199999999999997,"5min":5.4729999999999999,"15min":78.890000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.79500000000000004,"15min":1.2370000000000001},"min":{"1min":0.41799999999999998,"5min":0.317,"15min":0.28599999999999998},"max":{"1min":1.167,"5min":3.359,"15min":79.138999999999996},"last":0.48299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T06:46:36.942 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T06:46:36.942 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:46:37.012 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=f412d45efb6248bfab995ad413f30745 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T06:46:37.015 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T06:46:37.025 INFO:tasks.cephadm:Teardown begin 2023-10-09T06:46:37.026 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:46:37.106 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:46:37.169 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T06:46:37.169 DEBUG:teuthology.orchestra.run.smithi107:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:46:37.187 DEBUG:teuthology.orchestra.run.smithi150:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T06:46:37.206 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T06:46:37.207 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T06:46:37.207 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T06:46:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:37 smithi150 ceph-mon[194178]: pgmap v1829: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:37.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:37 smithi150 ceph-mon[194178]: from='client.38376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:37.518 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:37 smithi107 ceph-mon[220921]: pgmap v1829: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:37.519 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:37 smithi107 ceph-mon[220921]: from='client.38376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:37.519 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 systemd[1]: Stopping Ceph mon.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:37.519 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 ceph-mon[212750]: pgmap v1829: 103 pgs: 103 active+clean; 4.6 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail 2023-10-09T06:46:37.519 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 ceph-mon[212750]: from='client.38376 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T06:46:37.894 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[212706]: 2023-10-09T06:46:37.517+0000 7f7c7ed04700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[212706]: 2023-10-09T06:46:37.517+0000 7f7c7ed04700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE Bus STOPPING 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE Bus STOPPED 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE Bus STARTING 2023-10-09T06:46:37.895 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE Serving on http://:::9283 2023-10-09T06:46:37.896 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:37 smithi107 conmon[200038]: [09/Oct/2023:06:46:37] ENGINE Bus STARTED 2023-10-09T06:46:38.151 INFO:journalctl@ceph.mon.a.smithi107.stdout:Oct 09 06:46:37 smithi107 bash[678567]: 7f66f5126e5d2be5eec38dce6b0768742764d739ed0be71d0bed1918af4cbad3 2023-10-09T06:46:38.534 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T06:46:38.588 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:38.588 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T06:46:38.588 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T06:46:38.588 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T06:46:38.901 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:38 smithi107 systemd[1]: Stopping Ceph mon.c for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:39.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:38 smithi107 conmon[220891]: 2023-10-09T06:46:38.899+0000 7fd32ffd3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:39.151 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:38 smithi107 conmon[220891]: 2023-10-09T06:46:38.899+0000 7fd32ffd3700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:46:39.650 INFO:journalctl@ceph.mon.c.smithi107.stdout:Oct 09 06:46:39 smithi107 bash[678880]: 6c90cc5e902c4248629aeee86d9a59b278927dcf48dd15acf60daf19b58f0574 2023-10-09T06:46:39.792 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T06:46:39.830 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:39.830 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T06:46:39.831 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T06:46:39.831 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T06:46:40.116 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:39 smithi150 systemd[1]: Stopping Ceph mon.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:40 smithi150 conmon[194154]: 2023-10-09T06:46:40.114+0000 7f97b1121700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:40.404 INFO:journalctl@ceph.mon.b.smithi150.stdout:Oct 09 06:46:40 smithi150 conmon[194154]: 2023-10-09T06:46:40.114+0000 7f97b1121700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T06:46:41.062 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T06:46:41.110 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:41.110 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T06:46:41.110 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T06:46:41.111 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T06:46:41.392 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:41 smithi107 systemd[1]: Stopping Ceph mgr.y for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:41.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:41 smithi107 conmon[200038]: 2023-10-09T06:46:41.391+0000 7f54bbed7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:41.651 INFO:journalctl@ceph.mgr.y.smithi107.stdout:Oct 09 06:46:41 smithi107 conmon[200038]: 2023-10-09T06:46:41.391+0000 7f54bbed7700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T06:46:42.290 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T06:46:42.328 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:42.328 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T06:46:42.328 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T06:46:42.328 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T06:46:42.654 INFO:journalctl@ceph.mgr.x.smithi150.stdout:Oct 09 06:46:42 smithi150 systemd[1]: Stopping Ceph mgr.x for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:43.331 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T06:46:43.367 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:43.367 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T06:46:43.368 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T06:46:43.368 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T06:46:43.651 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:46:43 smithi107 systemd[1]: Stopping Ceph osd.0 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:43.651 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:46:43 smithi107 conmon[238556]: 2023-10-09T06:46:43.641+0000 7f98f189a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:43.651 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:46:43 smithi107 conmon[238556]: 2023-10-09T06:46:43.641+0000 7f98f189a700 -1 osd.0 604 *** Got signal Terminated *** 2023-10-09T06:46:43.651 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:46:43 smithi107 conmon[238556]: 2023-10-09T06:46:43.641+0000 7f98f189a700 -1 osd.0 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:46:50.151 INFO:journalctl@ceph.osd.0.smithi107.stdout:Oct 09 06:46:49 smithi107 bash[679478]: f47405cf3d52a3deb64f63cce517eb6b5aa39884b62173f5bd10a3103d607c1c 2023-10-09T06:46:52.263 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T06:46:52.304 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:46:52.304 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T06:46:52.304 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T06:46:52.305 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T06:46:52.611 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:46:52 smithi107 systemd[1]: Stopping Ceph osd.1 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:46:52.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:46:52 smithi107 conmon[254053]: 2023-10-09T06:46:52.610+0000 7f4729ec9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:46:52.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:46:52 smithi107 conmon[254053]: 2023-10-09T06:46:52.610+0000 7f4729ec9700 -1 osd.1 604 *** Got signal Terminated *** 2023-10-09T06:46:52.901 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:46:52 smithi107 conmon[254053]: 2023-10-09T06:46:52.610+0000 7f4729ec9700 -1 osd.1 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:46:58.651 INFO:journalctl@ceph.osd.1.smithi107.stdout:Oct 09 06:46:58 smithi107 bash[680060]: a3526ce24032259533e57873885a0d6ca44fe0856e142f63a434ce3cd6ac2c25 2023-10-09T06:47:00.306 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T06:47:00.348 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:00.348 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T06:47:00.348 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T06:47:00.349 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T06:47:00.651 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:47:00 smithi107 systemd[1]: Stopping Ceph osd.2 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:01.151 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:47:00 smithi107 conmon[264385]: 2023-10-09T06:47:00.658+0000 7f0ae64ee700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:01.151 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:47:00 smithi107 conmon[264385]: 2023-10-09T06:47:00.658+0000 7f0ae64ee700 -1 osd.2 604 *** Got signal Terminated *** 2023-10-09T06:47:01.151 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:47:00 smithi107 conmon[264385]: 2023-10-09T06:47:00.658+0000 7f0ae64ee700 -1 osd.2 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:07.151 INFO:journalctl@ceph.osd.2.smithi107.stdout:Oct 09 06:47:06 smithi107 bash[680634]: eea84a85a04b98a7842bc47d2086af7d3dd3bb2e4ce1aaff367bc837803bb3e0 2023-10-09T06:47:08.326 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T06:47:08.367 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:08.367 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T06:47:08.367 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T06:47:08.368 DEBUG:teuthology.orchestra.run.smithi107:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T06:47:08.651 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:08 smithi107 systemd[1]: Stopping Ceph osd.3 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:09.151 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:08 smithi107 conmon[274179]: 2023-10-09T06:47:08.668+0000 7fbafbd29700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:09.151 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:08 smithi107 conmon[274179]: 2023-10-09T06:47:08.668+0000 7fbafbd29700 -1 osd.3 604 *** Got signal Terminated *** 2023-10-09T06:47:09.151 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:08 smithi107 conmon[274179]: 2023-10-09T06:47:08.668+0000 7fbafbd29700 -1 osd.3 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:11.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:11 smithi150 conmon[132350]: 2023-10-09T06:47:11.151+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:12.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:12 smithi150 conmon[132350]: 2023-10-09T06:47:12.108+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:12.404 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:12 smithi150 conmon[122886]: 2023-10-09T06:47:12.302+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:12.904 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:12 smithi150 conmon[114006]: 2023-10-09T06:47:12.623+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:12.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:12 smithi150 conmon[141446]: 2023-10-09T06:47:12.518+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:13.151 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:12 smithi107 conmon[274179]: 2023-10-09T06:47:12.742+0000 7fbaf631b700 -1 osd.3 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:48.161533+0000 front 2023-10-09T06:46:48.161564+0000 (oldest deadline 2023-10-09T06:47:12.162110+0000) 2023-10-09T06:47:13.345 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:13 smithi150 conmon[132350]: 2023-10-09T06:47:13.089+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:13.646 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:13 smithi150 conmon[122886]: 2023-10-09T06:47:13.344+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:13.647 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:13 smithi150 conmon[141446]: 2023-10-09T06:47:13.539+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:13.904 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:13 smithi150 conmon[114006]: 2023-10-09T06:47:13.646+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:14.357 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:14 smithi150 conmon[132350]: 2023-10-09T06:47:14.092+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:14.638 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:14 smithi150 conmon[122886]: 2023-10-09T06:47:14.356+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:14.638 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:14 smithi150 conmon[141446]: 2023-10-09T06:47:14.564+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:14.901 INFO:journalctl@ceph.osd.3.smithi107.stdout:Oct 09 06:47:14 smithi107 bash[681207]: 069ecca357f7c74ca1a492ed3b0cb931ef40fade26905d11a12565a32983bea2 2023-10-09T06:47:14.904 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:14 smithi150 conmon[114006]: 2023-10-09T06:47:14.637+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:15.372 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:15 smithi150 conmon[132350]: 2023-10-09T06:47:15.089+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:15.654 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:15 smithi150 conmon[114006]: 2023-10-09T06:47:15.614+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:15.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:15 smithi150 conmon[141446]: 2023-10-09T06:47:15.538+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:15.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:15 smithi150 conmon[122886]: 2023-10-09T06:47:15.371+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:16.348 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[132350]: 2023-10-09T06:47:16.048+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:16.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T06:47:16.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:16.407 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T06:47:16.407 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T06:47:16.408 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T06:47:16.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[122886]: 2023-10-09T06:47:16.348+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:16.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[141446]: 2023-10-09T06:47:16.580+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:16.655 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:16 smithi150 systemd[1]: Stopping Ceph osd.4 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:16.655 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[114006]: 2023-10-09T06:47:16.578+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:17.026 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[114006]: 2023-10-09T06:47:16.684+0000 7f0409c9b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:17.027 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[114006]: 2023-10-09T06:47:16.684+0000 7f0409c9b700 -1 osd.4 604 *** Got signal Terminated *** 2023-10-09T06:47:17.027 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:16 smithi150 conmon[114006]: 2023-10-09T06:47:16.684+0000 7f0409c9b700 -1 osd.4 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:17.396 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:17 smithi150 conmon[132350]: 2023-10-09T06:47:17.025+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:17.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:17 smithi150 conmon[122886]: 2023-10-09T06:47:17.396+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:17.655 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:17 smithi150 conmon[114006]: 2023-10-09T06:47:17.622+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:17.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:17 smithi150 conmon[141446]: 2023-10-09T06:47:17.543+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:18.391 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:18 smithi150 conmon[132350]: 2023-10-09T06:47:18.041+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:18.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:18 smithi150 conmon[122886]: 2023-10-09T06:47:18.391+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:18.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:18 smithi150 conmon[141446]: 2023-10-09T06:47:18.562+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:19.073 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:18 smithi150 conmon[114006]: 2023-10-09T06:47:18.656+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:19.074 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:18 smithi150 conmon[114006]: 2023-10-09T06:47:18.656+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.722348+0000 front 2023-10-09T06:46:55.722372+0000 (oldest deadline 2023-10-09T06:47:18.622212+0000) 2023-10-09T06:47:19.379 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:19 smithi150 conmon[132350]: 2023-10-09T06:47:19.072+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:19.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:19 smithi150 conmon[122886]: 2023-10-09T06:47:19.378+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:19.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:19 smithi150 conmon[141446]: 2023-10-09T06:47:19.544+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:20.097 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:19 smithi150 conmon[114006]: 2023-10-09T06:47:19.700+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:20.097 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:19 smithi150 conmon[114006]: 2023-10-09T06:47:19.700+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.722348+0000 front 2023-10-09T06:46:55.722372+0000 (oldest deadline 2023-10-09T06:47:18.622212+0000) 2023-10-09T06:47:20.362 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[132350]: 2023-10-09T06:47:20.096+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:20.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[122886]: 2023-10-09T06:47:20.361+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:20.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[141446]: 2023-10-09T06:47:20.504+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:20.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[141446]: 2023-10-09T06:47:20.504+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:21.058 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[114006]: 2023-10-09T06:47:20.704+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:44.020299+0000 front 2023-10-09T06:46:44.020277+0000 (oldest deadline 2023-10-09T06:47:12.220940+0000) 2023-10-09T06:47:21.059 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:20 smithi150 conmon[114006]: 2023-10-09T06:47:20.704+0000 7f0407a9c700 -1 osd.4 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.722348+0000 front 2023-10-09T06:46:55.722372+0000 (oldest deadline 2023-10-09T06:47:18.622212+0000) 2023-10-09T06:47:21.360 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:21 smithi150 conmon[132350]: 2023-10-09T06:47:21.057+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:21.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:21 smithi150 conmon[122886]: 2023-10-09T06:47:21.360+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:21.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:21 smithi150 conmon[141446]: 2023-10-09T06:47:21.512+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:21.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:21 smithi150 conmon[141446]: 2023-10-09T06:47:21.512+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:22.319 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[132350]: 2023-10-09T06:47:22.009+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:22.654 INFO:journalctl@ceph.osd.4.smithi150.stdout:Oct 09 06:47:22 smithi150 bash[403207]: 08e858ffb7cef8a76834d14a8da74ae1664090b200a97231b9b0a314255b7ae6 2023-10-09T06:47:22.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[122886]: 2023-10-09T06:47:22.318+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:22.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[122886]: 2023-10-09T06:47:22.318+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:22.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[141446]: 2023-10-09T06:47:22.521+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:22.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[141446]: 2023-10-09T06:47:22.521+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:23.293 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[132350]: 2023-10-09T06:47:22.993+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:23.293 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:22 smithi150 conmon[132350]: 2023-10-09T06:47:22.993+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:23.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[122886]: 2023-10-09T06:47:23.292+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:23.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[122886]: 2023-10-09T06:47:23.292+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:23.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[141446]: 2023-10-09T06:47:23.496+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:23.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[141446]: 2023-10-09T06:47:23.496+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:24.154 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[132350]: 2023-10-09T06:47:23.976+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:24.155 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:23 smithi150 conmon[132350]: 2023-10-09T06:47:23.976+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:24.401 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T06:47:24.443 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:24.443 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T06:47:24.444 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T06:47:24.444 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T06:47:24.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[141446]: 2023-10-09T06:47:24.473+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:24.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[141446]: 2023-10-09T06:47:24.473+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:24.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[122886]: 2023-10-09T06:47:24.293+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:24.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[122886]: 2023-10-09T06:47:24.293+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:24.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 systemd[1]: Stopping Ceph osd.5 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:25.154 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[132350]: 2023-10-09T06:47:24.991+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:25.154 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[132350]: 2023-10-09T06:47:24.991+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:25.155 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[122886]: 2023-10-09T06:47:24.751+0000 7f72e3c01700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:25.155 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[122886]: 2023-10-09T06:47:24.751+0000 7f72e3c01700 -1 osd.5 604 *** Got signal Terminated *** 2023-10-09T06:47:25.155 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:24 smithi150 conmon[122886]: 2023-10-09T06:47:24.751+0000 7f72e3c01700 -1 osd.5 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:25.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:25 smithi150 conmon[141446]: 2023-10-09T06:47:25.490+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:25.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:25 smithi150 conmon[141446]: 2023-10-09T06:47:25.490+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:25.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:25 smithi150 conmon[122886]: 2023-10-09T06:47:25.268+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:25.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:25 smithi150 conmon[122886]: 2023-10-09T06:47:25.268+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:26.292 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[132350]: 2023-10-09T06:47:26.003+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:26.292 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[132350]: 2023-10-09T06:47:26.003+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:26.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[141446]: 2023-10-09T06:47:26.519+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:26.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[141446]: 2023-10-09T06:47:26.519+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:26.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[122886]: 2023-10-09T06:47:26.291+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:26.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:26 smithi150 conmon[122886]: 2023-10-09T06:47:26.291+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:27.332 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[132350]: 2023-10-09T06:47:27.005+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:27.332 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[132350]: 2023-10-09T06:47:27.005+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:27.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[141446]: 2023-10-09T06:47:27.565+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:27.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[141446]: 2023-10-09T06:47:27.565+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:27.656 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[122886]: 2023-10-09T06:47:27.331+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:27.656 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:27 smithi150 conmon[122886]: 2023-10-09T06:47:27.331+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:28.311 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[132350]: 2023-10-09T06:47:28.029+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:28.311 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[132350]: 2023-10-09T06:47:28.029+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:28.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[141446]: 2023-10-09T06:47:28.538+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:28.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[141446]: 2023-10-09T06:47:28.538+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:28.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[141446]: 2023-10-09T06:47:28.538+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:28.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[122886]: 2023-10-09T06:47:28.310+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:28.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[122886]: 2023-10-09T06:47:28.310+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:28.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[122886]: 2023-10-09T06:47:28.310+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:05.234501+0000 front 2023-10-09T06:47:05.234547+0000 (oldest deadline 2023-10-09T06:47:28.134455+0000) 2023-10-09T06:47:29.356 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[132350]: 2023-10-09T06:47:28.992+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:29.356 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[132350]: 2023-10-09T06:47:28.992+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:29.356 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:28 smithi150 conmon[132350]: 2023-10-09T06:47:28.992+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:29.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[141446]: 2023-10-09T06:47:29.536+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:29.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[141446]: 2023-10-09T06:47:29.536+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:29.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[141446]: 2023-10-09T06:47:29.536+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:29.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[122886]: 2023-10-09T06:47:29.355+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:46.631789+0000 front 2023-10-09T06:46:46.631867+0000 (oldest deadline 2023-10-09T06:47:11.331613+0000) 2023-10-09T06:47:29.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[122886]: 2023-10-09T06:47:29.355+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:55.933318+0000 front 2023-10-09T06:46:55.933240+0000 (oldest deadline 2023-10-09T06:47:21.733473+0000) 2023-10-09T06:47:29.655 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:29 smithi150 conmon[122886]: 2023-10-09T06:47:29.355+0000 7f72e1201700 -1 osd.5 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:05.234501+0000 front 2023-10-09T06:47:05.234547+0000 (oldest deadline 2023-10-09T06:47:28.134455+0000) 2023-10-09T06:47:30.360 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[132350]: 2023-10-09T06:47:30.036+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:30.360 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[132350]: 2023-10-09T06:47:30.036+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:30.360 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[132350]: 2023-10-09T06:47:30.036+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:30.654 INFO:journalctl@ceph.osd.5.smithi150.stdout:Oct 09 06:47:30 smithi150 bash[403792]: 7c00b8a830f4282395fb322c63dc101f7d1ecdadebe39c5b833893cbea3f781e 2023-10-09T06:47:30.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[141446]: 2023-10-09T06:47:30.540+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:30.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[141446]: 2023-10-09T06:47:30.540+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:30.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:30 smithi150 conmon[141446]: 2023-10-09T06:47:30.540+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:31.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[132350]: 2023-10-09T06:47:31.038+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:31.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[132350]: 2023-10-09T06:47:31.038+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:31.328 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[132350]: 2023-10-09T06:47:31.038+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:31.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[141446]: 2023-10-09T06:47:31.556+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:31.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[141446]: 2023-10-09T06:47:31.556+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:31.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:31 smithi150 conmon[141446]: 2023-10-09T06:47:31.556+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:32.207 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T06:47:32.249 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:32.249 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T06:47:32.249 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T06:47:32.249 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T06:47:32.312 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.004+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:32.313 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.004+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:32.313 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.004+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:32.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[141446]: 2023-10-09T06:47:32.560+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:32.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[141446]: 2023-10-09T06:47:32.560+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:32.655 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[141446]: 2023-10-09T06:47:32.560+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:32.655 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 systemd[1]: Stopping Ceph osd.6 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:32.655 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.549+0000 7f64ecfad700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:32.655 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.549+0000 7f64ecfad700 -1 osd.6 604 *** Got signal Terminated *** 2023-10-09T06:47:32.655 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[132350]: 2023-10-09T06:47:32.549+0000 7f64ecfad700 -1 osd.6 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[178500]: ts=2023-10-09T06:47:32.496Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.107:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.107:8765: connect: connection refused" 2023-10-09T06:47:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[178500]: ts=2023-10-09T06:47:32.497Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.107:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.107:8765: connect: connection refused" 2023-10-09T06:47:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[178500]: ts=2023-10-09T06:47:32.501Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.107:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.107:8765: connect: connection refused" 2023-10-09T06:47:32.656 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:32 smithi150 conmon[178500]: ts=2023-10-09T06:47:32.508Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.107:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.107:8765: connect: connection refused" 2023-10-09T06:47:33.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[132350]: 2023-10-09T06:47:33.041+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:33.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[132350]: 2023-10-09T06:47:33.041+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:33.405 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[132350]: 2023-10-09T06:47:33.041+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:33.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[141446]: 2023-10-09T06:47:33.568+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:33.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[141446]: 2023-10-09T06:47:33.568+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:33.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:33 smithi150 conmon[141446]: 2023-10-09T06:47:33.568+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:34.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[132350]: 2023-10-09T06:47:34.077+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:34.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[132350]: 2023-10-09T06:47:34.077+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:34.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[132350]: 2023-10-09T06:47:34.077+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:34.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[141446]: 2023-10-09T06:47:34.611+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:34.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[141446]: 2023-10-09T06:47:34.611+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:34.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:34 smithi150 conmon[141446]: 2023-10-09T06:47:34.611+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:35.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[132350]: 2023-10-09T06:47:35.071+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:35.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[132350]: 2023-10-09T06:47:35.071+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:35.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[132350]: 2023-10-09T06:47:35.071+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:36.088 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[141446]: 2023-10-09T06:47:35.658+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:36.088 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[141446]: 2023-10-09T06:47:35.658+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:36.088 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:35 smithi150 conmon[141446]: 2023-10-09T06:47:35.658+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:36.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[132350]: 2023-10-09T06:47:36.087+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:36.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[132350]: 2023-10-09T06:47:36.087+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:36.404 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[132350]: 2023-10-09T06:47:36.087+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:37.111 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[141446]: 2023-10-09T06:47:36.706+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:37.112 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[141446]: 2023-10-09T06:47:36.706+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:37.112 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[141446]: 2023-10-09T06:47:36.706+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:37.112 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:36 smithi150 conmon[141446]: 2023-10-09T06:47:36.706+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:37.405 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[132350]: 2023-10-09T06:47:37.110+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:43.830945+0000 front 2023-10-09T06:46:43.830881+0000 (oldest deadline 2023-10-09T06:47:10.831434+0000) 2023-10-09T06:47:37.405 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[132350]: 2023-10-09T06:47:37.110+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:56.732680+0000 front 2023-10-09T06:46:56.732787+0000 (oldest deadline 2023-10-09T06:47:22.632608+0000) 2023-10-09T06:47:37.406 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[132350]: 2023-10-09T06:47:37.110+0000 7f64e95ab700 -1 osd.6 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:02.633400+0000 front 2023-10-09T06:47:02.633427+0000 (oldest deadline 2023-10-09T06:47:28.533085+0000) 2023-10-09T06:47:38.075 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[141446]: 2023-10-09T06:47:37.693+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:38.075 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[141446]: 2023-10-09T06:47:37.693+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:38.075 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[141446]: 2023-10-09T06:47:37.693+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:38.075 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:37 smithi150 conmon[141446]: 2023-10-09T06:47:37.693+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:38.724 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:38 smithi150 bash[404381]: fbed7d32aa54ff39c2ed38cc304fb38cdd80b2e206afab94ecaed9bfd89d45f8 2023-10-09T06:47:39.049 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:38 smithi150 conmon[141446]: 2023-10-09T06:47:38.723+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:39.050 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:38 smithi150 conmon[141446]: 2023-10-09T06:47:38.723+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:39.050 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:38 smithi150 conmon[141446]: 2023-10-09T06:47:38.723+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:39.050 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:38 smithi150 conmon[141446]: 2023-10-09T06:47:38.723+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:39.864 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T06:47:39.931 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:39 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.6.service: Succeeded. 2023-10-09T06:47:39.932 INFO:journalctl@ceph.osd.6.smithi150.stdout:Oct 09 06:47:39 smithi150 systemd[1]: Stopped Ceph osd.6 for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:47:39.932 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:39 smithi150 conmon[141446]: 2023-10-09T06:47:39.675+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:39.932 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:39 smithi150 conmon[141446]: 2023-10-09T06:47:39.675+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:39.932 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:39 smithi150 conmon[141446]: 2023-10-09T06:47:39.675+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:39.932 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:39 smithi150 conmon[141446]: 2023-10-09T06:47:39.675+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:39.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:39.945 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T06:47:39.945 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T06:47:39.946 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T06:47:40.240 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 systemd[1]: Stopping Ceph osd.7 for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:40.653 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.239+0000 7fc7d9a96700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T06:47:40.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.239+0000 7fc7d9a96700 -1 osd.7 604 *** Got signal Terminated *** 2023-10-09T06:47:40.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.239+0000 7fc7d9a96700 -1 osd.7 604 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T06:47:40.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.653+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:40.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.653+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:40.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.653+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:40.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:40 smithi150 conmon[141446]: 2023-10-09T06:47:40.653+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:42.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:41 smithi150 conmon[141446]: 2023-10-09T06:47:41.682+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:42.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:41 smithi150 conmon[141446]: 2023-10-09T06:47:41.682+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:42.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:41 smithi150 conmon[141446]: 2023-10-09T06:47:41.682+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:42.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:41 smithi150 conmon[141446]: 2023-10-09T06:47:41.682+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:42.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:42 smithi150 conmon[141446]: 2023-10-09T06:47:42.635+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:42.904 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:42 smithi150 conmon[141446]: 2023-10-09T06:47:42.635+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:42.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:42 smithi150 conmon[141446]: 2023-10-09T06:47:42.635+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:42.905 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:42 smithi150 conmon[141446]: 2023-10-09T06:47:42.635+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:44.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:43 smithi150 conmon[141446]: 2023-10-09T06:47:43.677+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:44.155 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:43 smithi150 conmon[141446]: 2023-10-09T06:47:43.677+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:44.155 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:43 smithi150 conmon[141446]: 2023-10-09T06:47:43.677+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:44.155 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:43 smithi150 conmon[141446]: 2023-10-09T06:47:43.677+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:45.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:44 smithi150 conmon[141446]: 2023-10-09T06:47:44.679+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6806 osd.0 since back 2023-10-09T06:46:47.671496+0000 front 2023-10-09T06:46:47.671509+0000 (oldest deadline 2023-10-09T06:47:11.771539+0000) 2023-10-09T06:47:45.154 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:44 smithi150 conmon[141446]: 2023-10-09T06:47:44.679+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6814 osd.1 since back 2023-10-09T06:46:57.573040+0000 front 2023-10-09T06:46:57.573021+0000 (oldest deadline 2023-10-09T06:47:20.472855+0000) 2023-10-09T06:47:45.155 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:44 smithi150 conmon[141446]: 2023-10-09T06:47:44.679+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6822 osd.2 since back 2023-10-09T06:47:00.473291+0000 front 2023-10-09T06:47:00.473321+0000 (oldest deadline 2023-10-09T06:47:28.073896+0000) 2023-10-09T06:47:45.155 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:44 smithi150 conmon[141446]: 2023-10-09T06:47:44.679+0000 7fc7d7897700 -1 osd.7 604 heartbeat_check: no reply from 172.21.15.107:6830 osd.3 since back 2023-10-09T06:47:13.175976+0000 front 2023-10-09T06:47:13.176049+0000 (oldest deadline 2023-10-09T06:47:36.076031+0000) 2023-10-09T06:47:46.654 INFO:journalctl@ceph.osd.7.smithi150.stdout:Oct 09 06:47:46 smithi150 bash[404954]: 5e4873826583993634faef17705abf494286ba098132063647035e93c7d22fc5 2023-10-09T06:47:47.714 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T06:47:47.753 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:47.753 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T06:47:47.754 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T06:47:47.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T06:47:48.057 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:47 smithi150 systemd[1]: Stopping Ceph prometheus.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:48.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.056Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:47:48.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T06:47:48.405 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T06:47:48.406 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.057Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T06:47:48.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.058Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T06:47:48.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.058Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T06:47:48.407 INFO:journalctl@ceph.prometheus.a.smithi150.stdout:Oct 09 06:47:48 smithi150 conmon[178500]: ts=2023-10-09T06:47:48.058Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T06:47:48.645 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T06:47:48.682 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T06:47:48.683 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T06:47:48.683 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:47:50.151 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:47:49 smithi107 systemd[1]: Stopping Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:50.460 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:47:50 smithi107 conmon[198176]: ts=2023-10-09T06:47:50.177Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T06:47:50.460 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:47:50 smithi107 bash[682097]: 0715bcb46fa19d9241db61283fbe5ef20473f0dcc55ca2305582acde0cd05c1c 2023-10-09T06:47:51.385 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:47:50 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T06:47:51.385 INFO:journalctl@ceph.alertmanager.a.smithi107.stdout:Oct 09 06:47:50 smithi107 systemd[1]: Stopped Ceph alertmanager.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:47:51.386 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:47:51 smithi107 systemd[1]: Stopping Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:47:52.031 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:47:51 smithi107 bash[682388]: 32eddd377f02f6032d9a8b2ebb14f528ed9022108113b5ccc6f5f8c5c1936f5f 2023-10-09T06:47:52.031 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:47:51 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:47:52.032 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:47:51 smithi107 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T06:47:52.032 INFO:journalctl@ceph.node-exporter.a.smithi107.stdout:Oct 09 06:47:51 smithi107 systemd[1]: Stopped Ceph node-exporter.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:48:01.991 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T06:48:03.404 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:48:03 smithi150 systemd[1]: Stopping Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:48:04.154 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:48:03 smithi150 bash[406136]: 6b4c6b51266f7802bc2eea23772890cbf80cd043873bf983fb67ec63394cc0a6 2023-10-09T06:48:04.154 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:48:03 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:48:04.154 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:48:04 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T06:48:04.155 INFO:journalctl@ceph.node-exporter.b.smithi150.stdout:Oct 09 06:48:04 smithi150 systemd[1]: Stopped Ceph node-exporter.b for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:48:04.487 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:48:04 smithi150 systemd[1]: Stopping Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7... 2023-10-09T06:48:05.136 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:48:04 smithi150 bash[406415]: 96db62cb38d01fc96d6785a06c0a792a08ec563c8c0f3426a54991f086275416 2023-10-09T06:48:05.137 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:48:04 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T06:48:05.137 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:48:05 smithi150 systemd[1]: ceph-b7a98308-6667-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T06:48:05.137 INFO:journalctl@ceph.grafana.a.smithi150.stdout:Oct 09 06:48:05 smithi150 systemd[1]: Stopped Ceph grafana.a for b7a98308-6667-11ee-8db5-212e2dc638e7. 2023-10-09T06:48:10.550 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:48:10.587 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T06:48:10.617 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T06:48:10.620 DEBUG:teuthology.misc:Transferring archived files from smithi107:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi107/crash 2023-10-09T06:48:10.621 DEBUG:teuthology.orchestra.run.smithi107:> sudo tar c -f - -C /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:48:10.674 INFO:teuthology.orchestra.run.smithi107.stderr:tar: /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:48:10.674 INFO:teuthology.orchestra.run.smithi107.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:48:10.676 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi150/crash 2023-10-09T06:48:10.677 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T06:48:10.727 INFO:teuthology.orchestra.run.smithi150.stderr:tar: /var/lib/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T06:48:10.727 INFO:teuthology.orchestra.run.smithi150.stderr:tar: Error is not recoverable: exiting now 2023-10-09T06:48:10.728 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T06:48:10.729 DEBUG:teuthology.orchestra.run.smithi107:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T06:48:10.763 INFO:teuthology.orchestra.run.smithi107.stderr:grep: /var/log/ceph/b7a98308-6667-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T06:48:10.767 INFO:tasks.cephadm:Compressing logs... 2023-10-09T06:48:10.768 DEBUG:teuthology.orchestra.run.smithi107:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:48:10.811 DEBUG:teuthology.orchestra.run.smithi150:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T06:48:10.850 INFO:teuthology.orchestra.run.smithi150.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T06:48:10.859 INFO:teuthology.orchestra.run.smithi107.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T07:01:55.226 INFO:tasks.cephadm:Archiving logs... 2023-10-09T07:01:55.251 DEBUG:teuthology.misc:Transferring archived files from smithi107:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi107/log 2023-10-09T07:01:55.252 DEBUG:teuthology.orchestra.run.smithi107:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T07:03:43.709 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi150/log 2023-10-09T07:03:43.710 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T07:04:40.770 INFO:tasks.cephadm:Removing cluster... 2023-10-09T07:04:40.771 DEBUG:teuthology.orchestra.run.smithi107:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 --force 2023-10-09T07:04:42.247 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b7a98308-6667-11ee-8db5-212e2dc638e7 --force 2023-10-09T07:04:43.687 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T07:04:43.687 DEBUG:teuthology.orchestra.run.smithi107:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T07:04:43.709 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T07:04:43.740 INFO:tasks.cephadm:Teardown complete 2023-10-09T07:04:43.741 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T07:04:43.751 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T07:04:43.752 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T07:04:43.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T07:04:43.892 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T07:04:43.902 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T07:04:43.903 DEBUG:teuthology.orchestra.run.smithi107:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:04:43.906 DEBUG:teuthology.orchestra.run.smithi150:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T07:04:43.922 INFO:teuthology.orchestra.run.smithi150.stderr:bash: ntpq: command not found 2023-10-09T07:04:43.923 INFO:teuthology.orchestra.run.smithi107.stderr:bash: ntpq: command not found 2023-10-09T07:04:43.945 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T07:04:43.946 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2023-10-09T07:04:43.946 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 62 -693us[ -693us] +/- 102ms 2023-10-09T07:04:43.946 INFO:teuthology.orchestra.run.smithi150.stdout:^* hv02.front.sepia.ceph.com 3 6 377 64 +209us[ +244us] +/- 65ms 2023-10-09T07:04:43.946 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +307us[ +307us] +/- 72ms 2023-10-09T07:04:43.946 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:04:43.950 INFO:teuthology.orchestra.run.smithi107.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T07:04:43.950 INFO:teuthology.orchestra.run.smithi107.stdout:=============================================================================== 2023-10-09T07:04:43.951 INFO:teuthology.orchestra.run.smithi107.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 61 -736us[ -736us] +/- 102ms 2023-10-09T07:04:43.951 INFO:teuthology.orchestra.run.smithi107.stdout:^* hv02.front.sepia.ceph.com 3 7 377 66 +205us[ +209us] +/- 65ms 2023-10-09T07:04:43.951 INFO:teuthology.orchestra.run.smithi107.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +295us[ +295us] +/- 72ms 2023-10-09T07:04:43.951 INFO:teuthology.orchestra.run.smithi107.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T07:04:43.952 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T07:04:43.963 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T07:04:43.963 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T07:04:43.974 DEBUG:teuthology.orchestra.run.smithi107:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T07:04:44.110 DEBUG:teuthology.orchestra.run.smithi150:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T07:04:44.211 DEBUG:teuthology.orchestra.run.smithi107:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T07:04:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:44.244 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T07:04:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:44.274 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T07:04:44.284 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T07:04:44.295 INFO:teuthology.task.internal:Duration was 5397.121675 seconds 2023-10-09T07:04:44.295 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T07:04:44.304 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T07:04:44.305 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T07:04:44.307 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T07:04:44.378 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T07:04:44.382 INFO:teuthology.orchestra.run.smithi107.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T07:04:44.704 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T07:04:44.704 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi107.front.sepia.ceph.com 2023-10-09T07:04:44.705 DEBUG:teuthology.orchestra.run.smithi107:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T07:04:44.733 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi150.front.sepia.ceph.com 2023-10-09T07:04:44.733 DEBUG:teuthology.orchestra.run.smithi150:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T07:04:44.764 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T07:04:44.764 DEBUG:teuthology.orchestra.run.smithi107:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T07:04:44.775 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T07:04:44.917 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T07:04:44.917 DEBUG:teuthology.orchestra.run.smithi107:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T07:04:44.919 DEBUG:teuthology.orchestra.run.smithi150:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T07:04:45.277 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T07:04:45.288 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T07:04:45.288 DEBUG:teuthology.orchestra.run.smithi107:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T07:04:45.322 DEBUG:teuthology.orchestra.run.smithi150:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T07:04:45.355 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T07:04:45.366 DEBUG:teuthology.orchestra.run.smithi107:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:04:45.368 DEBUG:teuthology.orchestra.run.smithi150:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:04:45.406 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern = core 2023-10-09T07:04:45.432 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = core 2023-10-09T07:04:45.447 DEBUG:teuthology.orchestra.run.smithi107:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:04:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:45.499 DEBUG:teuthology.orchestra.run.smithi150:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T07:04:45.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:45.515 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T07:04:45.526 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T07:04:45.527 DEBUG:teuthology.misc:Transferring archived files from smithi107:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi107 2023-10-09T07:04:45.527 DEBUG:teuthology.orchestra.run.smithi107:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T07:04:45.599 DEBUG:teuthology.misc:Transferring archived files from smithi150:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513/remote/smithi150 2023-10-09T07:04:45.599 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T07:04:45.646 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T07:04:45.646 DEBUG:teuthology.orchestra.run.smithi107:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T07:04:45.648 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T07:04:45.694 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T07:04:45.706 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T07:04:45.706 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T07:04:45.718 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T07:04:45.718 DEBUG:teuthology.orchestra.run.smithi107:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T07:04:45.720 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T07:04:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: 262320 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 07:04 /home/ubuntu/cephtest 2023-10-09T07:04:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: 5898247 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 06:44 /home/ubuntu/cephtest/mnt.0 2023-10-09T07:04:45.734 INFO:teuthology.orchestra.run.smithi107.stdout: 5898252 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 06:44 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T07:04:45.736 INFO:teuthology.orchestra.run.smithi107.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T07:04:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T07:04:45.737 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi107 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T07:04:45.743 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T07:04:45.754 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T07:04:45.766 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi107 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T07:04:45.766 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T07:04:45.773 INFO:teuthology.orchestra.run.smithi150.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 07:04 /home/ubuntu/cephtest 2023-10-09T07:04:45.805 DEBUG:teuthology.nuke:shortname: smithi107 2023-10-09T07:04:45.805 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:04:45.829 DEBUG:teuthology.nuke:shortname: smithi150 2023-10-09T07:04:45.830 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T07:04:45.833 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi107.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:29:18.766103', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T07:04:45.858 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417513', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 05:29:18.764250', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T07:04:45.886 INFO:teuthology.orchestra.console.smithi107:Power off 2023-10-09T07:04:45.887 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T07:04:45.910 DEBUG:teuthology.orchestra.console.smithi107:power off output: Chassis Power Control: Down/Off 2023-10-09T07:04:45.910 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:45.927 INFO:teuthology.orchestra.console.smithi150:Power off 2023-10-09T07:04:45.928 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T07:04:45.936 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is on 2023-10-09T07:04:45.949 DEBUG:teuthology.orchestra.console.smithi150:power off output: Chassis Power Control: Down/Off 2023-10-09T07:04:45.949 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:45.972 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2023-10-09T07:04:49.938 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:49.973 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:50.048 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is on 2023-10-09T07:04:50.083 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2023-10-09T07:04:54.053 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:54.084 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:54.163 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is on 2023-10-09T07:04:54.194 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is off 2023-10-09T07:04:54.295 INFO:teuthology.orchestra.console.smithi150:Power off completed 2023-10-09T07:04:58.166 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T07:04:58.277 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is off 2023-10-09T07:04:58.378 INFO:teuthology.orchestra.console.smithi107:Power off completed 2023-10-09T07:04:58.482 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 5397.121675252914 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=f412d45efb6248bfab995ad413f30745 status: fail success: false 2023-10-09T07:04:58.483 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T07:04:58.568 INFO:teuthology.run:FAIL